diff --git a/setup.cfg b/setup.cfg index d2fb5e0a..27ac9c31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,10 +41,8 @@ test = pytest-cov [tool:pytest] -testpaths = "tests" -markers = - image: run test during image comparison only mode. - hash: run test during hash comparison only mode. +testpaths = tests +norecursedirs = tests/subtests/subtest filterwarnings = error ignore:distutils Version classes are deprecated diff --git a/tests/subtests/hashes/mpl33_ft261.json b/tests/subtests/hashes/mpl33_ft261.json deleted file mode 100644 index 5bec9574..00000000 --- a/tests/subtests/hashes/mpl33_ft261.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" -} diff --git a/tests/subtests/hashes/mpl34_ft261.json b/tests/subtests/hashes/mpl34_ft261.json deleted file mode 100644 index 5bec9574..00000000 --- a/tests/subtests/hashes/mpl34_ft261.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" -} diff --git a/tests/subtests/hashes/mpl35_ft261.json b/tests/subtests/hashes/mpl35_ft261.json deleted file mode 100644 index 5bec9574..00000000 --- a/tests/subtests/hashes/mpl35_ft261.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" -} diff --git a/tests/subtests/result_hashes/mpl33_ft261.json b/tests/subtests/result_hashes/mpl33_ft261.json index 6079e0fe..95904f0e 100644 --- a/tests/subtests/result_hashes/mpl33_ft261.json +++ b/tests/subtests/result_hashes/mpl33_ft261.json @@ -1,19 +1,34 @@ { - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", - "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", - "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", - "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", - "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" } diff --git a/tests/subtests/result_hashes/mpl34_ft261.json b/tests/subtests/result_hashes/mpl34_ft261.json index 6079e0fe..95904f0e 100644 --- a/tests/subtests/result_hashes/mpl34_ft261.json +++ b/tests/subtests/result_hashes/mpl34_ft261.json @@ -1,19 +1,34 @@ { - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", - "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", - "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", - "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", - "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" } diff --git a/tests/subtests/result_hashes/mpl35_ft261.json b/tests/subtests/result_hashes/mpl35_ft261.json index 6079e0fe..95904f0e 100644 --- a/tests/subtests/result_hashes/mpl35_ft261.json +++ b/tests/subtests/result_hashes/mpl35_ft261.json @@ -1,19 +1,34 @@ { - "subtests.subtest.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", - "subtests.subtest.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", - "subtests.subtest.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", - "subtests.subtest.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", - "subtests.subtest.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", - "subtests.subtest.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", - "subtests.subtest.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", - "subtests.subtest.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", - "subtests.subtest.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", - "subtests.subtest.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", - "subtests.subtest.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", - "subtests.subtest.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", - "subtests.subtest.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", - "subtests.subtest.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", - "subtests.subtest.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", - "subtests.subtest.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a" + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "b92c5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "567f014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "b6673bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "e37bd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_functions.test_hmissing_imatch": "592d12cc2d5749a6607bbf98d715b95c06a2af6572a7e298bcae349648b9997e", + "subtests.subtest.test_functions.test_hmissing_idiff": "9e98dbd2027525c776212daa061180b4fc40ad12dfc2cdfe4b86694ede14e0c3", + "subtests.subtest.test_functions.test_hmissing_idiffshape": "5534324f9da5c1c104c3ef3435dc6fa9792c0d0d8b762fad5d7f81abd91fbb89", + "subtests.subtest.test_functions.test_hmissing_imissing": "2d15274f0e9b44f1c16e6b237710cd36a3de5c5b7596ef0e65e7a33ce4624cf4", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "f26ca66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "0a0514c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "588ad00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "a78ad7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "2bb430367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" } diff --git a/tests/subtests/subtest/__init__.py b/tests/subtests/subtest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/subtests/baseline/test_hdiff_idiff.png b/tests/subtests/subtest/baseline/test_hdiff_idiff.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_idiff.png rename to tests/subtests/subtest/baseline/test_hdiff_idiff.png diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetup.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetup.png new file mode 100644 index 00000000..a1962b97 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetup.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetupclass.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetupclass.png new file mode 100644 index 00000000..349e4c5d Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testcasewithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testclass.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclass.png new file mode 100644 index 00000000..e5346158 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupclass.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupclass.png new file mode 100644 index 00000000..c2cb95a4 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupmethod.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupmethod.png new file mode 100644 index 00000000..a1962b97 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hdiff_idiff_testclasswithsetupmethod.png differ diff --git a/tests/subtests/baseline/test_hdiff_idiff_tolerance.png b/tests/subtests/subtest/baseline/test_hdiff_idiff_tolerance.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_idiff_tolerance.png rename to tests/subtests/subtest/baseline/test_hdiff_idiff_tolerance.png diff --git a/tests/subtests/baseline/test_hdiff_idiffshape.png b/tests/subtests/subtest/baseline/test_hdiff_idiffshape.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_idiffshape.png rename to tests/subtests/subtest/baseline/test_hdiff_idiffshape.png diff --git a/tests/subtests/baseline/test_hdiff_imatch.png b/tests/subtests/subtest/baseline/test_hdiff_imatch.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_removetext.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_removetext.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_removetext.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_removetext.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_savefig.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_savefig.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_savefig.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_savefig.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_style.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_style.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_style.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_style.png diff --git a/tests/subtests/baseline/test_hdiff_imatch_tolerance.png b/tests/subtests/subtest/baseline/test_hdiff_imatch_tolerance.png similarity index 100% rename from tests/subtests/baseline/test_hdiff_imatch_tolerance.png rename to tests/subtests/subtest/baseline/test_hdiff_imatch_tolerance.png diff --git a/tests/subtests/baseline/test_hmatch_idiff.png b/tests/subtests/subtest/baseline/test_hmatch_idiff.png similarity index 100% rename from tests/subtests/baseline/test_hmatch_idiff.png rename to tests/subtests/subtest/baseline/test_hmatch_idiff.png diff --git a/tests/subtests/baseline/test_hmatch_idiffshape.png b/tests/subtests/subtest/baseline/test_hmatch_idiffshape.png similarity index 100% rename from tests/subtests/baseline/test_hmatch_idiffshape.png rename to tests/subtests/subtest/baseline/test_hmatch_idiffshape.png diff --git a/tests/subtests/baseline/test_hmatch_imatch.png b/tests/subtests/subtest/baseline/test_hmatch_imatch.png similarity index 100% rename from tests/subtests/baseline/test_hmatch_imatch.png rename to tests/subtests/subtest/baseline/test_hmatch_imatch.png diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_first.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_first.png new file mode 100644 index 00000000..bdce3923 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_first.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_second.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_second.png new file mode 100644 index 00000000..df6c3983 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_multiplefigures_second.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetup.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetup.png new file mode 100644 index 00000000..460a6d39 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetup.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetupclass.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetupclass.png new file mode 100644 index 00000000..83437d00 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testcasewithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclass.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclass.png new file mode 100644 index 00000000..c41a213b Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithfixture.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithfixture.png new file mode 100644 index 00000000..9c45afe5 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithfixture.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupclass.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupclass.png new file mode 100644 index 00000000..1a185036 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupclass.png differ diff --git a/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupmethod.png b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupmethod.png new file mode 100644 index 00000000..460a6d39 Binary files /dev/null and b/tests/subtests/subtest/baseline/test_hmatch_imatch_testclasswithsetupmethod.png differ diff --git a/tests/subtests/baseline/test_hmissing_idiff.png b/tests/subtests/subtest/baseline/test_hmissing_idiff.png similarity index 100% rename from tests/subtests/baseline/test_hmissing_idiff.png rename to tests/subtests/subtest/baseline/test_hmissing_idiff.png diff --git a/tests/subtests/baseline/test_hmissing_idiffshape.png b/tests/subtests/subtest/baseline/test_hmissing_idiffshape.png similarity index 100% rename from tests/subtests/baseline/test_hmissing_idiffshape.png rename to tests/subtests/subtest/baseline/test_hmissing_idiffshape.png diff --git a/tests/subtests/baseline/test_hmissing_imatch.png b/tests/subtests/subtest/baseline/test_hmissing_imatch.png similarity index 100% rename from tests/subtests/baseline/test_hmissing_imatch.png rename to tests/subtests/subtest/baseline/test_hmissing_imatch.png diff --git a/tests/subtests/subtest/hashes/mpl33_ft261.json b/tests/subtests/subtest/hashes/mpl33_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl33_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl34_ft261.json b/tests/subtests/subtest/hashes/mpl34_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl34_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/hashes/mpl35_ft261.json b/tests/subtests/subtest/hashes/mpl35_ft261.json new file mode 100644 index 00000000..ed5bb0e0 --- /dev/null +++ b/tests/subtests/subtest/hashes/mpl35_ft261.json @@ -0,0 +1,30 @@ +{ + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": "8c4fa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": "d1ffa5f95b2a2b424ad9e441b6aeeced9d51a52fe7f19870eedcdfdcf6d39818", + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": "ee0c290d66d17e4e40991c31eee0c55bd724922a3ecb66f7300b2ce8abf3d204", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": "3f8d2c1facdf6e34e43b4343abd888bd5ee9d2d451ce285e62bf0c65d6cdcdce", + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": "9a6b008431f0b47a8866d87a9f1e9352fd73407a3ee2dd756a8ccbfa5a835a08", + "subtests.subtest.test_functions.test_hmatch_imatch": "d21af7f9a2c1cbaf3c9bca3598f1b32b36891ac9d5db47e81a7bcaa342f7d4fc", + "subtests.subtest.test_functions.test_hmatch_idiff": "085fcb22e9d6cfbb2bb6e0efbf749fa598be27e837c348130adc21a6dc2fc5fe", + "subtests.subtest.test_functions.test_hmatch_idiffshape": "a8f866c3b765e274c217d49ba72c9ce3bd4b316491ffd34a124ef03643ce45b8", + "subtests.subtest.test_functions.test_hmatch_imissing": "f06e910b6c80db28e1eb08fdb8e1ab9211434498c134d00820900a13a4f2568c", + "subtests.subtest.test_functions.test_hdiff_imatch": "d1ff5c6bc631fbdaffa23d3d57fc027768fcded889f3b269941da859110ce282", + "subtests.subtest.test_functions.test_hdiff_idiff": "d1ff014f73cdfea555e46a29aaac43c4394c3c4c21998e54971edb773eee6c95", + "subtests.subtest.test_functions.test_hdiff_idiffshape": "d1ff3bafdcc8350c612bc925269fc4332dd9062a6399701067863b178568b219", + "subtests.subtest.test_functions.test_hdiff_imissing": "d1ffd5868d14547557653c051d23d3fd48d198d3f59006dc5ba390433d6670ff", + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": "d1ffa66a7c02ae64c8b2512021e0450cbe64c084c9d5f7e2600a7342a559c0b1", + "subtests.subtest.test_special.test_hdiff_imatch_savefig": "d1ff14c35f1da18de3f4ceb1901501e5a8a5a0d18eb8a7b4db5cfde170b57423", + "subtests.subtest.test_special.test_hdiff_imatch_style": "d1ffd00c4b99c6087d04f84ca071a5997b4ecf76cf859ce3548634e67841a79b", + "subtests.subtest.test_special.test_hdiff_imatch_removetext": "d1ffd7512c6d886262b1bcb4501374bfc61ef8569d24930b0258dab08e6eca9a", + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": "d1ff30367cdf04b7b5f3af8c50652964ff27def9fc0a75a68b4775ba1eb8e827", + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": "af966da7f2e57908211d559ff0a24a52c363c3fabca9e810138ad60e3fd2bcc9", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": "d1ff0363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": "56d00363330cb7d959d6cfdc1335901716e841f54014cb6e657d1c4721384252", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": "d1ff6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602", + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": "5f1e6e2b775a1b1d1e9e20a4d929586a9accb08b2fddaec8a24fb4acfc83c602" +} diff --git a/tests/subtests/subtest/helpers.py b/tests/subtests/subtest/helpers.py new file mode 100644 index 00000000..91b00ed7 --- /dev/null +++ b/tests/subtests/subtest/helpers.py @@ -0,0 +1,30 @@ +from functools import wraps + +import matplotlib.pyplot as plt +import numpy as np +import pytest + + +def plot(line, **kwargs): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot(line, **kwargs) + return fig + + +def bar(height, **kwargs): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + x = np.arange(len(height)) + ax.bar(x, height, align="center", **kwargs) + return fig + + +def figure_test(test_function, **kwargs): + @pytest.mark.image + @pytest.mark.hash + @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}}, **kwargs) + @wraps(test_function) + def test_wrapper(*args, **kwargs): + return test_function(*args, **kwargs) + return test_wrapper diff --git a/tests/subtests/subtest/pytest.ini b/tests/subtests/subtest/pytest.ini new file mode 100644 index 00000000..faaaaad6 --- /dev/null +++ b/tests/subtests/subtest/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +markers = + image: run test during image comparison only mode. + hash: run test during hash comparison only mode. +filterwarnings = + error + ignore:distutils Version classes are deprecated + ignore:the imp module is deprecated in favour of importlib diff --git a/tests/subtests/subtest/test_classes.py b/tests/subtests/subtest/test_classes.py new file mode 100644 index 00000000..6e361276 --- /dev/null +++ b/tests/subtests/subtest/test_classes.py @@ -0,0 +1,93 @@ +import matplotlib.pyplot as plt +import pytest + +from pytest_mpl.plugin import switch_backend + +from .helpers import bar, figure_test + + +class TestClass: + + @figure_test + def test_hmatch_imatch_testclass(self): + return bar([1, 2, 3, 4]) + + @figure_test + def test_hdiff_idiff_testclass(self): + return bar([1, 3, 2, 4]) + + +class TestClassWithSetupMethod: + + def setup_method(self, method): + self.x = [4, 2, 3, 1] + + @figure_test + def test_hmatch_imatch_testclasswithsetupmethod(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testclasswithsetupmethod(self): + return bar(self.x) + + +class TestClassWithSetupClass: + + @classmethod + def setup_class(cls): + cls.x = [1, 0, 3, 2] + + @figure_test + def test_hmatch_imatch_testclasswithsetupclass(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testclasswithsetupclass(self): + return bar(self.x) + + +@pytest.fixture() +def figure_axes(): + with plt.style.context("classic", after_reset=True), switch_backend("agg"): + fig = bar([3, 0, 4, 1]) + yield fig.gca() + # Should not appear in test result + fig.gca().plot([3, 0, 4, 1], c="yellow") + + +class TestClassWithFixture: + @figure_test + def test_hmatch_imatch_testclasswithfixture(self, figure_axes): + figure_axes.plot([4, 1, 5, 2], c="red") + return figure_axes.get_figure() + + +def generate_two_figures(): + for num, line in ((1, [1, 0, 1, 0]), (2, [0, 1, 0, 1])): + plt.close(num) + fig = plt.figure(num) + ax = fig.add_subplot(1, 1, 1) + ax.plot(line) + + +class TestMultipleFigures: + """ + Test figures are accessible, and can be passed to individual functions. + + See https://github.com/matplotlib/pytest-mpl/issues/133 + """ + @classmethod + def setup_class(cls): + # setting style and backend inside decorator has no effect + # because figures are generated outside the test function + with plt.style.context("classic", after_reset=True), switch_backend("agg"): + generate_two_figures() + cls.figs = [plt.figure(i) for i in (1, 2)] + + @figure_test + def test_hmatch_imatch_multiplefigures_first(self): + return self.figs[0] + + @figure_test + def test_hmatch_imatch_multiplefigures_second(self): + return self.figs[1] diff --git a/tests/subtests/subtest.py b/tests/subtests/subtest/test_functions.py similarity index 50% rename from tests/subtests/subtest.py rename to tests/subtests/subtest/test_functions.py index c02abea6..045a1dfc 100644 --- a/tests/subtests/subtest.py +++ b/tests/subtests/subtest/test_functions.py @@ -1,18 +1,12 @@ -import matplotlib.pyplot as plt import pytest - -def plot(line, **kwargs): - fig = plt.figure() - ax = fig.add_subplot(1, 1, 1) - ax.plot(line, **kwargs) - return fig - +from .helpers import plot # ### Test all permutations of: # baseline hash: match, diff, or missing # baseline image: match, diff, or missing + # hash match @pytest.mark.image @@ -84,65 +78,3 @@ def test_hmissing_idiffshape(): @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {'Software': None}}) def test_hmissing_imissing(): return plot([2, 4, 3, 1]) - - -# ### Specialized tests - -# Tolerance: high to force image match -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(tolerance=200, savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_imatch_tolerance(): - return plot([1, 2, 3, 4], linestyle='--') - - -# Tolerance: non-default to verify option recorded in JSON -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(tolerance=3, savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_idiff_tolerance(): - return plot([1, 2, 3, 4], linestyle='--') - - -# Savefig kwargs -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r', 'metadata': {'Software': None}}) -def test_hdiff_imatch_savefig(): - return plot([1, 2, 3, 4]) - - -# TODO: Implement these path altering tests later -# # Different baseline directory -# # TODO: Test with a remote `baseline_dir` -# @pytest.mark.mpl_image_compare(baseline_dir='baseline/other') -# def test_hdiff_imatch_baselinedir(): -# return plot([4, 2, 1, 4]) -# -# -# # Different filename -# @pytest.mark.mpl_image_compare(filename='test_hdiff_imatch_filename_other.png') -# def test_hdiff_imatch_filename(): -# return plot([4, 2, 1, 4]) -# -# -# # Different hash library -# @pytest.mark.mpl_image_compare(hash_library='hashes/other/other.json') -# def test_hdiff_imatch_hashlibrary(): -# return plot([4, 2, 1, 4]) - - -# Different style -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(style='fivethirtyeight', savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_imatch_style(): - return plot([4, 2, 1, 4]) - - -# Remove text -@pytest.mark.image -@pytest.mark.hash -@pytest.mark.mpl_image_compare(remove_text=True, savefig_kwargs={'metadata': {'Software': None}}) -def test_hdiff_imatch_removetext(): - return plot([4, 2, 1, 4]) diff --git a/tests/subtests/subtest/test_special.py b/tests/subtests/subtest/test_special.py new file mode 100644 index 00000000..3659c26b --- /dev/null +++ b/tests/subtests/subtest/test_special.py @@ -0,0 +1,65 @@ +import pytest + +from .helpers import plot + +# ### Specialized tests + + +# Tolerance: high to force image match +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(tolerance=200, savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_imatch_tolerance(): + return plot([1, 2, 3, 4], linestyle='--') + + +# Tolerance: non-default to verify option recorded in JSON +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(tolerance=3, savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_idiff_tolerance(): + return plot([1, 2, 3, 4], linestyle='--') + + +# Savefig kwargs +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r', 'metadata': {'Software': None}}) +def test_hdiff_imatch_savefig(): + return plot([1, 2, 3, 4]) + + +# TODO: Implement these path altering tests later +# # Different baseline directory +# # TODO: Test with a remote `baseline_dir` +# @pytest.mark.mpl_image_compare(baseline_dir='baseline/other') +# def test_hdiff_imatch_baselinedir(): +# return plot([4, 2, 1, 4]) +# +# +# # Different filename +# @pytest.mark.mpl_image_compare(filename='test_hdiff_imatch_filename_other.png') +# def test_hdiff_imatch_filename(): +# return plot([4, 2, 1, 4]) +# +# +# # Different hash library +# @pytest.mark.mpl_image_compare(hash_library='hashes/other/other.json') +# def test_hdiff_imatch_hashlibrary(): +# return plot([4, 2, 1, 4]) + + +# Different style +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(style='fivethirtyeight', savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_imatch_style(): + return plot([4, 2, 1, 4]) + + +# Remove text +@pytest.mark.image +@pytest.mark.hash +@pytest.mark.mpl_image_compare(remove_text=True, savefig_kwargs={'metadata': {'Software': None}}) +def test_hdiff_imatch_removetext(): + return plot([4, 2, 1, 4]) diff --git a/tests/subtests/subtest/test_unittest.py b/tests/subtests/subtest/test_unittest.py new file mode 100644 index 00000000..6c66e00c --- /dev/null +++ b/tests/subtests/subtest/test_unittest.py @@ -0,0 +1,43 @@ +import unittest + +from .helpers import bar, figure_test + + +class TestCase(unittest.TestCase): + + @figure_test + def test_hmatch_imatch_testclass(self): + return bar([1, 2, 3, 4]) + + @figure_test + def test_hdiff_idiff_testclass(self): + return bar([1, 3, 2, 4]) + + +class TestCaseWithSetUp(unittest.TestCase): + + def setUp(self): + self.x = [4, 2, 3, 1] + + @figure_test + def test_hmatch_imatch_testcasewithsetup(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testcasewithsetup(self): + return bar(self.x) + + +class TestCaseWithSetUpClass(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.x = [4, 3, 2, 4] + + @figure_test + def test_hmatch_imatch_testcasewithsetupclass(self): + return bar(self.x) + + @figure_test + def test_hdiff_idiff_testcasewithsetupclass(self): + return bar(self.x) diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index 61efc2dd..cbea11ff 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": null, @@ -12,33 +12,150 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -47,11 +164,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": null, @@ -64,33 +181,33 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -99,11 +216,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": null, @@ -116,33 +233,33 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -151,11 +268,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": null, @@ -168,20 +285,33 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "passed", "image_status": "match", "hash_status": null, @@ -194,7 +324,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": null, @@ -207,7 +337,72 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": null, diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json index 04a1d288..f5f2cf7f 100644 --- a/tests/subtests/summaries/test_generate.json +++ b/tests/subtests/summaries/test_generate.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -12,7 +12,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -25,7 +25,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -38,7 +38,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -51,7 +51,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -64,7 +64,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -77,7 +77,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -90,7 +90,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -103,7 +103,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -116,7 +116,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -129,7 +129,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -142,7 +142,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -155,7 +155,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -168,7 +168,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -181,7 +181,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -194,7 +194,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", @@ -207,7 +207,202 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json index b8b686a0..aaba44b9 100644 --- a/tests/subtests/summaries/test_generate_hashes_only.json +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -12,33 +12,150 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -47,11 +164,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -64,33 +181,33 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -99,11 +216,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -116,33 +233,33 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -151,11 +268,11 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -168,20 +285,33 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -194,7 +324,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": "generated", @@ -207,7 +337,72 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": "generated", diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json index 553b5589..9e701649 100644 --- a/tests/subtests/summaries/test_generate_images_only.json +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -12,7 +12,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -25,7 +25,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -38,7 +38,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -51,7 +51,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -64,7 +64,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -77,7 +77,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -90,7 +90,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -103,7 +103,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -116,7 +116,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -129,7 +129,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -142,7 +142,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -155,7 +155,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -168,7 +168,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -181,7 +181,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -194,7 +194,7 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, @@ -207,7 +207,202 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index cdd18c90..f463019d 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": null, "hash_status": "match", @@ -12,94 +12,289 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hdiff_imatch": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": null, "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json index d4366938..5d25715f 100644 --- a/tests/subtests/summaries/test_html_generate.json +++ b/tests/subtests/summaries/test_html_generate.json @@ -1,10 +1,10 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -12,12 +12,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -25,12 +25,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -38,12 +38,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -51,12 +51,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -64,12 +64,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -77,12 +77,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -90,12 +90,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -103,12 +103,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -116,12 +116,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -129,12 +129,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -142,12 +142,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -155,12 +155,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imissing/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -168,12 +168,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -181,12 +181,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -194,12 +194,12 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -207,12 +207,207 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", "hash_status": "generated", "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json index eb455d56..3036ca89 100644 --- a/tests/subtests/summaries/test_html_generate_hashes_only.json +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -1,44 +1,161 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -47,50 +164,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -99,50 +216,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "generated", @@ -151,72 +268,150 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "generated", "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": "generated", "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": null } diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json index 9d1947eb..0b1c3eb8 100644 --- a/tests/subtests/summaries/test_html_generate_images_only.json +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -1,10 +1,10 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -12,12 +12,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -25,12 +25,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -38,12 +38,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -51,12 +51,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -64,12 +64,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -77,12 +77,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -90,12 +90,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -103,12 +103,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -116,12 +116,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -129,12 +129,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -142,12 +142,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -155,12 +155,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imissing/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -168,12 +168,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -181,12 +181,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -194,12 +194,12 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, @@ -207,12 +207,207 @@ "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "skipped", "image_status": "generated", "hash_status": null, "status_msg": "Skipped test, since generating image.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_idiffshape": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": null, diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 789795c0..c613126c 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": null, "hash_status": "match", @@ -8,98 +8,293 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hdiff_imatch": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": null, "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { "status": "failed", "image_status": null, "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": null, + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index fa10a4fb..04534f58 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,44 +1,161 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -47,50 +164,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -99,50 +216,50 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": null, @@ -151,72 +268,150 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_special.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": "match", + "hash_status": null, + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": null, + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", "baseline_hash": null, "result_hash": null }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { "status": "passed", "image_status": "match", "hash_status": null, "status_msg": "Image comparison passed.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": null, "result_hash": null } diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 24bfd2fa..62aa6269 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,5 +1,5 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": null, "hash_status": "match", @@ -12,7 +12,111 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { "status": "passed", "image_status": null, "hash_status": "match", @@ -25,7 +129,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "passed", "image_status": null, "hash_status": "match", @@ -38,7 +142,7 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "passed", "image_status": null, "hash_status": "match", @@ -51,172 +155,263 @@ "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hmatch_imissing": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": "match", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": null, + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index 4e8f4250..e59d399d 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,44 +1,161 @@ { - "subtests.subtest.test_hmatch_imatch": { + "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass": { "status": "passed", "image_status": "match", "hash_status": "match", "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", - "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "result_image": "subtests.subtest.test_classes.TestClass.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiff": { + "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClass\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClass.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hmatch_imatch_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupMethod\\.test_hdiff_idiff_testclasswithsetupmethod\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupMethod.test_hdiff_idiff_testclasswithsetupmethod/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hmatch_imatch_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_classes\\.TestClassWithSetupClass\\.test_hdiff_idiff_testclasswithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result-failed-diff.png", + "rms": 24.563779228775037, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithSetupClass.test_hdiff_idiff_testclasswithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestClassWithFixture.test_hmatch_imatch_testclasswithfixture/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_first/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_classes.TestMultipleFigures.test_hmatch_imatch_multiplefigures_second/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_functions.test_hmatch_imatch/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_functions.test_hmatch_idiff": { "status": "passed", "image_status": "diff", "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmatch_idiff/result-failed-diff.png", "rms": 24.618234716477044, "tolerance": 2, - "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_idiffshape": { + "subtests.subtest.test_functions.test_hmatch_idiffshape": { "status": "passed", "image_status": "diff", "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmatch_idiffshape/baseline.png", + "baseline_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmatch_imissing": { + "subtests.subtest.test_functions.test_hmatch_imissing": { "status": "passed", "image_status": "missing", "hash_status": "match", @@ -47,176 +164,254 @@ "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmatch_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch": { + "subtests.subtest.test_functions.test_hdiff_imatch": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imatch/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff": { + "subtests.subtest.test_functions.test_hdiff_idiff": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hdiff_idiff/result-failed-diff.png", "rms": 24.620185259705547, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiff/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiffshape": { + "subtests.subtest.test_functions.test_hdiff_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hdiff_idiffshape/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_idiffshape\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_idiffshape/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imissing": { + "subtests.subtest.test_functions.test_hdiff_imissing": { "status": "failed", "image_status": "missing", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_functions\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hdiff_imissing/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imatch": { + "subtests.subtest.test_functions.test_hmissing_imatch": { "status": "failed", "image_status": "match", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imatch/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiff": { + "subtests.subtest.test_functions.test_hmissing_idiff": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", - "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", - "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.6[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_functions.test_hmissing_idiff/result-failed-diff.png", "rms": 24.6217358806762, "tolerance": 2, - "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiff/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_idiffshape": { + "subtests.subtest.test_functions.test_hmissing_idiffshape": { "status": "failed", "image_status": "diff", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", - "baseline_image": "subtests.subtest.test_hmissing_idiffshape/baseline.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_idiffshape' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nError: Image dimensions did not match\\.\n Expected shape: \\(300, 400\\)\n .*baseline\\.png\n Actual shape: \\(600, 800\\)\n .*result\\.png", + "baseline_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/baseline.png", "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_idiffshape/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_idiffshape/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hmissing_imissing": { + "subtests.subtest.test_functions.test_hmissing_imissing": { "status": "failed", "image_status": "missing", "hash_status": "missing", - "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_functions\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is ###_RESULT_HASH_###\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, "rms": null, "tolerance": null, - "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "result_image": "subtests.subtest.test_functions.test_hmissing_imissing/result.png", "baseline_hash": null, "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_tolerance": { + "subtests.subtest.test_special.test_hdiff_imatch_tolerance": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, "tolerance": 200, - "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_idiff_tolerance": { + "subtests.subtest.test_special.test_hdiff_idiff_tolerance": { "status": "failed", "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", - "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", - "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 24.576566752362574, "tolerance": 3, - "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_idiff_tolerance/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_savefig": { + "subtests.subtest.test_special.test_hdiff_imatch_savefig": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_savefig/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_style": { + "subtests.subtest.test_special.test_hdiff_imatch_style": { "status": "failed", "image_status": "match", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_style/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_special.test_hdiff_imatch_removetext": { + "status": "failed", + "image_status": "match", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_special\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "baseline_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_special.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass": { + "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCase\\.test_hdiff_idiff_testclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result-failed-diff.png", + "rms": 24.536840442259244, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCase.test_hdiff_idiff_testclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "result_image": "subtests.subtest.test_unittest.TestCase.test_hmatch_imatch_testclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" }, - "subtests.subtest.test_hdiff_imatch_removetext": { + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup": { "status": "failed", + "image_status": "diff", + "hash_status": "diff", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUp\\.test_hdiff_idiff_testcasewithsetup\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result-failed-diff.png", + "rms": 24.537034749650488, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hdiff_idiff_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup": { + "status": "passed", "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUp.test_hmatch_imatch_testcasewithsetup/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass": { + "status": "failed", + "image_status": "diff", "hash_status": "diff", - "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", - "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "status_msg": "REGEX:Hash ###_RESULT_HASH_### doesn't match hash ###_BASELINE_HASH_### in library .*\\.json for test subtests\\.subtest\\.test_unittest\\.TestCaseWithSetUpClass\\.test_hdiff_idiff_testcasewithsetupclass\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 24\\.5[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/baseline.png", + "diff_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result-failed-diff.png", + "rms": 24.520257314106637, + "tolerance": 2, + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hdiff_idiff_testcasewithsetupclass/result.png", + "baseline_hash": "###_BASELINE_HASH_###", + "result_hash": "###_RESULT_HASH_###" + }, + "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass": { + "status": "passed", + "image_status": "match", + "hash_status": "match", + "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", + "baseline_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/baseline.png", "diff_image": null, "rms": null, "tolerance": 2, - "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "result_image": "subtests.subtest.test_unittest.TestCaseWithSetUpClass.test_hmatch_imatch_testcasewithsetupclass/result.png", "baseline_hash": "###_BASELINE_HASH_###", "result_hash": "###_RESULT_HASH_###" } diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 4993fa9f..6bf4c8d1 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -6,7 +6,6 @@ import subprocess from pathlib import Path -import matplotlib import matplotlib.ft2font import pytest from packaging.version import Version @@ -18,10 +17,10 @@ MPL_VERSION = Version(matplotlib.__version__) FTV = matplotlib.ft2font.__freetype_version__.replace('.', '') VERSION_ID = f"mpl{MPL_VERSION.major}{MPL_VERSION.minor}_ft{FTV}" -HASH_LIBRARY = Path(__file__).parent / 'hashes' / (VERSION_ID + ".json") +HASH_LIBRARY = Path(__file__).parent / 'subtest' / 'hashes' / (VERSION_ID + ".json") RESULT_LIBRARY = Path(__file__).parent / 'result_hashes' / (VERSION_ID + ".json") HASH_LIBRARY_FLAG = rf'--mpl-hash-library={HASH_LIBRARY}' -FULL_BASELINE_PATH = Path(__file__).parent / 'baseline' +FULL_BASELINE_PATH = Path(__file__).parent / 'subtest' / 'baseline' BASELINE_IMAGES_FLAG_REL = ['--mpl-baseline-path=baseline', '--mpl-baseline-relative'] BASELINE_IMAGES_FLAG_ABS = rf'--mpl-baseline-path={FULL_BASELINE_PATH}' @@ -30,7 +29,7 @@ HASH_COMPARISON_MODE = ["-k", "hash"] # HYBRID_MODE = [] -TEST_FILE = Path(__file__).parent / 'subtest.py' +TEST_FILE = Path(__file__).parent / 'subtest' # Global settings to update baselines when running pytest # Note: when updating baseline make sure you don't commit "fixes"