diff --git a/src/manifests/test_run_manifest.py b/src/manifests/test_report_manifest.py similarity index 94% rename from src/manifests/test_run_manifest.py rename to src/manifests/test_report_manifest.py index 863629a2a0..73a5740c5a 100644 --- a/src/manifests/test_run_manifest.py +++ b/src/manifests/test_report_manifest.py @@ -10,9 +10,9 @@ from manifests.component_manifest import Component, ComponentManifest, Components -class TestRunManifest(ComponentManifest['TestRunManifest', 'TestComponents']): +class TestReportManifest(ComponentManifest['TestReportManifest', 'TestComponents']): """ - TestRunManifest contains the aggregated test results for the components. + TestReportManifest contains the aggregated test results for the components. The format for schema version 1.0 is: schema-version: '1.0' @@ -149,7 +149,7 @@ def __to_dict__(self) -> dict: } -TestRunManifest.VERSIONS = {"1.0": TestRunManifest} +TestReportManifest.VERSIONS = {"1.0": TestReportManifest} TestComponent.__test__ = False # type: ignore[attr-defined] -TestRunManifest.__test__ = False # type: ignore[attr-defined] +TestReportManifest.__test__ = False # type: ignore[attr-defined] diff --git a/src/report_workflow/README.md b/src/report_workflow/README.md new file mode 100644 index 0000000000..e348cc1c6b --- /dev/null +++ b/src/report_workflow/README.md @@ -0,0 +1,24 @@ +#### Generate test-report manifest for each test. +As the name specifies, the test report workflow helps to automatically generate a consolidated report of the tests run at distribution level along with commands to reproduce the error and associated failures. + +*Usage* +``` +./report.sh --artifact-paths opensearch=<...> opensearch-dashboards=<...> --test-run-id <...> --test-type integ-test --base-path <...> +``` +e.g. +``` +./report.sh manifests/2.9.0/opensearch-2.9.0-test.yml -p opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.9.0/8172/linux/x64/tar --test-run-id 5328 --test-type integ-test --base-path https://ci.opensearch.org/ci/dbc/integ-test/2.9.0/8172/linux/x64/tar +``` +The following options are available. + +| name | description | +|---------------------------------|--------------------------------------------------------------------------------| +| test-manifest-path | Specify a test manifest path. | +| -p, --artifact-paths | Artifact paths of distributions used for testing. | +| --base-path | Base paths of testing logs. | +| --test-type | Type of tests report generates on. | +| --output-path | Specify the path location for the test-report manifest. | +| --test-run-id | Specify the unique execution id that matches the id of the test. | +| --component | Specify a specific component or components instead of the entire distribution. | +| --verbose | Show more verbose output. | + diff --git a/src/report_workflow/report_args.py b/src/report_workflow/report_args.py index 9b85d5663d..9a8b807a5b 100644 --- a/src/report_workflow/report_args.py +++ b/src/report_workflow/report_args.py @@ -30,7 +30,7 @@ def __init__(self) -> None: parser.add_argument("--base-path", type=str, default="", help="Specify base paths for the integration test logs.") parser.add_argument("--test-type", type=str, default="integ-test", help="Specify test type of this.") - parser.add_argument("--output-path", type=str, help="Specify the path location for the test-run manifest.") + parser.add_argument("--output-path", type=str, help="Specify the path location for the test-report manifest.") parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test") parser.add_argument("--component", type=str, dest="components", nargs='*', help="Test a specific component or components instead of the entire distribution.") parser.add_argument( diff --git a/src/report_workflow/test_run_runner.py b/src/report_workflow/test_report_runner.py similarity index 88% rename from src/report_workflow/test_run_runner.py rename to src/report_workflow/test_report_runner.py index 2af4f7d189..f53cb413bd 100644 --- a/src/report_workflow/test_run_runner.py +++ b/src/report_workflow/test_report_runner.py @@ -15,22 +15,22 @@ import yaml from manifests.test_manifest import TestManifest -from manifests.test_run_manifest import TestRunManifest +from manifests.test_report_manifest import TestReportManifest from report_workflow.report_args import ReportArgs -class TestRunRunner: +class TestReportRunner: args: ReportArgs test_manifest: TestManifest tests_dir: str - test_run_manifest: TestRunManifest + test_report_manifest: TestReportManifest test_run_data: dict def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None: self.args = args self.base_path = args.base_path self.test_manifest = test_manifest - self.test_run_data = self.test_run_manifest_data_template("manifest") + self.test_run_data = self.test_report_manifest_data_template("manifest") self.product_name = test_manifest.__to_dict__().get("name") self.name = self.product_name.replace(" ", "-").lower() self.components = self.args.components @@ -63,13 +63,13 @@ def update_test_run_data(self) -> dict: return test_run_data def generate_report(self, data: dict, output_dir: str) -> Any: - test_run_manifest = TestRunManifest(data) - test_run_manifetest_run_manifest_file = os.path.join(output_dir, "test-run.yml") - logging.info(f"Generating test-run.yml in {output_dir}") - return test_run_manifest.to_file(test_run_manifetest_run_manifest_file) + test_report_manifest = TestReportManifest(data) + test_report_manifest_file = os.path.join(output_dir, "test-report.yml") + logging.info(f"Generating test-report.yml in {output_dir}") + return test_report_manifest.to_file(test_report_manifest_file) def component_entry(self, component_name: str) -> Any: - component = self.test_run_manifest_data_template("component") + component = self.test_report_manifest_data_template("component") component["name"] = component_name component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name) @@ -102,7 +102,7 @@ def component_entry(self, component_name: str) -> Any: component["configs"].append(config_dict) return component - def test_run_manifest_data_template(self, template_type: str) -> Any: + def test_report_manifest_data_template(self, template_type: str) -> Any: templates = { "manifest": { "schema-version": "1.0", @@ -134,4 +134,4 @@ def generate_test_command(test_type: str, test_manifest_path: str, artifacts_pat return command -TestRunRunner.__test__ = False # type:ignore +TestReportRunner.__test__ = False # type:ignore diff --git a/src/run_test_report.py b/src/run_test_report.py index f9bd14daa9..1ae5adc3ba 100644 --- a/src/run_test_report.py +++ b/src/run_test_report.py @@ -10,7 +10,7 @@ from manifests.test_manifest import TestManifest from report_workflow.report_args import ReportArgs -from report_workflow.test_run_runner import TestRunRunner +from report_workflow.test_report_runner import TestReportRunner from system import console @@ -21,11 +21,11 @@ def main() -> Any: test_manifest = TestManifest.from_path(args.test_manifest_path) - test_run_runner = TestRunRunner(args, test_manifest) + test_report_runner = TestReportRunner(args, test_manifest) - test_run_data = test_run_runner.update_data() + test_report_data = test_report_runner.update_data() - test_run = test_run_runner.generate_report(test_run_data, args.output_path or os.getcwd()) + test_run = test_report_runner.generate_report(test_report_data, args.output_path or os.getcwd()) return test_run diff --git a/tests/test_run_test_report.py b/tests/test_run_test_report.py index fe5a080612..6cd76fbbb5 100644 --- a/tests/test_run_test_report.py +++ b/tests/test_run_test_report.py @@ -39,7 +39,7 @@ def test_usage(self, *mocks: Any) -> None: self.assertTrue(out.startswith("usage:")) @patch("argparse._sys.argv", ["run_test_report.py", TEST_MANIFEST_PATH, "-p", "opensearch=foo"]) - @patch('run_test_report.TestRunRunner') + @patch('run_test_report.TestReportRunner') def test_main(self, runner_mock: Mock, *mocks: Any) -> None: main() diff --git a/tests/tests_manifests/test_test_run_manifest.py b/tests/tests_manifests/test_test_run_manifest.py index 203bde3c53..2723bf8fd7 100644 --- a/tests/tests_manifests/test_test_run_manifest.py +++ b/tests/tests_manifests/test_test_run_manifest.py @@ -10,16 +10,16 @@ import yaml -from manifests.test_run_manifest import TestRunManifest +from manifests.test_report_manifest import TestReportManifest -class TestTestRunManifest(unittest.TestCase): +class TestTestReportManifest(unittest.TestCase): def setUp(self) -> None: self.maxDiff = None self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data")) self.manifest_filename = os.path.join(self.data_path, "test-run.yml") - self.manifest = TestRunManifest.from_path(self.manifest_filename) + self.manifest = TestReportManifest.from_path(self.manifest_filename) def test_test_run(self) -> None: self.assertEqual(self.manifest.name, "OpenSearch") diff --git a/tests/tests_report_workflow/test_test_run_runner.py b/tests/tests_report_workflow/test_test_report_runner.py similarity index 79% rename from tests/tests_report_workflow/test_test_run_runner.py rename to tests/tests_report_workflow/test_test_report_runner.py index b8d1a50461..a6c1bde1ea 100644 --- a/tests/tests_report_workflow/test_test_run_runner.py +++ b/tests/tests_report_workflow/test_test_report_runner.py @@ -11,10 +11,11 @@ from unittest.mock import MagicMock, call, mock_open, patch from manifests.test_manifest import TestManifest -from report_workflow.test_run_runner import TestRunRunner +from report_workflow.test_report_runner import TestReportRunner +from system.temporary_directory import TemporaryDirectory -class TestTestRunRunner(unittest.TestCase): +class TestTestReportRunner(unittest.TestCase): TEST_MANIFEST_PATH = os.path.join( os.path.dirname(__file__), "data", "test_manifest.yml" ) @@ -35,12 +36,35 @@ def test_runner_init(self, report_args_mock: MagicMock, test_manifest_mock: Magi report_args_mock.test_run_id = 123 report_args_mock.test_type = "integ-test" - test_run_runner = TestRunRunner(report_args_mock, self.TEST_MANIFEST) + test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST) self.assertEqual(test_run_runner.name, "opensearch") self.assertEqual(test_run_runner.test_run_id, 123) self.assertEqual(test_run_runner.test_type, "integ-test") self.assertEqual(test_run_runner.test_manifest_path, self.TEST_MANIFEST_PATH) + @patch("yaml.safe_load") + @patch("urllib.request.urlopen") + @patch("validators.url") + @patch("report_workflow.report_args.ReportArgs") + def test_generate_file(self, report_args_mock: MagicMock, validators_mock: MagicMock, urlopen_mock: MagicMock, yaml_safe_load_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock" + report_args_mock.test_type = "integ-test" + + validators_mock.return_value = True + yaml_safe_load_mock.return_value = {"test_result": "PASS"} + urlopen_mock.return_value = MagicMock() + + test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST) + test_run_runner_data = test_run_runner.update_data() + + with TemporaryDirectory() as path: + output_path = os.path.join(path.name, "test-report.yml") + test_run_runner.generate_report(test_run_runner_data, path.name) + self.assertTrue(os.path.isfile(output_path)) + @patch("report_workflow.report_args.ReportArgs") @patch("manifests.test_manifest.TestManifest") def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, test_manifest_mock: MagicMock) -> None: @@ -49,7 +73,7 @@ def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, te report_args_mock.test_run_id = 123 report_args_mock.test_type = "integ-test" - test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() + test_run_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=foo/bar"])) self.assertEqual(test_run_dict.get("TestType"), "integ-test") self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH) @@ -64,7 +88,7 @@ def test_runner_update_test_run_data_url(self, report_args_mock: MagicMock, test report_args_mock.test_run_id = 123 report_args_mock.test_type = "integ-test" - test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() + test_run_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=https://foo/bar"])) self.assertEqual(test_run_dict.get("TestType"), "integ-test") self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH) @@ -86,7 +110,7 @@ def test_runner_component_entry_url(self, report_args_mock: MagicMock, validator yaml_safe_load_mock.return_value = {"test_result": "PASS"} urlopen_mock.return_value = MagicMock() - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") urlopen_mock.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml')]) self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") @@ -107,7 +131,7 @@ def test_runner_component_entry_local(self, report_args_mock: MagicMock, validat yaml_safe_load_mock.return_value = {"test_result": "PASS"} mock_open.return_value = MagicMock() - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')]) self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") @@ -125,7 +149,7 @@ def test_runner_component_entry_url_invalid(self, report_args_mock: MagicMock, v validators_mock.return_value = True - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "URL not available") @@ -145,7 +169,7 @@ def test_runner_component_entry_local_invalid(self, report_args_mock: MagicMock, yaml_safe_load_mock.return_value = {"test_result": "PASS"} mock_open.side_effect = FileNotFoundError - test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + test_run_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')]) self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available") self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security")