From f473c0e914e5b3739a76691b251ec8dd58633c00 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 1 Feb 2024 16:00:00 -0800 Subject: [PATCH 1/7] update tree comparison for tests to be order independent for children --- .../test_bottom_folder.py | 0 .../expected_discovery_test_output.py | 22 +++++++++---------- .../expected_execution_test_output.py | 14 ++++++------ .../tests/pytestadapter/test_discovery.py | 16 ++++++++------ .../tests/pytestadapter/test_execution.py | 4 ++-- .../helpers.py => tree_comparison_helper.py} | 9 +++----- .../expected_discovery_test_output.py | 4 +++- .../tests/unittestadapter/test_discovery.py | 4 +++- .../tests/unittestadapter/test_utils.py | 4 +++- 9 files changed, 41 insertions(+), 36 deletions(-) rename pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/{z_nested_folder_one => nested_folder_one}/test_bottom_folder.py (100%) rename pythonFiles/tests/{unittestadapter/helpers.py => tree_comparison_helper.py} (80%) diff --git a/pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py b/pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/nested_folder_one/test_bottom_folder.py similarity index 100% rename from pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py rename to pythonFiles/tests/pytestadapter/.data/dual_level_nested_folder/nested_folder_one/test_bottom_folder.py diff --git a/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py b/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py index d4e91f56b5fe..2d86710e776b 100644 --- a/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py +++ b/pythonFiles/tests/pytestadapter/expected_discovery_test_output.py @@ -317,7 +317,7 @@ # └── test_top_folder.py # └── test_top_function_t # └── test_top_function_f -# └── z_nested_folder_one +# └── nested_folder_one # └── test_bottom_folder.py # └── test_bottom_function_t # └── test_bottom_function_f @@ -326,14 +326,14 @@ TEST_DATA_PATH / "dual_level_nested_folder" / "test_top_folder.py" ) -test_z_nested_folder_one_path = ( - TEST_DATA_PATH / "dual_level_nested_folder" / "z_nested_folder_one" +test_nested_folder_one_path = ( + TEST_DATA_PATH / "dual_level_nested_folder" / "nested_folder_one" ) test_bottom_folder_path = ( TEST_DATA_PATH / "dual_level_nested_folder" - / "z_nested_folder_one" + / "nested_folder_one" / "test_bottom_folder.py" ) @@ -392,10 +392,10 @@ ], }, { - "name": "z_nested_folder_one", - "path": os.fspath(test_z_nested_folder_one_path), + "name": "nested_folder_one", + "path": os.fspath(test_nested_folder_one_path), "type_": "folder", - "id_": os.fspath(test_z_nested_folder_one_path), + "id_": os.fspath(test_nested_folder_one_path), "children": [ { "name": "test_bottom_folder.py", @@ -412,11 +412,11 @@ ), "type_": "test", "id_": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t", test_bottom_folder_path, ), "runID": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t", test_bottom_folder_path, ), }, @@ -429,11 +429,11 @@ ), "type_": "test", "id_": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", test_bottom_folder_path, ), "runID": get_absolute_test_id( - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", test_bottom_folder_path, ), }, diff --git a/pythonFiles/tests/pytestadapter/expected_execution_test_output.py b/pythonFiles/tests/pytestadapter/expected_execution_test_output.py index cf8997a252d3..44f3d3d0abce 100644 --- a/pythonFiles/tests/pytestadapter/expected_execution_test_output.py +++ b/pythonFiles/tests/pytestadapter/expected_execution_test_output.py @@ -308,7 +308,7 @@ # └── test_top_folder.py # └── test_top_function_t: success # └── test_top_function_f: failure -# └── z_nested_folder_one +# └── nested_folder_one # └── test_bottom_folder.py # └── test_bottom_function_t: success # └── test_bottom_function_f: failure @@ -318,7 +318,7 @@ dual_level_nested_folder_bottom_path = ( TEST_DATA_PATH / "dual_level_nested_folder" - / "z_nested_folder_one" + / "nested_folder_one" / "test_bottom_folder.py" ) dual_level_nested_folder_execution_expected_output = { @@ -345,11 +345,11 @@ "subtest": None, }, get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_t", dual_level_nested_folder_bottom_path, ): { "test": get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_t", dual_level_nested_folder_bottom_path, ), "outcome": "success", @@ -358,11 +358,11 @@ "subtest": None, }, get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_f", dual_level_nested_folder_bottom_path, ): { "test": get_absolute_test_id( - "z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "nested_folder_one/test_bottom_folder.py::test_bottom_function_f", dual_level_nested_folder_bottom_path, ), "outcome": "failure", @@ -479,7 +479,7 @@ dual_level_nested_folder_bottom_path = ( TEST_DATA_PATH / "dual_level_nested_folder" - / "z_nested_folder_one" + / "nested_folder_one" / "test_bottom_folder.py" ) unittest_folder_add_path = TEST_DATA_PATH / "unittest_folder" / "test_add.py" diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 2630ddef68b0..1b7e8b0fa01f 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -6,6 +6,8 @@ import pytest +from pythonFiles.tests.tree_comparison_helper import is_same_tree + from . import expected_discovery_test_output from .helpers import TEST_DATA_PATH, runner, runner_with_cwd @@ -195,7 +197,7 @@ def test_pytest_collect(file, expected_const): assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) assert actual_item.get("status") == "success" assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) - assert actual_item.get("tests") == expected_const + assert is_same_tree(actual_item.get("tests"), expected_const) def test_pytest_root_dir(): @@ -219,9 +221,9 @@ def test_pytest_root_dir(): assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) assert actual_item.get("status") == "success" assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") - assert ( - actual_item.get("tests") - == expected_discovery_test_output.root_with_config_expected_output + assert is_same_tree( + actual_item.get("tests"), + expected_discovery_test_output.root_with_config_expected_output, ) @@ -245,7 +247,7 @@ def test_pytest_config_file(): assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) assert actual_item.get("status") == "success" assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") - assert ( - actual_item.get("tests") - == expected_discovery_test_output.root_with_config_expected_output + assert is_same_tree( + actual_item.get("tests"), + expected_discovery_test_output.root_with_config_expected_output, ) diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index 767d54a6cabe..dd32b61fa262 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -193,8 +193,8 @@ def test_bad_id_error_execution(): [ "dual_level_nested_folder/test_top_folder.py::test_top_function_t", "dual_level_nested_folder/test_top_folder.py::test_top_function_f", - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_t", - "dual_level_nested_folder/z_nested_folder_one/test_bottom_folder.py::test_bottom_function_f", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t", + "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", ], expected_execution_test_output.dual_level_nested_folder_execution_expected_output, ), diff --git a/pythonFiles/tests/unittestadapter/helpers.py b/pythonFiles/tests/tree_comparison_helper.py similarity index 80% rename from pythonFiles/tests/unittestadapter/helpers.py rename to pythonFiles/tests/tree_comparison_helper.py index 303d021368f7..edf6aa8ff869 100644 --- a/pythonFiles/tests/unittestadapter/helpers.py +++ b/pythonFiles/tests/tree_comparison_helper.py @@ -1,10 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import pathlib - -TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" - def is_same_tree(tree1, tree2) -> bool: """Helper function to test if two test trees are the same. @@ -17,8 +13,9 @@ def is_same_tree(tree1, tree2) -> bool: # Compare child test nodes if they exist, otherwise compare test items. if "children" in tree1 and "children" in tree2: - children1 = tree1["children"] - children2 = tree2["children"] + # sort children by path before comparing since order doesn't matter of children + children1 = sorted(tree1["children"], key=lambda x: x["path"]) + children2 = sorted(tree2["children"], key=lambda x: x["path"]) # Compare test nodes. if len(children1) != len(children2): diff --git a/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py b/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py index db509ebeca3c..1007a8f42dfd 100644 --- a/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py +++ b/pythonFiles/tests/unittestadapter/expected_discovery_test_output.py @@ -3,9 +3,11 @@ import os from unittestadapter.pvsc_utils import TestNodeTypeEnum -from .helpers import TEST_DATA_PATH import pathlib +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" + + skip_unittest_folder_discovery_output = { "path": os.fspath(TEST_DATA_PATH / "unittest_skip"), "name": "unittest_skip", diff --git a/pythonFiles/tests/unittestadapter/test_discovery.py b/pythonFiles/tests/unittestadapter/test_discovery.py index 4249ca4faef2..53bbfdbca4cb 100644 --- a/pythonFiles/tests/unittestadapter/test_discovery.py +++ b/pythonFiles/tests/unittestadapter/test_discovery.py @@ -10,7 +10,9 @@ from unittestadapter.pvsc_utils import TestNodeTypeEnum, parse_unittest_args from . import expected_discovery_test_output -from .helpers import TEST_DATA_PATH, is_same_tree +from pythonFiles.tests.tree_comparison_helper import is_same_tree + +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" @pytest.mark.parametrize( diff --git a/pythonFiles/tests/unittestadapter/test_utils.py b/pythonFiles/tests/unittestadapter/test_utils.py index 9fe2af8256cd..d991e87c16b1 100644 --- a/pythonFiles/tests/unittestadapter/test_utils.py +++ b/pythonFiles/tests/unittestadapter/test_utils.py @@ -15,7 +15,9 @@ get_test_case, ) -from .helpers import TEST_DATA_PATH, is_same_tree +from pythonFiles.tests.tree_comparison_helper import is_same_tree + +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" @pytest.mark.parametrize( From 21346da35171f944400b8f4618aaea108da1eb24 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 5 Feb 2024 13:52:16 -0800 Subject: [PATCH 2/7] add import path for tests --- pythonFiles/tests/pytestadapter/test_discovery.py | 5 +++++ pythonFiles/tests/unittestadapter/test_discovery.py | 5 +++++ pythonFiles/tests/unittestadapter/test_utils.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 1b7e8b0fa01f..13fa46a9395a 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -1,11 +1,16 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os +import pathlib import shutil +import sys from typing import Any, Dict, List, Optional import pytest +script_dir = pathlib.Path(__file__).parent.parent.parent +sys.path.append(os.fspath(script_dir)) + from pythonFiles.tests.tree_comparison_helper import is_same_tree from . import expected_discovery_test_output diff --git a/pythonFiles/tests/unittestadapter/test_discovery.py b/pythonFiles/tests/unittestadapter/test_discovery.py index 53bbfdbca4cb..5c9faa9cc077 100644 --- a/pythonFiles/tests/unittestadapter/test_discovery.py +++ b/pythonFiles/tests/unittestadapter/test_discovery.py @@ -3,12 +3,17 @@ import os import pathlib +import sys from typing import List import pytest from unittestadapter.discovery import discover_tests from unittestadapter.pvsc_utils import TestNodeTypeEnum, parse_unittest_args +script_dir = pathlib.Path(__file__).parent.parent.parent +sys.path.append(os.fspath(script_dir)) + + from . import expected_discovery_test_output from pythonFiles.tests.tree_comparison_helper import is_same_tree diff --git a/pythonFiles/tests/unittestadapter/test_utils.py b/pythonFiles/tests/unittestadapter/test_utils.py index d991e87c16b1..b236ed833136 100644 --- a/pythonFiles/tests/unittestadapter/test_utils.py +++ b/pythonFiles/tests/unittestadapter/test_utils.py @@ -3,6 +3,7 @@ import os import pathlib +import sys import unittest import pytest @@ -15,6 +16,9 @@ get_test_case, ) +script_dir = pathlib.Path(__file__).parent.parent.parent +sys.path.append(os.fspath(script_dir)) + from pythonFiles.tests.tree_comparison_helper import is_same_tree TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" From dea7827c7f91a6072c8ae16ae6d97dc36977323d Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 7 Feb 2024 09:46:15 -0800 Subject: [PATCH 3/7] printing for error handling --- pythonFiles/tests/pytestadapter/test_discovery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 13fa46a9395a..8ba831dc463b 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -10,6 +10,7 @@ script_dir = pathlib.Path(__file__).parent.parent.parent sys.path.append(os.fspath(script_dir)) +print("SCRIPT DIR: ", script_dir) from pythonFiles.tests.tree_comparison_helper import is_same_tree From b647b413d6bc71c91b9fdc11398008a2e6e9dd58 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 7 Feb 2024 09:58:24 -0800 Subject: [PATCH 4/7] redo path --- pythonFiles/tests/pytestadapter/test_discovery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 8ba831dc463b..197ab19eca4c 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -8,11 +8,11 @@ import pytest -script_dir = pathlib.Path(__file__).parent.parent.parent +script_dir = pathlib.Path(__file__).parent.parent sys.path.append(os.fspath(script_dir)) print("SCRIPT DIR: ", script_dir) -from pythonFiles.tests.tree_comparison_helper import is_same_tree +from tree_comparison_helper import is_same_tree from . import expected_discovery_test_output from .helpers import TEST_DATA_PATH, runner, runner_with_cwd From 885ddf7d84598da82c0cec85b64852dc490580a6 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 7 Feb 2024 10:26:25 -0800 Subject: [PATCH 5/7] retry path --- pythonFiles/tests/pytestadapter/test_discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 197ab19eca4c..e84c2102fe73 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -12,7 +12,7 @@ sys.path.append(os.fspath(script_dir)) print("SCRIPT DIR: ", script_dir) -from tree_comparison_helper import is_same_tree +from tests.tree_comparison_helper import is_same_tree from . import expected_discovery_test_output from .helpers import TEST_DATA_PATH, runner, runner_with_cwd From cafc20c45153cb5e5ce84a8e94a3a5494cb26106 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 7 Feb 2024 10:31:16 -0800 Subject: [PATCH 6/7] update path p2 --- pythonFiles/tests/unittestadapter/test_discovery.py | 4 ++-- pythonFiles/tests/unittestadapter/test_utils.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pythonFiles/tests/unittestadapter/test_discovery.py b/pythonFiles/tests/unittestadapter/test_discovery.py index 5c9faa9cc077..a68774d3f2dc 100644 --- a/pythonFiles/tests/unittestadapter/test_discovery.py +++ b/pythonFiles/tests/unittestadapter/test_discovery.py @@ -10,12 +10,12 @@ from unittestadapter.discovery import discover_tests from unittestadapter.pvsc_utils import TestNodeTypeEnum, parse_unittest_args -script_dir = pathlib.Path(__file__).parent.parent.parent +script_dir = pathlib.Path(__file__).parent.parent sys.path.append(os.fspath(script_dir)) from . import expected_discovery_test_output -from pythonFiles.tests.tree_comparison_helper import is_same_tree +from tests.tree_comparison_helper import is_same_tree TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" diff --git a/pythonFiles/tests/unittestadapter/test_utils.py b/pythonFiles/tests/unittestadapter/test_utils.py index b236ed833136..d5f6fbbe9f18 100644 --- a/pythonFiles/tests/unittestadapter/test_utils.py +++ b/pythonFiles/tests/unittestadapter/test_utils.py @@ -16,10 +16,11 @@ get_test_case, ) -script_dir = pathlib.Path(__file__).parent.parent.parent +script_dir = pathlib.Path(__file__).parent.parent sys.path.append(os.fspath(script_dir)) -from pythonFiles.tests.tree_comparison_helper import is_same_tree +from tests.tree_comparison_helper import is_same_tree + TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" From c341e2fcd63ae7e78efa602ecf71113e0098fbc0 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 8 Feb 2024 11:38:41 -0800 Subject: [PATCH 7/7] remove print --- pythonFiles/tests/pytestadapter/test_discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index e84c2102fe73..9956b82a6345 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -10,7 +10,6 @@ script_dir = pathlib.Path(__file__).parent.parent sys.path.append(os.fspath(script_dir)) -print("SCRIPT DIR: ", script_dir) from tests.tree_comparison_helper import is_same_tree