diff --git a/.gitignore b/.gitignore index f703e34173fd..1b47f15705bb 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,8 @@ dist/** package.nls.*.json l10n/ python-env-tools/** +# coverage files produced as test output +python_files/tests/*/.data/.coverage* +python_files/tests/*/.data/*/.coverage* +src/testTestingRootWkspc/coverageWorkspace/.coverage + diff --git a/build/functional-test-requirements.txt b/build/functional-test-requirements.txt index d45208f671f4..5c3a9e3116ed 100644 --- a/build/functional-test-requirements.txt +++ b/build/functional-test-requirements.txt @@ -1,3 +1,5 @@ # List of requirements for functional tests versioneer numpy +pytest +pytest-cov diff --git a/build/test-requirements.txt b/build/test-requirements.txt index 4229104ddcc9..c5c18a048f56 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -27,3 +27,7 @@ namedpipe; platform_system == "Windows" # typing for Django files django-stubs + +# for coverage +coverage +pytest-cov diff --git a/python_files/tests/pytestadapter/.data/coverage_gen/__init__.py b/python_files/tests/pytestadapter/.data/coverage_gen/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/coverage_gen/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/python_files/tests/pytestadapter/.data/coverage_gen/reverse.py b/python_files/tests/pytestadapter/.data/coverage_gen/reverse.py new file mode 100644 index 000000000000..cb6755a3a369 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/coverage_gen/reverse.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +def reverse_string(s): + if s is None or s == "": + return "Error: Input is None" + return s[::-1] + +def reverse_sentence(sentence): + if sentence is None or sentence == "": + return "Error: Input is None" + words = sentence.split() + reversed_words = [reverse_string(word) for word in words] + return " ".join(reversed_words) + +# Example usage +if __name__ == "__main__": + sample_string = "hello" + print(reverse_string(sample_string)) # Output: "olleh" diff --git a/python_files/tests/pytestadapter/.data/coverage_gen/test_reverse.py b/python_files/tests/pytestadapter/.data/coverage_gen/test_reverse.py new file mode 100644 index 000000000000..e7319f143608 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/coverage_gen/test_reverse.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from .reverse import reverse_sentence, reverse_string + + +def test_reverse_sentence(): + """ + Tests the reverse_sentence function to ensure it correctly reverses each word in a sentence. + + Test cases: + - "hello world" should be reversed to "olleh dlrow" + - "Python is fun" should be reversed to "nohtyP si nuf" + - "a b c" should remain "a b c" as each character is a single word + """ + assert reverse_sentence("hello world") == "olleh dlrow" + assert reverse_sentence("Python is fun") == "nohtyP si nuf" + assert reverse_sentence("a b c") == "a b c" + +def test_reverse_sentence_error(): + assert reverse_sentence("") == "Error: Input is None" + assert reverse_sentence(None) == "Error: Input is None" + + +def test_reverse_string(): + assert reverse_string("hello") == "olleh" + assert reverse_string("Python") == "nohtyP" + # this test specifically does not cover the error cases diff --git a/python_files/tests/pytestadapter/helpers.py b/python_files/tests/pytestadapter/helpers.py index 4f6631a44c00..991c7efbc60c 100644 --- a/python_files/tests/pytestadapter/helpers.py +++ b/python_files/tests/pytestadapter/helpers.py @@ -203,6 +203,26 @@ def runner_with_cwd(args: List[str], path: pathlib.Path) -> Optional[List[Dict[s return runner_with_cwd_env(args, path, {}) +def split_array_at_item(arr: List[str], item: str) -> Tuple[List[str], List[str]]: + """ + Splits an array into two subarrays at the specified item. + + Args: + arr (List[str]): The array to be split. + item (str): The item at which to split the array. + + Returns: + Tuple[List[str], List[str]]: A tuple containing two subarrays. The first subarray includes the item and all elements before it. The second subarray includes all elements after the item. If the item is not found, the first subarray is the original array and the second subarray is empty. + """ + if item in arr: + index = arr.index(item) + before = arr[: index + 1] + after = arr[index + 1 :] + return before, after + else: + return arr, [] + + def runner_with_cwd_env( args: List[str], path: pathlib.Path, env_add: Dict[str, str] ) -> Optional[List[Dict[str, Any]]]: @@ -217,10 +237,34 @@ def runner_with_cwd_env( # If we are running Django, generate a unittest-specific pipe name. process_args = [sys.executable, *args] pipe_name = generate_random_pipe_name("unittest-discovery-test") + elif "_TEST_VAR_UNITTEST" in env_add: + before_args, after_ids = split_array_at_item(args, "*test*.py") + process_args = [sys.executable, *before_args] + pipe_name = generate_random_pipe_name("unittest-execution-test") + test_ids_pipe = os.fspath( + script_dir / "tests" / "unittestadapter" / ".data" / "coverage_ex" / "10943021.txt" + ) + env_add.update({"RUN_TEST_IDS_PIPE": test_ids_pipe}) + test_ids_arr = after_ids + with open(test_ids_pipe, "w") as f: # noqa: PTH123 + f.write("\n".join(test_ids_arr)) else: process_args = [sys.executable, "-m", "pytest", "-p", "vscode_pytest", "-s", *args] pipe_name = generate_random_pipe_name("pytest-discovery-test") + if "COVERAGE_ENABLED" in env_add and "_TEST_VAR_UNITTEST" not in env_add: + process_args = [ + sys.executable, + "-m", + "pytest", + "-p", + "vscode_pytest", + "--cov=.", + "--cov-branch", + "-s", + *args, + ] + # Generate pipe name, pipe name specific per OS type. # Windows design diff --git a/python_files/tests/pytestadapter/test_coverage.py b/python_files/tests/pytestadapter/test_coverage.py new file mode 100644 index 000000000000..31e2be24437e --- /dev/null +++ b/python_files/tests/pytestadapter/test_coverage.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import os +import pathlib +import sys + +script_dir = pathlib.Path(__file__).parent.parent +sys.path.append(os.fspath(script_dir)) + +from .helpers import ( # noqa: E402 + TEST_DATA_PATH, + runner_with_cwd_env, +) + + +def test_simple_pytest_coverage(): + """ + Test coverage payload is correct for simple pytest example. Output of coverage run is below. + + Name Stmts Miss Branch BrPart Cover + --------------------------------------------------- + __init__.py 0 0 0 0 100% + reverse.py 13 3 8 2 76% + test_reverse.py 11 0 0 0 100% + --------------------------------------------------- + TOTAL 24 3 8 2 84% + + """ + args = [] + env_add = {"COVERAGE_ENABLED": "True"} + cov_folder_path = TEST_DATA_PATH / "coverage_gen" + actual = runner_with_cwd_env(args, cov_folder_path, env_add) + assert actual + coverage = actual[-1] + assert coverage + results = coverage["result"] + assert results + assert len(results) == 3 + focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_gen" / "reverse.py")) + assert focal_function_coverage + assert focal_function_coverage.get("lines_covered") is not None + assert focal_function_coverage.get("lines_missed") is not None + assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14, 17} + assert set(focal_function_coverage.get("lines_missed")) == {18, 19, 6} + assert ( + focal_function_coverage.get("executed_branches") > 0 + ), "executed_branches are a number greater than 0." + assert ( + focal_function_coverage.get("total_branches") > 0 + ), "total_branches are a number greater than 0." diff --git a/python_files/tests/unittestadapter/.data/coverage_ex/__init__.py b/python_files/tests/unittestadapter/.data/coverage_ex/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/python_files/tests/unittestadapter/.data/coverage_ex/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/python_files/tests/unittestadapter/.data/coverage_ex/reverse.py b/python_files/tests/unittestadapter/.data/coverage_ex/reverse.py new file mode 100644 index 000000000000..4840b7d05bf3 --- /dev/null +++ b/python_files/tests/unittestadapter/.data/coverage_ex/reverse.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +def reverse_string(s): + if s is None or s == "": + return "Error: Input is None" + return s[::-1] + +def reverse_sentence(sentence): + if sentence is None or sentence == "": + return "Error: Input is None" + words = sentence.split() + reversed_words = [reverse_string(word) for word in words] + return " ".join(reversed_words) diff --git a/python_files/tests/unittestadapter/.data/coverage_ex/test_reverse.py b/python_files/tests/unittestadapter/.data/coverage_ex/test_reverse.py new file mode 100644 index 000000000000..2521e3dc1935 --- /dev/null +++ b/python_files/tests/unittestadapter/.data/coverage_ex/test_reverse.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import unittest +from reverse import reverse_sentence, reverse_string + +class TestReverseFunctions(unittest.TestCase): + + def test_reverse_sentence(self): + """ + Tests the reverse_sentence function to ensure it correctly reverses each word in a sentence. + + Test cases: + - "hello world" should be reversed to "olleh dlrow" + - "Python is fun" should be reversed to "nohtyP si nuf" + - "a b c" should remain "a b c" as each character is a single word + """ + self.assertEqual(reverse_sentence("hello world"), "olleh dlrow") + self.assertEqual(reverse_sentence("Python is fun"), "nohtyP si nuf") + self.assertEqual(reverse_sentence("a b c"), "a b c") + + def test_reverse_sentence_error(self): + self.assertEqual(reverse_sentence(""), "Error: Input is None") + self.assertEqual(reverse_sentence(None), "Error: Input is None") + + def test_reverse_string(self): + self.assertEqual(reverse_string("hello"), "olleh") + self.assertEqual(reverse_string("Python"), "nohtyP") + # this test specifically does not cover the error cases + +if __name__ == '__main__': + unittest.main() diff --git a/python_files/tests/unittestadapter/test_coverage.py b/python_files/tests/unittestadapter/test_coverage.py new file mode 100644 index 000000000000..0089e9ae5504 --- /dev/null +++ b/python_files/tests/unittestadapter/test_coverage.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import pathlib +import sys + +sys.path.append(os.fspath(pathlib.Path(__file__).parent)) + +python_files_path = pathlib.Path(__file__).parent.parent.parent +sys.path.insert(0, os.fspath(python_files_path)) +sys.path.insert(0, os.fspath(python_files_path / "lib" / "python")) + +from tests.pytestadapter import helpers # noqa: E402 + +TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" + + +def test_basic_coverage(): + """This test runs on a simple django project with three tests, two of which pass and one that fails.""" + coverage_ex_folder: pathlib.Path = TEST_DATA_PATH / "coverage_ex" + execution_script: pathlib.Path = python_files_path / "unittestadapter" / "execution.py" + test_ids = [ + "test_reverse.TestReverseFunctions.test_reverse_sentence", + "test_reverse.TestReverseFunctions.test_reverse_sentence_error", + "test_reverse.TestReverseFunctions.test_reverse_string", + ] + argv = [os.fsdecode(execution_script), "--udiscovery", "-vv", "-s", ".", "-p", "*test*.py"] + argv = argv + test_ids + + actual = helpers.runner_with_cwd_env( + argv, + coverage_ex_folder, + {"COVERAGE_ENABLED": os.fspath(coverage_ex_folder), "_TEST_VAR_UNITTEST": "True"}, + ) + + assert actual + coverage = actual[-1] + assert coverage + results = coverage["result"] + assert results + assert len(results) == 3 + focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_ex" / "reverse.py")) + assert focal_function_coverage + assert focal_function_coverage.get("lines_covered") is not None + assert focal_function_coverage.get("lines_missed") is not None + assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14} + assert set(focal_function_coverage.get("lines_missed")) == {6} + assert ( + focal_function_coverage.get("executed_branches") > 0 + ), "executed_branches are a number greater than 0." + assert ( + focal_function_coverage.get("total_branches") > 0 + ), "total_branches are a number greater than 0." diff --git a/python_files/unittestadapter/execution.py b/python_files/unittestadapter/execution.py index 8e4b2462e681..2c49182c8633 100644 --- a/python_files/unittestadapter/execution.py +++ b/python_files/unittestadapter/execution.py @@ -10,7 +10,7 @@ import traceback import unittest from types import TracebackType -from typing import Dict, List, Optional, Tuple, Type, Union +from typing import Dict, Iterator, List, Optional, Tuple, Type, Union # Adds the scripts directory to the PATH as a workaround for enabling shell for test execution. path_var_name = "PATH" if "PATH" in os.environ else "Path" @@ -24,8 +24,10 @@ from django_handler import django_execution_runner # noqa: E402 from unittestadapter.pvsc_utils import ( # noqa: E402 + CoveragePayloadDict, EOTPayloadDict, ExecutionPayloadDict, + FileCoverageInfo, TestExecutionStatus, VSCodeUnittestError, parse_unittest_args, @@ -304,7 +306,6 @@ def send_run_data(raw_data, test_run_pipe): run_test_ids_pipe = os.environ.get("RUN_TEST_IDS_PIPE") test_run_pipe = os.getenv("TEST_RUN_PIPE") - if not run_test_ids_pipe: print("Error[vscode-unittest]: RUN_TEST_IDS_PIPE env var is not set.") raise VSCodeUnittestError("Error[vscode-unittest]: RUN_TEST_IDS_PIPE env var is not set.") @@ -312,6 +313,7 @@ def send_run_data(raw_data, test_run_pipe): print("Error[vscode-unittest]: TEST_RUN_PIPE env var is not set.") raise VSCodeUnittestError("Error[vscode-unittest]: TEST_RUN_PIPE env var is not set.") test_ids = [] + cwd = pathlib.Path(start_dir).absolute() try: # Read the test ids from the file, attempt to delete file afterwords. ids_path = pathlib.Path(run_test_ids_pipe) @@ -324,7 +326,6 @@ def send_run_data(raw_data, test_run_pipe): except Exception as e: # No test ids received from buffer, return error payload - cwd = pathlib.Path(start_dir).absolute() status: TestExecutionStatus = TestExecutionStatus.error payload: ExecutionPayloadDict = { "cwd": str(cwd), @@ -334,6 +335,27 @@ def send_run_data(raw_data, test_run_pipe): } send_post_request(payload, test_run_pipe) + workspace_root = os.environ.get("COVERAGE_ENABLED") + # For unittest COVERAGE_ENABLED is to the root of the workspace so correct data is collected + cov = None + is_coverage_run = os.environ.get("COVERAGE_ENABLED") is not None + if is_coverage_run: + print( + "COVERAGE_ENABLED env var set, starting coverage. workspace_root used as parent dir:", + workspace_root, + ) + import coverage + + source_ar: List[str] = [] + if workspace_root: + source_ar.append(workspace_root) + if top_level_dir: + source_ar.append(top_level_dir) + if start_dir: + source_ar.append(os.path.abspath(start_dir)) # noqa: PTH100 + cov = coverage.Coverage(branch=True, source=source_ar) # is at least 1 of these required?? + cov.start() + # If no error occurred, we will have test ids to run. if manage_py_path := os.environ.get("MANAGE_PY_PATH"): print("MANAGE_PY_PATH env var set, running Django test suite.") @@ -351,3 +373,37 @@ def send_run_data(raw_data, test_run_pipe): failfast, locals_, ) + + if is_coverage_run: + from coverage.plugin import FileReporter + from coverage.report_core import get_analysis_to_report + from coverage.results import Analysis + + if not cov: + raise VSCodeUnittestError("Coverage is enabled but cov is not set") + cov.stop() + cov.save() + analysis_iterator: Iterator[Tuple[FileReporter, Analysis]] = get_analysis_to_report( + cov, None + ) + + file_coverage_map: Dict[str, FileCoverageInfo] = {} + for fr, analysis in analysis_iterator: + file_str: str = fr.filename + executed_branches = analysis.numbers.n_executed_branches + total_branches = analysis.numbers.n_branches + + file_info: FileCoverageInfo = { + "lines_covered": list(analysis.executed), # set + "lines_missed": list(analysis.missing), # set + "executed_branches": executed_branches, # int + "total_branches": total_branches, # int + } + file_coverage_map[file_str] = file_info + payload_cov: CoveragePayloadDict = CoveragePayloadDict( + coverage=True, + cwd=os.fspath(cwd), + result=file_coverage_map, + error=None, + ) + send_post_request(payload_cov, test_run_pipe) diff --git a/python_files/unittestadapter/pvsc_utils.py b/python_files/unittestadapter/pvsc_utils.py index 12a299a8992f..25088f0cb7a2 100644 --- a/python_files/unittestadapter/pvsc_utils.py +++ b/python_files/unittestadapter/pvsc_utils.py @@ -81,6 +81,22 @@ class EOTPayloadDict(TypedDict): eot: bool +class FileCoverageInfo(TypedDict): + lines_covered: List[int] + lines_missed: List[int] + executed_branches: int + total_branches: int + + +class CoveragePayloadDict(Dict): + """A dictionary that is used to send a execution post request to the server.""" + + coverage: bool + cwd: str + result: Optional[Dict[str, FileCoverageInfo]] + error: Optional[str] # Currently unused need to check + + # Helper functions for data retrieval. @@ -300,7 +316,7 @@ def parse_unittest_args( def send_post_request( - payload: Union[ExecutionPayloadDict, DiscoveryPayloadDict, EOTPayloadDict], + payload: Union[ExecutionPayloadDict, DiscoveryPayloadDict, EOTPayloadDict, CoveragePayloadDict], test_run_pipe: Optional[str], ): """ diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index baa9df90eddd..6f04c45f00e6 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -14,6 +14,7 @@ Any, Dict, Generator, + Iterator, Literal, TypedDict, ) @@ -65,6 +66,8 @@ def __init__(self, message): TEST_RUN_PIPE = os.getenv("TEST_RUN_PIPE") SYMLINK_PATH = None +INCLUDE_BRANCHES = False + def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001 global TEST_RUN_PIPE @@ -81,6 +84,10 @@ def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001 global IS_DISCOVERY IS_DISCOVERY = True + if "--cov-branch" in args: + global INCLUDE_BRANCHES + INCLUDE_BRANCHES = True + # check if --rootdir is in the args for arg in args: if "--rootdir=" in arg: @@ -356,6 +363,13 @@ def check_skipped_condition(item): return False +class FileCoverageInfo(TypedDict): + lines_covered: list[int] + lines_missed: list[int] + executed_branches: int + total_branches: int + + def pytest_sessionfinish(session, exitstatus): """A pytest hook that is called after pytest has fulled finished. @@ -420,9 +434,54 @@ def pytest_sessionfinish(session, exitstatus): None, ) # send end of transmission token + + # send coverageee if enabled + is_coverage_run = os.environ.get("COVERAGE_ENABLED") + if is_coverage_run == "True": + # load the report and build the json result to return + import coverage + from coverage.report_core import get_analysis_to_report + + if TYPE_CHECKING: + from coverage.plugin import FileReporter + from coverage.results import Analysis + + cov = coverage.Coverage() + cov.load() + analysis_iterator: Iterator[tuple[FileReporter, Analysis]] = get_analysis_to_report( + cov, None + ) + + file_coverage_map: dict[str, FileCoverageInfo] = {} + for fr, analysis in analysis_iterator: + file_str: str = fr.filename + executed_branches = analysis.numbers.n_executed_branches + total_branches = analysis.numbers.n_branches + if not INCLUDE_BRANCHES: + print("coverage not run with branches") + # if covearge wasn't run with branches, set the total branches value to -1 to signal that it is not available + executed_branches = 0 + total_branches = -1 + + file_info: FileCoverageInfo = { + "lines_covered": list(analysis.executed), # set + "lines_missed": list(analysis.missing), # set + "executed_branches": executed_branches, # int + "total_branches": total_branches, # int + } + file_coverage_map[file_str] = file_info + + payload: CoveragePayloadDict = CoveragePayloadDict( + coverage=True, + cwd=os.fspath(cwd), + result=file_coverage_map, + error=None, + ) + send_post_request(payload) + command_type = "discovery" if IS_DISCOVERY else "execution" - payload: EOTPayloadDict = {"command_type": command_type, "eot": True} - send_post_request(payload) + payload_eot: EOTPayloadDict = {"command_type": command_type, "eot": True} + send_post_request(payload_eot) def build_test_tree(session: pytest.Session) -> TestNode: @@ -738,6 +797,15 @@ class ExecutionPayloadDict(Dict): error: str | None # Currently unused need to check +class CoveragePayloadDict(Dict): + """A dictionary that is used to send a execution post request to the server.""" + + coverage: bool + cwd: str + result: dict[str, FileCoverageInfo] | None + error: str | None # Currently unused need to check + + class EOTPayloadDict(TypedDict): """A dictionary that is used to send a end of transmission post request to the server.""" @@ -822,14 +890,14 @@ def post_response(cwd: str, session_node: TestNode) -> None: class PathEncoder(json.JSONEncoder): """A custom JSON encoder that encodes pathlib.Path objects as strings.""" - def default(self, obj): - if isinstance(obj, pathlib.Path): - return os.fspath(obj) - return super().default(obj) + def default(self, o): + if isinstance(o, pathlib.Path): + return os.fspath(o) + return super().default(o) def send_post_request( - payload: ExecutionPayloadDict | DiscoveryPayloadDict | EOTPayloadDict, + payload: ExecutionPayloadDict | DiscoveryPayloadDict | EOTPayloadDict | CoveragePayloadDict, cls_encoder=None, ): """ diff --git a/python_files/vscode_pytest/run_pytest_script.py b/python_files/vscode_pytest/run_pytest_script.py index 79e039607c4b..9abe3fd6b86c 100644 --- a/python_files/vscode_pytest/run_pytest_script.py +++ b/python_files/vscode_pytest/run_pytest_script.py @@ -34,6 +34,20 @@ def run_pytest(args): sys.path.insert(0, os.getcwd()) # noqa: PTH109 # Get the rest of the args to run with pytest. args = sys.argv[1:] + + # Check if coverage is enabled and adjust the args accordingly. + is_coverage_run = os.environ.get("COVERAGE_ENABLED") + coverage_enabled = False + if is_coverage_run == "True": + # If coverage is enabled, check if the coverage plugin is already in the args, if so keep user args. + for arg in args: + if "--cov" in arg: + coverage_enabled = True + break + if not coverage_enabled: + print("Coverage is enabled, adding branch coverage as an argument.") + args = [*args, "--cov=.", "--cov-branch"] + run_test_ids_pipe = os.environ.get("RUN_TEST_IDS_PIPE") if run_test_ids_pipe: try: diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 16ee79371b37..54a21a712133 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -10,9 +10,20 @@ import { Location, TestRun, MarkdownString, + TestCoverageCount, + FileCoverage, + FileCoverageDetail, + StatementCoverage, + Range, } from 'vscode'; import * as util from 'util'; -import { DiscoveredTestPayload, EOTTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; +import { + CoveragePayload, + DiscoveredTestPayload, + EOTTestPayload, + ExecutionTestPayload, + ITestResultResolver, +} from './types'; import { TestProvider } from '../../types'; import { traceError, traceVerbose } from '../../../logging'; import { Testing } from '../../../common/utils/localize'; @@ -36,6 +47,8 @@ export class PythonResultResolver implements ITestResultResolver { public subTestStats: Map = new Map(); + public detailedCoverageMap = new Map(); + constructor(testController: TestController, testProvider: TestProvider, private workspaceUri: Uri) { this.testController = testController; this.testProvider = testProvider; @@ -105,7 +118,7 @@ export class PythonResultResolver implements ITestResultResolver { } public resolveExecution( - payload: ExecutionTestPayload | EOTTestPayload, + payload: ExecutionTestPayload | EOTTestPayload | CoveragePayload, runInstance: TestRun, deferredTillEOT: Deferred, ): void { @@ -113,9 +126,71 @@ export class PythonResultResolver implements ITestResultResolver { // eot sent once per connection traceVerbose('EOT received, resolving deferredTillServerClose'); deferredTillEOT.resolve(); + } else if ('coverage' in payload) { + // coverage data is sent once per connection + traceVerbose('Coverage data received.'); + this._resolveCoverage(payload as CoveragePayload, runInstance); } else { this._resolveExecution(payload as ExecutionTestPayload, runInstance); } + if ('coverage' in payload) { + // coverage data is sent once per connection + traceVerbose('Coverage data received.'); + this._resolveCoverage(payload as CoveragePayload, runInstance); + } + } + + public _resolveCoverage(payload: CoveragePayload, runInstance: TestRun): void { + if (payload.result === undefined) { + return; + } + for (const [key, value] of Object.entries(payload.result)) { + const fileNameStr = key; + const fileCoverageMetrics = value; + const linesCovered = fileCoverageMetrics.lines_covered ? fileCoverageMetrics.lines_covered : []; // undefined if no lines covered + const linesMissed = fileCoverageMetrics.lines_missed ? fileCoverageMetrics.lines_missed : []; // undefined if no lines missed + const executedBranches = fileCoverageMetrics.executed_branches; + const totalBranches = fileCoverageMetrics.total_branches; + + const lineCoverageCount = new TestCoverageCount( + linesCovered.length, + linesCovered.length + linesMissed.length, + ); + const uri = Uri.file(fileNameStr); + let fileCoverage: FileCoverage; + if (totalBranches === -1) { + // branch coverage was not enabled and should not be displayed + fileCoverage = new FileCoverage(uri, lineCoverageCount); + } else { + const branchCoverageCount = new TestCoverageCount(executedBranches, totalBranches); + fileCoverage = new FileCoverage(uri, lineCoverageCount, branchCoverageCount); + } + runInstance.addCoverage(fileCoverage); + + // create detailed coverage array for each file (only line coverage on detailed, not branch) + const detailedCoverageArray: FileCoverageDetail[] = []; + // go through all covered lines, create new StatementCoverage, and add to detailedCoverageArray + for (const line of linesCovered) { + // line is 1-indexed, so we need to subtract 1 to get the 0-indexed line number + // true value means line is covered + const statementCoverage = new StatementCoverage( + true, + new Range(line - 1, 0, line - 1, Number.MAX_SAFE_INTEGER), + ); + detailedCoverageArray.push(statementCoverage); + } + for (const line of linesMissed) { + // line is 1-indexed, so we need to subtract 1 to get the 0-indexed line number + // false value means line is NOT covered + const statementCoverage = new StatementCoverage( + false, + new Range(line - 1, 0, line - 1, Number.MAX_SAFE_INTEGER), + ); + detailedCoverageArray.push(statementCoverage); + } + + this.detailedCoverageMap.set(fileNameStr, detailedCoverageArray); + } } public _resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): void { diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index 319898f3189a..7846461a46a9 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -4,6 +4,7 @@ import { CancellationToken, Event, + FileCoverageDetail, OutputChannel, TestController, TestItem, @@ -150,7 +151,7 @@ export type TestCommandOptions = { command: TestDiscoveryCommand | TestExecutionCommand; token?: CancellationToken; outChannel?: OutputChannel; - debugBool?: boolean; + profileKind?: TestRunProfileKind; testIds?: string[]; }; @@ -195,18 +196,21 @@ export interface ITestResultResolver { runIdToVSid: Map; runIdToTestItem: Map; vsIdToRunId: Map; + detailedCoverageMap: Map; + resolveDiscovery( payload: DiscoveredTestPayload | EOTTestPayload, deferredTillEOT: Deferred, token?: CancellationToken, ): void; resolveExecution( - payload: ExecutionTestPayload | EOTTestPayload, + payload: ExecutionTestPayload | EOTTestPayload | CoveragePayload, runInstance: TestRun, deferredTillEOT: Deferred, ): void; _resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): void; _resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): void; + _resolveCoverage(payload: CoveragePayload, runInstance: TestRun): void; } export interface ITestDiscoveryAdapter { // ** first line old method signature, second line new method signature @@ -217,11 +221,11 @@ export interface ITestDiscoveryAdapter { // interface for execution/runner adapter export interface ITestExecutionAdapter { // ** first line old method signature, second line new method signature - runTests(uri: Uri, testIds: string[], debugBool?: boolean): Promise; + runTests(uri: Uri, testIds: string[], profileKind?: boolean | TestRunProfileKind): Promise; runTests( uri: Uri, testIds: string[], - debugBool?: boolean, + profileKind?: boolean | TestRunProfileKind, runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, @@ -260,6 +264,27 @@ export type EOTTestPayload = { eot: boolean; }; +export type CoveragePayload = { + coverage: boolean; + cwd: string; + result?: { + [filePathStr: string]: FileCoverageMetrics; + }; + error: string; +}; + +// using camel-case for these types to match the python side +export type FileCoverageMetrics = { + // eslint-disable-next-line camelcase + lines_covered: number[]; + // eslint-disable-next-line camelcase + lines_missed: number[]; + // eslint-disable-next-line camelcase + executed_branches: number; + // eslint-disable-next-line camelcase + total_branches: number; +}; + export type ExecutionTestPayload = { cwd: string; status: 'success' | 'error'; diff --git a/src/client/testing/testController/controller.ts b/src/client/testing/testController/controller.ts index 58edfb059666..dd624078a534 100644 --- a/src/client/testing/testController/controller.ts +++ b/src/client/testing/testController/controller.ts @@ -16,6 +16,8 @@ import { Uri, EventEmitter, TextDocument, + FileCoverageDetail, + TestRun, } from 'vscode'; import { IExtensionSingleActivationService } from '../../activation/types'; import { ICommandManager, IWorkspaceService } from '../../common/application/types'; @@ -38,7 +40,6 @@ import { ITestFrameworkController, TestRefreshOptions, ITestExecutionAdapter, - ITestResultResolver, } from './common/types'; import { UnittestTestDiscoveryAdapter } from './unittest/testDiscoveryAdapter'; import { UnittestTestExecutionAdapter } from './unittest/testExecutionAdapter'; @@ -118,6 +119,14 @@ export class PythonTestController implements ITestController, IExtensionSingleAc this.disposables.push(delayTrigger); this.refreshData = delayTrigger; + const coverageProfile = this.testController.createRunProfile( + 'Coverage Tests', + TestRunProfileKind.Coverage, + this.runTests.bind(this), + true, + RunTestTag, + ); + this.disposables.push( this.testController.createRunProfile( 'Run Tests', @@ -133,6 +142,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc true, DebugTestTag, ), + coverageProfile, ); this.testController.resolveHandler = this.resolveChildren.bind(this); this.testController.refreshHandler = (token: CancellationToken) => { @@ -160,7 +170,8 @@ export class PythonTestController implements ITestController, IExtensionSingleAc let discoveryAdapter: ITestDiscoveryAdapter; let executionAdapter: ITestExecutionAdapter; let testProvider: TestProvider; - let resultResolver: ITestResultResolver; + let resultResolver: PythonResultResolver; + if (settings.testing.unittestEnabled) { testProvider = UNITTEST_PROVIDER; resultResolver = new PythonResultResolver(this.testController, testProvider, workspace.uri); @@ -384,6 +395,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc }); const unconfiguredWorkspaces: WorkspaceFolder[] = []; + try { await Promise.all( workspaces.map(async (workspace) => { @@ -406,6 +418,28 @@ export class PythonTestController implements ITestController, IExtensionSingleAc const settings = this.configSettings.getSettings(workspace.uri); if (testItems.length > 0) { + // coverage?? + const testAdapter = + this.testAdapters.get(workspace.uri) || + (this.testAdapters.values().next().value as WorkspaceTestAdapter); + + if (request.profile?.kind && request.profile?.kind === TestRunProfileKind.Coverage) { + request.profile.loadDetailedCoverage = ( + _testRun: TestRun, + fileCoverage, + _token, + ): Thenable => { + const details = testAdapter.resultResolver.detailedCoverageMap.get( + fileCoverage.uri.fsPath, + ); + if (details === undefined) { + // given file has no detailed coverage data + return Promise.resolve([]); + } + return Promise.resolve(details); + }; + } + if (settings.testing.pytestEnabled) { sendTelemetryEvent(EventName.UNITTEST_RUN, undefined, { tool: 'pytest', @@ -413,15 +447,12 @@ export class PythonTestController implements ITestController, IExtensionSingleAc }); // ** experiment to roll out NEW test discovery mechanism if (pythonTestAdapterRewriteEnabled(this.serviceContainer)) { - const testAdapter = - this.testAdapters.get(workspace.uri) || - (this.testAdapters.values().next().value as WorkspaceTestAdapter); return testAdapter.executeTests( this.testController, runInstance, testItems, token, - request.profile?.kind === TestRunProfileKind.Debug, + request.profile?.kind, this.pythonExecFactory, this.debugLauncher, ); @@ -444,15 +475,12 @@ export class PythonTestController implements ITestController, IExtensionSingleAc }); // ** experiment to roll out NEW test discovery mechanism if (pythonTestAdapterRewriteEnabled(this.serviceContainer)) { - const testAdapter = - this.testAdapters.get(workspace.uri) || - (this.testAdapters.values().next().value as WorkspaceTestAdapter); return testAdapter.executeTests( this.testController, runInstance, testItems, token, - request.profile?.kind === TestRunProfileKind.Debug, + request.profile?.kind, this.pythonExecFactory, this.debugLauncher, ); diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 9d48003525d6..bfaaab9d6586 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -1,13 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { TestRun, Uri } from 'vscode'; +import { TestRun, TestRunProfileKind, Uri } from 'vscode'; import * as path from 'path'; import { ChildProcess } from 'child_process'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { Deferred } from '../../../common/utils/async'; import { traceError, traceInfo, traceVerbose } from '../../../logging'; -import { EOTTestPayload, ExecutionTestPayload, ITestExecutionAdapter, ITestResultResolver } from '../common/types'; +import { + CoveragePayload, + EOTTestPayload, + ExecutionTestPayload, + ITestExecutionAdapter, + ITestResultResolver, +} from '../common/types'; import { ExecutionFactoryCreateWithEnvironmentOptions, IPythonExecutionFactory, @@ -31,7 +37,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { async runTests( uri: Uri, testIds: string[], - debugBool?: boolean, + profileKind?: TestRunProfileKind, runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, @@ -41,7 +47,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { const deferredTillServerClose: Deferred = utils.createTestingDeferred(); // create callback to handle data received on the named pipe - const dataReceivedCallback = (data: ExecutionTestPayload | EOTTestPayload) => { + const dataReceivedCallback = (data: ExecutionTestPayload | EOTTestPayload | CoveragePayload) => { if (runInstance && !runInstance.token.isCancellationRequested) { this.resultResolver?.resolveExecution(data, runInstance, deferredTillEOT); } else { @@ -75,7 +81,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { deferredTillEOT, serverDispose, runInstance, - debugBool, + profileKind, executionFactory, debugLauncher, ); @@ -102,7 +108,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { deferredTillEOT: Deferred, serverDispose: () => void, runInstance?: TestRun, - debugBool?: boolean, + profileKind?: TestRunProfileKind, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise { @@ -120,6 +126,10 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { const pythonPathCommand = [fullPluginPath, ...pythonPathParts].join(path.delimiter); mutableEnv.PYTHONPATH = pythonPathCommand; mutableEnv.TEST_RUN_PIPE = resultNamedPipeName; + if (profileKind && profileKind === TestRunProfileKind.Coverage) { + mutableEnv.COVERAGE_ENABLED = 'True'; + } + const debugBool = profileKind && profileKind === TestRunProfileKind.Debug; // Create the Python environment in which to execute the command. const creationOptions: ExecutionFactoryCreateWithEnvironmentOptions = { diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index b3e134a30dd6..8e5277fe68d9 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import * as path from 'path'; -import { TestRun, Uri } from 'vscode'; +import { TestRun, TestRunProfileKind, Uri } from 'vscode'; import { ChildProcess } from 'child_process'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { Deferred, createDeferred } from '../../../common/utils/async'; @@ -43,7 +43,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { public async runTests( uri: Uri, testIds: string[], - debugBool?: boolean, + profileKind?: TestRunProfileKind, runInstance?: TestRun, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, @@ -81,7 +81,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { deferredTillEOT, serverDispose, runInstance, - debugBool, + profileKind, executionFactory, debugLauncher, ); @@ -107,7 +107,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { deferredTillEOT: Deferred, serverDispose: () => void, runInstance?: TestRun, - debugBool?: boolean, + profileKind?: TestRunProfileKind, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise { @@ -124,12 +124,15 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { const pythonPathCommand = [cwd, ...pythonPathParts].join(path.delimiter); mutableEnv.PYTHONPATH = pythonPathCommand; mutableEnv.TEST_RUN_PIPE = resultNamedPipeName; + if (profileKind && profileKind === TestRunProfileKind.Coverage) { + mutableEnv.COVERAGE_ENABLED = cwd; + } const options: TestCommandOptions = { workspaceFolder: uri, command, cwd, - debugBool, + profileKind, testIds, outChannel: this.outputChannel, token: runInstance?.token, @@ -161,7 +164,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { } try { - if (options.debugBool) { + if (options.profileKind && options.profileKind === TestRunProfileKind.Debug) { const launchOptions: LaunchOptions = { cwd: options.cwd, args, diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts index 5fe69dfe3d69..a0e65cfb5061 100644 --- a/src/client/testing/testController/workspaceTestAdapter.ts +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import * as util from 'util'; -import { CancellationToken, TestController, TestItem, TestRun, Uri } from 'vscode'; +import { CancellationToken, TestController, TestItem, TestRun, TestRunProfileKind, Uri } from 'vscode'; import { createDeferred, Deferred } from '../../common/utils/async'; import { Testing } from '../../common/utils/localize'; import { traceError } from '../../logging'; @@ -34,7 +34,7 @@ export class WorkspaceTestAdapter { private discoveryAdapter: ITestDiscoveryAdapter, private executionAdapter: ITestExecutionAdapter, private workspaceUri: Uri, - private resultResolver: ITestResultResolver, + public resultResolver: ITestResultResolver, ) {} public async executeTests( @@ -42,7 +42,7 @@ export class WorkspaceTestAdapter { runInstance: TestRun, includes: TestItem[], token?: CancellationToken, - debugBool?: boolean, + profileKind?: boolean | TestRunProfileKind, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, ): Promise { @@ -76,13 +76,13 @@ export class WorkspaceTestAdapter { await this.executionAdapter.runTests( this.workspaceUri, testCaseIds, - debugBool, + profileKind, runInstance, executionFactory, debugLauncher, ); } else { - await this.executionAdapter.runTests(this.workspaceUri, testCaseIds, debugBool); + await this.executionAdapter.runTests(this.workspaceUri, testCaseIds, profileKind); } deferred.resolve(); } catch (ex) { diff --git a/src/test/mocks/vsc/index.ts b/src/test/mocks/vsc/index.ts index 4ba0c0bcbf92..152beb64cdf4 100644 --- a/src/test/mocks/vsc/index.ts +++ b/src/test/mocks/vsc/index.ts @@ -576,3 +576,21 @@ export class Location { this.range = rangeOrPosition; } } + +/** + * The kind of executions that {@link TestRunProfile TestRunProfiles} control. + */ +export enum TestRunProfileKind { + /** + * The `Run` test profile kind. + */ + Run = 1, + /** + * The `Debug` test profile kind. + */ + Debug = 2, + /** + * The `Coverage` test profile kind. + */ + Coverage = 3, +} diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index ad5c66df4cda..d0dd5b02d283 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { TestController, TestRun, Uri } from 'vscode'; +import { TestController, TestRun, TestRunProfileKind, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; @@ -71,6 +71,12 @@ suite('End to End Tests: test adapters', () => { 'testTestingRootWkspc', 'symlink_parent-folder', ); + const rootPathCoverageWorkspace = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'testTestingRootWkspc', + 'coverageWorkspace', + ); suiteSetup(async () => { serviceContainer = (await initialize()).serviceContainer; @@ -199,7 +205,6 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); - test('unittest discovery adapter large workspace', async () => { // result resolver and saved data for assertions let actualData: { @@ -562,7 +567,7 @@ suite('End to End Tests: test adapters', () => { .runTests( workspaceUri, ['test_simple.SimpleClass.test_simple_unit'], - false, + TestRunProfileKind.Run, testRun.object, pythonExecFactory, ) @@ -642,7 +647,7 @@ suite('End to End Tests: test adapters', () => { .runTests( workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], - false, + TestRunProfileKind.Run, testRun.object, pythonExecFactory, ) @@ -717,7 +722,7 @@ suite('End to End Tests: test adapters', () => { .runTests( workspaceUri, [`${rootPathSmallWorkspace}/test_simple.py::test_a`], - false, + TestRunProfileKind.Run, testRun.object, pythonExecFactory, ) @@ -751,6 +756,118 @@ suite('End to End Tests: test adapters', () => { } }); }); + + test('Unittest execution with coverage, small workspace', async () => { + // result resolver and saved data for assertions + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + resultResolver._resolveCoverage = async (payload, _token?) => { + assert.strictEqual(payload.cwd, rootPathCoverageWorkspace, 'Expected cwd to be the workspace folder'); + assert.ok(payload.result, 'Expected results to be present'); + const simpleFileCov = payload.result[`${rootPathCoverageWorkspace}/even.py`]; + assert.ok(simpleFileCov, 'Expected test_simple.py coverage to be present'); + // since only one test was run, the other test in the same file will have missed coverage lines + assert.strictEqual(simpleFileCov.lines_covered.length, 3, 'Expected 1 line to be covered in even.py'); + assert.strictEqual(simpleFileCov.lines_missed.length, 1, 'Expected 3 lines to be missed in even.py'); + assert.strictEqual(simpleFileCov.executed_branches, 1, 'Expected 3 lines to be missed in even.py'); + assert.strictEqual(simpleFileCov.total_branches, 2, 'Expected 3 lines to be missed in even.py'); + return Promise.resolve(); + }; + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathCoverageWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; + // run execution + const executionAdapter = new UnittestTestExecutionAdapter( + configService, + testOutputChannel.object, + resultResolver, + envVarsService, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + let collectedOutput = ''; + testRun + .setup((t) => t.appendOutput(typeMoq.It.isAny())) + .callback((output: string) => { + collectedOutput += output; + traceLog('appendOutput was called with:', output); + }) + .returns(() => false); + await executionAdapter + .runTests( + workspaceUri, + ['test_even.TestNumbers.test_odd'], + TestRunProfileKind.Coverage, + testRun.object, + pythonExecFactory, + ) + .finally(() => { + assert.ok(collectedOutput, 'expect output to be collected'); + }); + }); + test('pytest coverage execution, small workspace', async () => { + // result resolver and saved data for assertions + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + resultResolver._resolveCoverage = async (payload, _runInstance?) => { + assert.strictEqual(payload.cwd, rootPathCoverageWorkspace, 'Expected cwd to be the workspace folder'); + assert.ok(payload.result, 'Expected results to be present'); + const simpleFileCov = payload.result[`${rootPathCoverageWorkspace}/even.py`]; + assert.ok(simpleFileCov, 'Expected test_simple.py coverage to be present'); + // since only one test was run, the other test in the same file will have missed coverage lines + assert.strictEqual(simpleFileCov.lines_covered.length, 3, 'Expected 1 line to be covered in even.py'); + assert.strictEqual(simpleFileCov.lines_missed.length, 1, 'Expected 3 lines to be missed in even.py'); + assert.strictEqual(simpleFileCov.executed_branches, 1, 'Expected 3 lines to be missed in even.py'); + assert.strictEqual(simpleFileCov.total_branches, 2, 'Expected 3 lines to be missed in even.py'); + + return Promise.resolve(); + }; + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathCoverageWorkspace); + configService.getSettings(workspaceUri).testing.pytestArgs = []; + + // run pytest execution + const executionAdapter = new PytestTestExecutionAdapter( + configService, + testOutputChannel.object, + resultResolver, + envVarsService, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + let collectedOutput = ''; + testRun + .setup((t) => t.appendOutput(typeMoq.It.isAny())) + .callback((output: string) => { + collectedOutput += output; + traceLog('appendOutput was called with:', output); + }) + .returns(() => false); + await executionAdapter + .runTests( + workspaceUri, + [`${rootPathCoverageWorkspace}/test_even.py::TestNumbers::test_odd`], + TestRunProfileKind.Coverage, + testRun.object, + pythonExecFactory, + ) + .then(() => { + assert.ok(collectedOutput, 'expect output to be collected'); + }); + }); test('pytest execution adapter large workspace', async () => { // result resolver and saved data for assertions resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); @@ -810,17 +927,19 @@ suite('End to End Tests: test adapters', () => { traceLog('appendOutput was called with:', output); }) .returns(() => false); - await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).then(() => { - // verify that the _resolveExecution was called once per test - assert.strictEqual(callCount, 2000, 'Expected _resolveExecution to be called once'); - assert.strictEqual(failureOccurred, false, failureMsg); + await executionAdapter + .runTests(workspaceUri, testIds, TestRunProfileKind.Run, testRun.object, pythonExecFactory) + .then(() => { + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 2000, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); - // verify output works for large repo - assert.ok( - collectedOutput.includes('test session starts'), - 'The test string does not contain the expected stdout output from pytest.', - ); - }); + // verify output works for large repo + assert.ok( + collectedOutput.includes('test session starts'), + 'The test string does not contain the expected stdout output from pytest.', + ); + }); }); test('unittest discovery adapter seg fault error handling', async () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); @@ -1008,10 +1127,12 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); - assert.strictEqual(failureOccurred, false, failureMsg); - }); + await executionAdapter + .runTests(workspaceUri, testIds, TestRunProfileKind.Run, testRun.object, pythonExecFactory) + .finally(() => { + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); + }); }); test('pytest execution adapter seg fault error handling', async () => { resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); @@ -1069,9 +1190,11 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); - assert.strictEqual(failureOccurred, false, failureMsg); - }); + await executionAdapter + .runTests(workspaceUri, testIds, TestRunProfileKind.Run, testRun.object, pythonExecFactory) + .finally(() => { + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); + }); }); }); diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index 040734601a09..8ab701ad6f57 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as assert from 'assert'; -import { TestRun, Uri } from 'vscode'; +import { TestRun, Uri, TestRunProfileKind } from 'vscode'; import * as typeMoq from 'typemoq'; import * as sinon from 'sinon'; import * as path from 'path'; @@ -121,7 +121,7 @@ suite('pytest test execution adapter', () => { adapter = new PytestTestExecutionAdapter(configService, outputChannel.object); const testIds = ['test1id', 'test2id']; - adapter.runTests(uri, testIds, false, testRun.object, execFactory.object); + adapter.runTests(uri, testIds, TestRunProfileKind.Run, testRun.object, execFactory.object); // add in await and trigger await deferred2.promise; @@ -150,7 +150,7 @@ suite('pytest test execution adapter', () => { const uri = Uri.file(myTestPath); const outputChannel = typeMoq.Mock.ofType(); adapter = new PytestTestExecutionAdapter(configService, outputChannel.object); - adapter.runTests(uri, [], false, testRun.object, execFactory.object); + adapter.runTests(uri, [], TestRunProfileKind.Run, testRun.object, execFactory.object); await deferred2.promise; await deferred3.promise; @@ -174,6 +174,7 @@ suite('pytest test execution adapter', () => { assert.equal(options.env?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); assert.equal(options.env?.TEST_RUN_PIPE, expectedExtraVariables.TEST_RUN_PIPE); assert.equal(options.env?.RUN_TEST_IDS_PIPE, expectedExtraVariables.RUN_TEST_IDS_PIPE); + assert.equal(options.env?.COVERAGE_ENABLED, undefined); // coverage not enabled assert.equal(options.cwd, uri.fsPath); assert.equal(options.throwOnStdErr, true); return true; @@ -208,7 +209,7 @@ suite('pytest test execution adapter', () => { const uri = Uri.file(myTestPath); const outputChannel = typeMoq.Mock.ofType(); adapter = new PytestTestExecutionAdapter(configService, outputChannel.object); - adapter.runTests(uri, [], false, testRun.object, execFactory.object); + adapter.runTests(uri, [], TestRunProfileKind.Run, testRun.object, execFactory.object); await deferred2.promise; await deferred3.promise; @@ -267,7 +268,14 @@ suite('pytest test execution adapter', () => { const uri = Uri.file(myTestPath); const outputChannel = typeMoq.Mock.ofType(); adapter = new PytestTestExecutionAdapter(configService, outputChannel.object); - await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); + await adapter.runTests( + uri, + [], + TestRunProfileKind.Debug, + testRun.object, + execFactory.object, + debugLauncher.object, + ); await deferred3.promise; debugLauncher.verify( (x) => @@ -286,4 +294,46 @@ suite('pytest test execution adapter', () => { typeMoq.Times.once(), ); }); + test('pytest execution with coverage turns on correctly', async () => { + const deferred2 = createDeferred(); + const deferred3 = createDeferred(); + execFactory = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => { + deferred2.resolve(); + return Promise.resolve(execService.object); + }); + utilsWriteTestIdsFileStub.callsFake(() => { + deferred3.resolve(); + return Promise.resolve('testIdPipe-mockName'); + }); + const testRun = typeMoq.Mock.ofType(); + testRun.setup((t) => t.token).returns(() => ({ onCancellationRequested: () => undefined } as any)); + const uri = Uri.file(myTestPath); + const outputChannel = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(configService, outputChannel.object); + adapter.runTests(uri, [], TestRunProfileKind.Coverage, testRun.object, execFactory.object); + + await deferred2.promise; + await deferred3.promise; + await deferred4.promise; + mockProc.trigger('close'); + + const pathToPythonFiles = path.join(EXTENSION_ROOT_DIR, 'python_files'); + const pathToPythonScript = path.join(pathToPythonFiles, 'vscode_pytest', 'run_pytest_script.py'); + const rootDirArg = `--rootdir=${myTestPath}`; + const expectedArgs = [pathToPythonScript, rootDirArg]; + execService.verify( + (x) => + x.execObservable( + expectedArgs, + typeMoq.It.is((options) => { + assert.equal(options.env?.COVERAGE_ENABLED, 'True'); + return true; + }), + ), + typeMoq.Times.once(), + ); + }); }); diff --git a/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts index 563735e6a467..41f2fe257681 100644 --- a/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts +++ b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { CancellationTokenSource, TestRun, Uri } from 'vscode'; +import { CancellationTokenSource, TestRun, TestRunProfileKind, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as sinon from 'sinon'; import * as path from 'path'; @@ -126,7 +126,7 @@ suite('Execution Flow Run Adapters', () => { await testAdapter.runTests( Uri.file(myTestPath), [], - false, + TestRunProfileKind.Run, testRunMock.object, execFactoryStub.object, debugLauncher.object, @@ -220,7 +220,7 @@ suite('Execution Flow Run Adapters', () => { await testAdapter.runTests( Uri.file(myTestPath), [], - true, + TestRunProfileKind.Debug, testRunMock.object, execFactoryStub.object, debugLauncher.object, diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index 0cb64a8c75cd..88292c2254d8 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as assert from 'assert'; -import { TestRun, Uri } from 'vscode'; +import { TestRun, TestRunProfileKind, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as sinon from 'sinon'; import * as path from 'path'; @@ -121,7 +121,7 @@ suite('Unittest test execution adapter', () => { adapter = new UnittestTestExecutionAdapter(configService, outputChannel.object); const testIds = ['test1id', 'test2id']; - adapter.runTests(uri, testIds, false, testRun.object, execFactory.object); + adapter.runTests(uri, testIds, TestRunProfileKind.Run, testRun.object, execFactory.object); // add in await and trigger await deferred2.promise; @@ -150,7 +150,7 @@ suite('Unittest test execution adapter', () => { const uri = Uri.file(myTestPath); const outputChannel = typeMoq.Mock.ofType(); adapter = new UnittestTestExecutionAdapter(configService, outputChannel.object); - adapter.runTests(uri, [], false, testRun.object, execFactory.object); + adapter.runTests(uri, [], TestRunProfileKind.Run, testRun.object, execFactory.object); await deferred2.promise; await deferred3.promise; @@ -173,6 +173,7 @@ suite('Unittest test execution adapter', () => { assert.equal(options.env?.PYTHONPATH, expectedExtraVariables.PYTHONPATH); assert.equal(options.env?.TEST_RUN_PIPE, expectedExtraVariables.TEST_RUN_PIPE); assert.equal(options.env?.RUN_TEST_IDS_PIPE, expectedExtraVariables.RUN_TEST_IDS_PIPE); + assert.equal(options.env?.COVERAGE_ENABLED, undefined); // coverage not enabled assert.equal(options.cwd, uri.fsPath); assert.equal(options.throwOnStdErr, true); return true; @@ -207,7 +208,7 @@ suite('Unittest test execution adapter', () => { const uri = Uri.file(myTestPath); const outputChannel = typeMoq.Mock.ofType(); adapter = new UnittestTestExecutionAdapter(configService, outputChannel.object); - adapter.runTests(uri, [], false, testRun.object, execFactory.object); + adapter.runTests(uri, [], TestRunProfileKind.Run, testRun.object, execFactory.object); await deferred2.promise; await deferred3.promise; @@ -266,7 +267,14 @@ suite('Unittest test execution adapter', () => { const uri = Uri.file(myTestPath); const outputChannel = typeMoq.Mock.ofType(); adapter = new UnittestTestExecutionAdapter(configService, outputChannel.object); - await adapter.runTests(uri, [], true, testRun.object, execFactory.object, debugLauncher.object); + await adapter.runTests( + uri, + [], + TestRunProfileKind.Debug, + testRun.object, + execFactory.object, + debugLauncher.object, + ); await deferred3.promise; debugLauncher.verify( (x) => @@ -284,4 +292,45 @@ suite('Unittest test execution adapter', () => { typeMoq.Times.once(), ); }); + test('unittest execution with coverage turned on correctly', async () => { + const deferred2 = createDeferred(); + const deferred3 = createDeferred(); + execFactory = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => { + deferred2.resolve(); + return Promise.resolve(execService.object); + }); + utilsWriteTestIdsFileStub.callsFake(() => { + deferred3.resolve(); + return Promise.resolve('testIdPipe-mockName'); + }); + const testRun = typeMoq.Mock.ofType(); + testRun.setup((t) => t.token).returns(() => ({ onCancellationRequested: () => undefined } as any)); + const uri = Uri.file(myTestPath); + const outputChannel = typeMoq.Mock.ofType(); + adapter = new UnittestTestExecutionAdapter(configService, outputChannel.object); + adapter.runTests(uri, [], TestRunProfileKind.Coverage, testRun.object, execFactory.object); + + await deferred2.promise; + await deferred3.promise; + await deferred4.promise; + mockProc.trigger('close'); + + const pathToPythonFiles = path.join(EXTENSION_ROOT_DIR, 'python_files'); + const pathToExecutionScript = path.join(pathToPythonFiles, 'unittestadapter', 'execution.py'); + const expectedArgs = [pathToExecutionScript, '--udiscovery', '.']; + execService.verify( + (x) => + x.execObservable( + expectedArgs, + typeMoq.It.is((options) => { + assert.equal(options.env?.COVERAGE_ENABLED, uri.fsPath); + return true; + }), + ), + typeMoq.Times.once(), + ); + }); }); diff --git a/src/test/vscode-mock.ts b/src/test/vscode-mock.ts index ec44d302d063..0605b1718166 100644 --- a/src/test/vscode-mock.ts +++ b/src/test/vscode-mock.ts @@ -113,7 +113,7 @@ mockedVSCode.ViewColumn = vscodeMocks.vscMockExtHostedTypes.ViewColumn; mockedVSCode.TextEditorRevealType = vscodeMocks.vscMockExtHostedTypes.TextEditorRevealType; mockedVSCode.TreeItem = vscodeMocks.vscMockExtHostedTypes.TreeItem; mockedVSCode.TreeItemCollapsibleState = vscodeMocks.vscMockExtHostedTypes.TreeItemCollapsibleState; -mockedVSCode.CodeActionKind = vscodeMocks.CodeActionKind; +(mockedVSCode as any).CodeActionKind = vscodeMocks.CodeActionKind; mockedVSCode.CompletionItemKind = vscodeMocks.CompletionItemKind; mockedVSCode.CompletionTriggerKind = vscodeMocks.CompletionTriggerKind; mockedVSCode.DebugAdapterExecutable = vscodeMocks.DebugAdapterExecutable; @@ -133,3 +133,4 @@ mockedVSCode.LogLevel = vscodeMocks.LogLevel; (mockedVSCode as any).ProtocolTypeHierarchyItem = vscodeMocks.vscMockExtHostedTypes.ProtocolTypeHierarchyItem; (mockedVSCode as any).CancellationError = vscodeMocks.vscMockExtHostedTypes.CancellationError; (mockedVSCode as any).LSPCancellationError = vscodeMocks.vscMockExtHostedTypes.LSPCancellationError; +mockedVSCode.TestRunProfileKind = vscodeMocks.TestRunProfileKind; diff --git a/src/testTestingRootWkspc/coverageWorkspace/even.py b/src/testTestingRootWkspc/coverageWorkspace/even.py new file mode 100644 index 000000000000..e395b024ecc5 --- /dev/null +++ b/src/testTestingRootWkspc/coverageWorkspace/even.py @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + + +def number_type(n: int) -> str: + if n % 2 == 0: + return "even" + return "odd" diff --git a/src/testTestingRootWkspc/coverageWorkspace/test_even.py b/src/testTestingRootWkspc/coverageWorkspace/test_even.py new file mode 100644 index 000000000000..ca78535860f4 --- /dev/null +++ b/src/testTestingRootWkspc/coverageWorkspace/test_even.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from even import number_type +import unittest + + +class TestNumbers(unittest.TestCase): + def test_odd(self): + n = number_type(1) + assert n == "odd"