Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Only analyze files that are children of the targets in build() #225

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
f0ffe1a
Check that changed files are children of the targets in build.py.
devdanzin Oct 9, 2023
476b297
Add all files in build() if config.targets is empty.
devdanzin Oct 9, 2023
54f6268
Check against config.targets being ['.'] in build().
devdanzin Oct 9, 2023
db5cbeb
Add a comment to build().
devdanzin Oct 9, 2023
6b359b0
Add a test for filtering of targets in build().
devdanzin Oct 12, 2023
71fb874
Merge branch 'master' into build_target_children
devdanzin Oct 12, 2023
4f4936e
Fix typing of config.targets in build().
devdanzin Oct 12, 2023
6adb0e3
Fix test_build_targets() to use a default config with '.' as path.
devdanzin Oct 12, 2023
5757f5b
Sometimes config.targets is a tuple, check isinstance against Sequence.
devdanzin Oct 13, 2023
9ec37ef
Fix test_run_operator() when Windows test system is mounted in other …
devdanzin Oct 13, 2023
2802a16
Fix test_build_targets() to run on Python 3.7.
devdanzin Oct 13, 2023
546fef9
Fix test_run_operator() when Windows test system is mounted in other …
devdanzin Oct 13, 2023
b95114f
Fix test_build_simple when config.path and file path are on different…
devdanzin Oct 14, 2023
26262d3
Fix typo.
devdanzin Oct 14, 2023
63d975a
Resolve paths in config targets, so building in root ('.') works.
devdanzin Jan 4, 2024
886b4e6
Resolve config.path to correclty check whether a target is a descenda…
devdanzin Jan 10, 2024
0ee3dad
Move resolve() of config.path and config.paths out of for loop.
devdanzin Jan 10, 2024
4761779
Correctly check target against 'resolved_path / file' to determine wh…
devdanzin Jan 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions src/wily/commands/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import multiprocessing
import os
import pathlib
from collections.abc import Sequence
from sys import exit
from typing import Any, Dict, List, Tuple

Expand Down Expand Up @@ -39,6 +40,7 @@ def run_operator(
# Normalize paths for non-seed passes
for key in list(data.keys()):
if os.path.isabs(key):
# ToDo: Handle paths on different drives in Windows
rel = os.path.relpath(key, config.path)
data[rel] = data[key]
del data[key]
Expand Down Expand Up @@ -95,17 +97,26 @@ def build(config: WilyConfig, archiver: Archiver, operators: List[Operator]) ->
try:
with multiprocessing.Pool(processes=len(operators)) as pool:
prev_stats: Dict[str, Dict] = {}
assert isinstance(config.targets, Sequence)
resolved_path = pathlib.Path(config.path).resolve()
resolved_targets = [
pathlib.Path(target).resolve() for target in config.targets
]
for revision in revisions:
# Checkout target revision
archiver_instance.checkout(revision, config.checkout_options)
stats: Dict[str, Dict] = {"operator_data": {}}

# TODO : Check that changed files are children of the targets
targets = [
str(pathlib.Path(config.path) / pathlib.Path(file))
for file in revision.added_files + revision.modified_files
# if any([True for target in config.targets if
# target in pathlib.Path(pathlib.Path(config.path) / pathlib.Path(file)).parents])
if config.targets == ["."] # Add all files if no target is set
# Check that changed files are children of the targets
or any(
True
for target in resolved_targets
if target in (resolved_path / file).parents
)
]

# Run each operator as a separate process
Expand Down
52 changes: 48 additions & 4 deletions test/unit/test_build_unit.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import pathlib
import sys
from unittest.mock import patch
from unittest.mock import MagicMock, patch

import pytest

Expand Down Expand Up @@ -38,7 +39,7 @@ def revisions(self, path, max_revisions):
message="None again",
tracked_files=["a", "b", "c", "d"],
tracked_dirs=["d"],
added_files=["e"],
added_files=["e", "d/h"],
modified_files=["f"],
deleted_files=["a"],
),
Expand All @@ -50,7 +51,7 @@ def checkout(self, revision, options):

class MockOperatorCls(BaseOperator):
name = "test"
data = {"C:\\home\\test1.py" if sys.platform == "win32" else "/home/test1.py": None}
data = {"\\home\\test1.py" if sys.platform == "win32" else "/home/test1.py": None}

def __init__(self, *args, **kwargs):
pass
Expand All @@ -72,16 +73,59 @@ def config():

def test_build_simple(config):
_test_operators = (MockOperator,)
# Remove drive from config path, as that breaks os.path.relpath on GitHub
# test runner because config path and test directory point to different drives
path = config.path
config.path = str(pathlib.Path().joinpath("/", *pathlib.Path(path).parts[1:]))

with patch("wily.state.resolve_archiver", return_value=MockArchiver), patch(
"wily.commands.build.resolve_operator", return_value=MockOperator
):
result = build.build(config, MockArchiver, _test_operators) # type: ignore
assert result is None


def test_build_targets():
mock_state = MagicMock()
mock_starmap = MagicMock()
mock_pool = MagicMock()
mock_pool.return_value = mock_pool
mock_pool.__enter__.return_value = mock_pool
mock_pool.starmap = mock_starmap

config = DEFAULT_CONFIG
config.path = "."
_test_operators = (MockOperator,)
d_path = pathlib.Path("d/")
h_path = str(d_path / "h")

config.targets = ["."]
with patch("wily.state.resolve_archiver", return_value=MockArchiver), patch(
"wily.commands.build.resolve_operator", return_value=MockOperator
), patch("wily.commands.build.State", mock_state), patch(
"wily.commands.build.multiprocessing.Pool", mock_pool
):
build.build(config, MockArchiver, _test_operators) # type: ignore
assert len(mock_starmap.mock_calls) == 6
assert mock_starmap.call_args_list[0][0][1][-1][-1] == ["e", h_path, "f"]
assert mock_starmap.call_args_list[1][0][1][-1][-1] == []

config.targets = [str(d_path)]
mock_starmap.reset_mock()
with patch("wily.state.resolve_archiver", return_value=MockArchiver), patch(
"wily.commands.build.resolve_operator", return_value=MockOperator
), patch("wily.commands.build.State", mock_state), patch(
"wily.commands.build.multiprocessing.Pool", mock_pool
):
build.build(config, MockArchiver, _test_operators) # type: ignore
assert len(mock_starmap.mock_calls) == 6
assert mock_starmap.call_args_list[0][0][1][-1][-1] == [h_path]
assert mock_starmap.call_args_list[1][0][1][-1][-1] == []


def test_run_operator(config):
rev = Revision("123", None, None, 1, "message", [], [], [], [], [])
name, data = build.run_operator(MockOperator, rev, config, ["test1.py"])
assert name == "mock"
path = "C:\\home\\test1.py" if sys.platform == "win32" else "/home/test1.py"
path = "\\home\\test1.py" if sys.platform == "win32" else "/home/test1.py"
assert data == {os.path.relpath(path, config.path): None}
Loading