diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index 3c9a42aa2..a1c04f485 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -1091,7 +1091,7 @@ def _display_python_version(): def main(args): - os.environ["TASK_WORKDIR"] = os.getcwd() + task_workdir = os.environ["TASK_WORKDIR"] = os.getcwd() print_line( b"setup", b"run-task started in %s\n" % os.environ["TASK_WORKDIR"].encode("utf-8"), @@ -1316,12 +1316,9 @@ def main(args): resource_process = None try: - for k in ["MOZ_FETCHES_DIR", "UPLOAD_DIR"] + [ - "{}_PATH".format(repository["project"].upper()) - for repository in repositories - ]: - if k in os.environ: - os.environ[k] = os.path.abspath(os.environ[k]) + for k, v in os.environ.items(): + if "$TASK_WORKDIR" in v: + os.environ[k] = v.replace("$TASK_WORKDIR", task_workdir) print_line( b"setup", b"%s is %s\n" % (k.encode("utf-8"), os.environ[k].encode("utf-8")), diff --git a/src/taskgraph/transforms/run/__init__.py b/src/taskgraph/transforms/run/__init__.py index 520cdf0a6..e08bc1f64 100644 --- a/src/taskgraph/transforms/run/__init__.py +++ b/src/taskgraph/transforms/run/__init__.py @@ -367,7 +367,7 @@ def cmp_artifacts(a): "task-reference": json.dumps(task_fetches, sort_keys=True) } - env.setdefault("MOZ_FETCHES_DIR", "fetches") + env.setdefault("MOZ_FETCHES_DIR", "$TASK_WORKDIR/fetches") yield task diff --git a/src/taskgraph/transforms/run/common.py b/src/taskgraph/transforms/run/common.py index 37095a356..dfd12572d 100644 --- a/src/taskgraph/transforms/run/common.py +++ b/src/taskgraph/transforms/run/common.py @@ -10,6 +10,7 @@ import hashlib import json +from taskgraph.util import path from taskgraph.util.taskcluster import get_artifact_prefix @@ -98,16 +99,16 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): assert is_mac or is_win or is_linux if is_win: - checkoutdir = "./build" + checkoutdir = "build" hgstore = "y:/hg-shared" elif is_docker: checkoutdir = "{workdir}/checkouts".format(**task["run"]) hgstore = f"{checkoutdir}/hg-store" else: - checkoutdir = "./checkouts" + checkoutdir = "checkouts" hgstore = f"{checkoutdir}/hg-shared" - vcsdir = checkoutdir + "/" + get_vcsdir_name(worker["os"]) + vcsdir = f"{checkoutdir}/{get_vcsdir_name(worker["os"])}" cache_name = "checkouts" # Robust checkout does not clean up subrepositories, so ensure that tasks @@ -138,7 +139,8 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): "REPOSITORIES": json.dumps( {repo.prefix: repo.name for repo in repo_configs.values()} ), - "VCS_PATH": vcsdir, + # If vcsdir is already absolute this will return it unmodified. + "VCS_PATH": path.join("$TASK_WORKDIR", vcsdir), } ) for repo_config in repo_configs.values(): @@ -162,3 +164,5 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): # only some worker platforms have taskcluster-proxy enabled if task["worker"]["implementation"] in ("docker-worker",): taskdesc["worker"]["taskcluster-proxy"] = True + + return vcsdir diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index f0b48c3f6..124242ae2 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -70,7 +70,7 @@ def common_setup(config, task, taskdesc, command): for (repo, config) in run["checkout"].items() } - support_vcs_checkout( + vcs_path = support_vcs_checkout( config, task, taskdesc, @@ -78,7 +78,6 @@ def common_setup(config, task, taskdesc, command): sparse=bool(run["sparse-profile"]), ) - vcs_path = taskdesc["worker"]["env"]["VCS_PATH"] for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index 6e335e394..023436a77 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -471,13 +471,19 @@ def inner(extra_args=None, env=None): def test_main_interpolate_environment(run_main): result, out, err, env = run_main( - env={"MOZ_FETCHES_DIR": "file", "UPLOAD_DIR": "file", "FOO": "file"} + env={ + "MOZ_FETCHES_DIR": "$TASK_WORKDIR/file", + "UPLOAD_DIR": "$TASK_WORKDIR/file", + "FOO": "$TASK_WORKDIR/file", + "BAR": "file", + } ) assert result == 0 assert env == { "MOZ_FETCHES_DIR": "/builds/worker/file", "UPLOAD_DIR": "/builds/worker/file", - "FOO": "file", + "FOO": "/builds/worker/file", + "BAR": "file", "TASK_WORKDIR": "/builds/worker", } diff --git a/test/test_transforms_run_run_task.py b/test/test_transforms_run_run_task.py index 5dfff6442..3debe831a 100644 --- a/test/test_transforms_run_run_task.py +++ b/test/test_transforms_run_run_task.py @@ -99,7 +99,7 @@ def assert_generic_worker(task): "worker": { "command": [ "C:/mozilla-build/python3/python3.exe run-task " - '--ci-checkout=./build/src/ -- bash -cx "echo hello ' + '--ci-checkout=build/src/ -- bash -cx "echo hello ' 'world"' ], "env": { @@ -111,11 +111,11 @@ def assert_generic_worker(task): "HG_STORE_PATH": "y:/hg-shared", "MOZ_SCM_LEVEL": "1", "REPOSITORIES": '{"ci": "Taskgraph"}', - "VCS_PATH": "./build/src", + "VCS_PATH": "$TASK_WORKDIR/build/src", }, "implementation": "generic-worker", "mounts": [ - {"cache-name": "checkouts", "directory": "./build"}, + {"cache-name": "checkouts", "directory": "build"}, { "content": { "url": "https://tc-tests.localhost/api/queue/v1/task//artifacts/public/run-task" # noqa @@ -159,7 +159,7 @@ def assert_run_task_command_generic_worker(task): [ "/foo/bar/python3", "run-task", - "--ci-checkout=./checkouts/vcs/", + "--ci-checkout=checkouts/vcs/", "--", "bash", "-cx",