Address @ncalexan comments

automation/taskcluster/decision_task.py

@@ -55,10 +55,11 @@ def main(task_for):
 # Calls "$PLATFORM_libs" functions and returns
 # their tasks IDs.
 def libs_for(*platforms):
-    return list(map(lambda p: globals()[p + "_libs"](), platforms))
+    is_release = os.environ["TASK_FOR"] == "github-release"
+    return list(map(lambda p: globals()[p + "_libs"](is_release), platforms))
 
-def android_libs():
-    return (
+def android_libs(is_release):
+    task = (
         linux_build_task("Android libs (all architectures): build")
         .with_script("""
             pushd libs
@@ -69,11 +70,14 @@ def android_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.android." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.android." + CONFIG.git_sha_for_directory("libs"))
 
-def desktop_linux_libs():
-    return (
+def desktop_linux_libs(is_release):
+    task = (
         linux_build_task("Desktop libs (Linux): build")
         .with_script("""
             pushd libs
@@ -84,11 +88,14 @@ def desktop_linux_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.desktop.linux." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.desktop.linux." + CONFIG.git_sha_for_directory("libs"))
 
-def desktop_macos_libs():
-    return (
+def desktop_macos_libs(is_release):
+    task = (
         linux_cross_compile_build_task("Desktop libs (macOS): build")
         .with_script("""
             pushd libs
@@ -99,11 +106,14 @@ def desktop_macos_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.desktop.macos." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.desktop.macos." + CONFIG.git_sha_for_directory("libs"))
 
-def desktop_win32_x86_64_libs():
-    return (
+def desktop_win32_x86_64_libs(is_release):
+    task = (
         linux_build_task("Desktop libs (win32-x86-64): build")
         .with_script("""
             apt-get install --quiet --yes --no-install-recommends mingw-w64
@@ -115,8 +125,11 @@ def desktop_win32_x86_64_libs():
         .with_artifacts(
             "/build/repo/target.tar.gz",
         )
-        .find_or_create("build.libs.desktop.win32-x86-64." + CONFIG.git_sha_for_directory("libs"))
     )
+    if is_release:
+        return task.create()
+    else:
+        return task.find_or_create("build.libs.desktop.win32-x86-64." + CONFIG.git_sha_for_directory("libs"))
 
 def android_task(task_name, libs_tasks):
     task = linux_cross_compile_build_task(task_name)
@@ -202,17 +215,17 @@ def android_multiarch_release():
     for module_info in module_definitions():
         module = module_info['name']
         build_task = module_build_tasks[module]
-        for artifact_info in module_info['artifacts']:
-            artifact_name = artifact_info['name']
-            artifact = artifact_info['path']
+        for artifact in module_info['artifacts']:
+            artifact_name = artifact['name']
+            artifact_path = artifact['path']
             (
                 BeetmoverTask("Publish Android module: {} via beetmover".format(artifact_name))
                 .with_description("Publish release module {} to {}".format(artifact_name, bucket_public_url))
                 .with_worker_type(worker_type)
                 # We want to make sure ALL builds succeeded before doing a release.
                 .with_dependencies(*module_build_tasks.values())
                 .with_upstream_artifact({
-                    "paths": [artifact],
+                    "paths": [artifact_path],
                     "taskId": build_task,
                     "taskType": "build",
                     "zipExtract": True,
@@ -230,12 +243,17 @@ def android_multiarch_release():
 def dockerfile_path(name):
     return os.path.join(os.path.dirname(__file__), "docker", name + ".dockerfile")
 
-
 def linux_task(name):
-    return DockerWorkerTask(name).with_worker_type("application-services-r")
-
+    task = (
+        DockerWorkerTask(name)
+        .with_worker_type("application-services-r")
+    )
+    if os.environ["TASK_FOR"] == "github-release":
+        task.with_features("chainOfTrust")
+    return task
 
 def linux_build_task(name):
+    use_indexed_docker_image = os.environ["TASK_FOR"] != "github-release"
     task = (
         linux_task(name)
         # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
@@ -252,7 +270,7 @@ def linux_build_task(name):
         .with_index_and_artifacts_expire_in(build_artifacts_expire_in)
         .with_artifacts("/build/sccache.log")
         .with_max_run_time_minutes(120)
-        .with_dockerfile(dockerfile_path("build"))
+        .with_dockerfile(dockerfile_path("build"), use_indexed_docker_image)
         .with_env(**build_env, **linux_build_env)
         .with_script("""
             rustup toolchain install stable

automation/taskcluster/decisionlib.py

@@ -80,6 +80,7 @@ class Shared:
     """
     def __init__(self):
         self.now = datetime.datetime.utcnow()
+        self.tasks_cache = {}
         self.found_or_created_indexed_tasks = {}
         self.all_tasks = []
 
@@ -210,11 +211,6 @@ def create(self):
         if any(r.startswith("index.") for r in routes):
             self.extra.setdefault("index", {})["expires"] = \
                 SHARED.from_now_json(self.index_and_artifacts_expire_in)
-        if hasattr(self, 'features') and self.features.get('chainOfTrust'):
-            image = self.docker_image
-            if image and isinstance(image, dict):
-                cot = self.extra.setdefault("chainOfTrust", {})
-                cot.setdefault('inputs', {})['docker-image'] = image['taskId']
 
         dict_update_if_truthy(
             queue_payload,
@@ -262,6 +258,20 @@ def find_or_create(self, index_path=None):
         SHARED.found_or_created_indexed_tasks[index_path] = task_id
         return task_id
 
+    def reuse_or_create(self, cache_id=None):
+        """
+        See if we can re-use a task with the same cache_id or
+        create a new one, this is similar as `find_or_create`
+        except that the scope of this function is limited to
+        its execution since nothing is persisted.
+        """
+        task_id = SHARED.tasks_cache.get(cache_id)
+        if task_id is not None:
+            return task_id
+        task_id = self.create()
+        SHARED.tasks_cache[cache_id] = task_id
+        return task_id
+
 class BeetmoverTask(Task):
     def __init__(self, name):
         super().__init__(name)
@@ -337,8 +347,10 @@ def build_worker_payload(self):
                 deindent("\n".join(self.scripts))
             ],
         }
-        if len(self.artifacts) > 0 and "chainOfTrust" not in self.features:
-            self.features["chainOfTrust"] = True
+        if self.features.get("chainOfTrust"):
+            if isinstance(self.docker_image, dict):
+                cot = self.extra.setdefault("chainOfTrust", {})
+                cot.setdefault('inputs', {})['docker-image'] = self.docker_image['taskId']
         return dict_update_if_truthy(
             worker_payload,
             env=self.env,
@@ -396,7 +408,7 @@ def with_repo(self):
             git reset --hard "$APPSERVICES_HEAD_REV"
         """)
 
-    def with_dockerfile(self, dockerfile):
+    def with_dockerfile(self, dockerfile, use_indexed_task=True):
         """
         Build a Docker image based on the given `Dockerfile`, and use it for this task.
 
@@ -434,15 +446,21 @@ def with_dockerfile(self, dockerfile):
                 "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
                 "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"
             )
-            .find_or_create("appservices-docker-image." + digest)
         )
+        if self.features.get("chainOfTrust"):
+            image_build_task.with_features("chainOfTrust")
+        task_index = "appservices-docker-image." + digest
+        if use_indexed_task:
+            image_build_task_id = image_build_task.find_or_create(task_index)
+        else:
+            image_build_task_id = image_build_task.reuse_or_create(task_index)
 
         return self \
-        .with_dependencies(image_build_task) \
+        .with_dependencies(image_build_task_id) \
         .with_docker_image({
             "type": "task-image",
             "path": "public/image.tar.lz4",
-            "taskId": image_build_task,
+            "taskId": image_build_task_id,
         })
 
 

automation/taskcluster/docker/build.dockerfile

@@ -87,8 +87,7 @@ RUN curl -L https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_S
 # Android NDK
 
 # r15c agrees with mozilla-central and, critically, supports the --deprecated-headers flag needed to
-# build OpenSSL
-
+# build OpenSSL.
 ENV ANDROID_NDK_VERSION "r15c"
 
 ENV ANDROID_NDK_ROOT /build/android-ndk

build.gradle

@@ -109,8 +109,11 @@ import com.sun.jna.Platform as DefaultPlatform
 ext.libsGitSha = "git --git-dir=${rootProject.rootDir}/.git diff --name-only master -- :/libs".execute().text.allWhitespace ?
                  "git --git-dir=${rootProject.rootDir}/.git rev-parse HEAD:libs".execute().text.trim() : null
 
+// Use in-tree libs from the source directory in CI or if the git SHA is unset; otherwise use
+// downloaded libs.
+def useDownloadedLibs = !System.getenv('CI') && ext.libsGitSha != null
 
-if (rootProject.ext.libsGitSha != null) {
+if (useDownloadedLibs) {
     task downloadAndroidLibs(type: Download) {
         src "https://index.taskcluster.net/v1/task/project.application-services.application-services.build.libs.android.${rootProject.ext.libsGitSha}/artifacts/public/target.tar.gz"
         dest new File(buildDir, "libs.android.${rootProject.ext.libsGitSha}.tar.gz")
@@ -240,9 +243,7 @@ switch (DefaultPlatform.RESOURCE_PREFIX) {
 ext.cargoExec = { spec, toolchain ->
     spec.environment("OPENSSL_STATIC", "1")
 
-    // Use in-tree libs from the source directory in CI or if the git SHA is unset; otherwise use
-    // downloaded libs.
-    def libsRootDir = (System.getenv('CI') || ext.libsGitSha == null) ? rootProject.rootDir : rootProject.buildDir
+    def libsRootDir = useDownloadedLibs ? rootProject.buildDir : rootProject.rootDir
 
     spec.environment("NSS_DIR",               new File(libsRootDir, "libs/${toolchain.folder}/nss").absolutePath)
     spec.environment("OPENSSL_DIR",           new File(libsRootDir, "libs/${toolchain.folder}/openssl").absolutePath)