Add commits differentiation logic for incremental build (#4207)

Signed-off-by: Zelin Hao <[email protected]>

manifests/2.12.0/opensearch-dashboards-2.12.0.yml

@@ -1,5 +1,5 @@
 ---
-schema-version: '1.0'
+schema-version: '1.1'
 build:
   name: OpenSearch Dashboards
   version: 2.12.0

src/build_workflow/build_args.py

@@ -27,6 +27,7 @@ class BuildArgs:
     architecture: str
     distribution: str
     continue_on_error: bool
+    incremental: bool
 
     def __init__(self) -> None:
         parser = argparse.ArgumentParser(description="Build an OpenSearch Distribution")
@@ -103,6 +104,14 @@ def __init__(self) -> None:
             action="store_true",
             help="Do not fail the distribution build on any plugin component failure.",
         )
+        parser.add_argument(
+            "-i",
+            "--incremental",
+            dest="incremental",
+            default=False,
+            action="store_true",
+            help="Given previous build artifacts are present, build incrementally.",
+        )
 
         args = parser.parse_args()
         self.logging_level = args.logging_level
@@ -116,6 +125,7 @@ def __init__(self) -> None:
         self.distribution = args.distribution
         self.script_path = sys.argv[0].replace("/src/run_build.py", "/build.sh")
         self.continue_on_error = args.continue_on_error
+        self.incremental = args.incremental
 
     def component_command(self, name: str) -> str:
         return " ".join(

src/build_workflow/build_incremental.py

@@ -0,0 +1,64 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+from typing import List
+
+from manifests.build_manifest import BuildManifest
+from manifests.input_manifest import InputManifest
+
+
+class BuildIncremental:
+    def __init__(self, input_manifest: InputManifest, distribution: str):
+        self.distribution = distribution
+        self.input_manifest = input_manifest
+
+    # Given input manifest and return a list of what components changed and added.
+    def commits_diff(self, input_manifest: InputManifest) -> List[str]:
+        build_manifest_path = os.path.join(self.distribution, "builds", input_manifest.build.filename, "manifest.yml")
+        if not os.path.exists(build_manifest_path):
+            logging.info("Previous build manifest does not exist. Rebuilding Core.")
+            return [input_manifest.build.name.replace(" ", "-")]
+        previous_build_manifest = BuildManifest.from_path(build_manifest_path)
+        stable_input_manifest = input_manifest.stable()
+        if previous_build_manifest.build.version != stable_input_manifest.build.version:
+            logging.info("The version of previous build manifest doesn't match the current input manifest. Rebuilding Core.")
+            return [input_manifest.build.name.replace(" ", "-")]
+        components = []
+        for component in stable_input_manifest.components.select():
+            if component.name not in previous_build_manifest.components:
+                components.append(component.name)
+                logging.info(f"Adding {component.name} since it is missing from previous build manifest")
+                continue
+            if component.ref != previous_build_manifest.components[component.name].commit_id:  # type: ignore[attr-defined]
+                components.append(component.name)
+                logging.info(f"Adding {component.name} because it has different commit ID and needs to be rebuilt.")
+                continue
+        return components
+
+    # Given updated plugins and look into the depends_on of all components to finalize a list of rebuilding components.
+    def rebuild_plugins(self, changed_plugins: List, input_manifest: InputManifest) -> List[str]:
+        if not changed_plugins:
+            return []
+
+        if any(core in changed_plugins for core in ("OpenSearch", "OpenSearch-Dashboards")):
+            logging.info("Core engine has new changes, rebuilding all components.")
+            return [component.name for component in input_manifest.components.select()]
+
+        queue = changed_plugins
+        rebuild_list = []
+
+        rebuild_list.append("OpenSearch-Dashboards") if input_manifest.build.filename == "opensearch-dashboards" else rebuild_list
+        while queue:
+            plugin = queue.pop(0)
+            rebuild_list.append(plugin) if plugin not in rebuild_list else rebuild_list
+            for dependent_plugin in input_manifest.plugins_depend_on(plugin):
+                queue.append(dependent_plugin) if dependent_plugin not in rebuild_list else queue
+
+        logging.info(f"Rebuilding list is {rebuild_list}")
+        return rebuild_list

src/manifests/input_manifest.py

@@ -130,6 +130,13 @@ def stable(self) -> 'InputManifest':
         manifest.components.__stabilize__()
         return manifest
 
+    def plugins_depend_on(self, plugin: str) -> List[str]:
+        plugins = []
+        for component in self.components.select():
+            if component.depends_on is not None and plugin in component.depends_on:
+                plugins.append(component.name)
+        return plugins
+
     class Ci:
         def __init__(self, data: dict) -> None:
             self.image = None if data is None else self.Image(data.get("image", None))

src/run_build.py

@@ -11,6 +11,7 @@
 import sys
 
 from build_workflow.build_args import BuildArgs
+from build_workflow.build_incremental import BuildIncremental
 from build_workflow.build_recorder import BuildRecorder
 from build_workflow.build_target import BuildTarget
 from build_workflow.builders import Builders
@@ -41,6 +42,12 @@ def main() -> int:
 
     output_dir = BuildOutputDir(manifest.build.filename, args.distribution).dir
 
+    if args.incremental:
+        buildIncremental = BuildIncremental(manifest, args.distribution)
+        list_of_updated_plugins = buildIncremental.commits_diff(manifest)
+        logging.info(f"Plugins for incremental build: {buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)}")
+        return 0
+
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
         logging.info(f"Building in {work_dir.name}")