[GR-58956] [GR-59164] [GR-58956] Backport to 24.1: Add barista benchm…

…ark suite.

PullRequest: graal/19129

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.27.5.1",
+  "mx_version": "7.27.5.3",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {

compiler/ci/ci_common/benchmark-builders.jsonnet

@@ -14,6 +14,7 @@
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.dacapo + PR_bench_libgraal,
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.scala_dacapo + PR_bench_libgraal,
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.renaissance + PR_bench_libgraal,
+    c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.barista,
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.specjvm2008 + PR_bench_libgraal,
     c.on_demand                + hw.e3 + jdk + cc.libgraal + bench.dacapo_size_variants,
     c.on_demand                + hw.e3 + jdk + cc.libgraal + bench.scala_dacapo_size_variants,

compiler/ci/ci_common/benchmark-suites.libsonnet

@@ -8,7 +8,7 @@
 
   // convenient sets of benchmark suites for easy reuse
   groups:: {
-    open_suites:: unique_suites([$.awfy, $.dacapo, $.scala_dacapo, $.renaissance]),
+    open_suites:: unique_suites([$.awfy, $.dacapo, $.scala_dacapo, $.renaissance, $.barista]),
     spec_suites:: unique_suites([$.specjvm2008, $.specjbb2015]),
     jmh_micros_suites:: unique_suites([$.micros_graal_dist]),
     graal_internals_suites:: unique_suites([$.micros_graal_whitebox]),
@@ -112,6 +112,47 @@
 
   renaissance: self.renaissance_template(),
 
+  barista_template(suite_version=null, suite_name="barista", max_jdk_version=null, cmd_app_prefix=["hwloc-bind --cpubind node:0.core:0-3.pu:0 --membind node:0"], non_prefix_barista_args=[]):: cc.compiler_benchmark + {
+    suite:: suite_name,
+    local barista_version = "v0.2.0",
+    local suite_version_args = if suite_version != null then ["--bench-suite-version=" + suite_version] else [],
+    local prefix_barista_arg = if std.length(cmd_app_prefix) > 0 then [std.format("--cmd-app-prefix=%s", std.join(" ", cmd_app_prefix))] else [],
+    local all_barista_args = prefix_barista_arg + non_prefix_barista_args,
+    local barista_args_with_separator = if std.length(all_barista_args) > 0 then ["--"] + all_barista_args else [],
+    downloads+: {
+      "WRK": { "name": "wrk", "version": "a211dd5", platformspecific: true},
+      "WRK2": { "name": "wrk2", "version": "2.1", platformspecific: true},
+      "BARISTA_BENCHMARKS": { "name": "barista", "version": "0.2.0"}
+    },
+    packages+: {
+      maven: "==3.8.6",
+      "pip:toml": "==0.10.2"
+    },
+    setup: [
+      ["set-export", "PATH", "$WRK:$PATH"],
+      ["set-export", "PATH", "$WRK2:$PATH"],
+      ["git", "clone", "--depth", "1", "--branch", barista_version, ["mx", "urlrewrite", "https://github.com/graalvm/barista-suite.git"], "$BARISTA_HOME"],
+      ["cp", "-r", "$BARISTA_BENCHMARKS/*", "$BARISTA_HOME"] // copy the prebuilt jar/nib files
+    ] + super.setup,
+    run+: [
+      self.benchmark_cmd + ["barista:*"] + suite_version_args + ["--"] + self.extra_vm_args + barista_args_with_separator
+    ],
+    notify_emails+: ["[email protected]"],
+    timelimit: "1:20:00",
+    should_use_hwloc: false, // hwloc-bind is passed to barista with '--cmd-app-prefix'
+    environment+: {
+      BARISTA_HOME: "$BUILD_DIR/barista-suite",
+      XMX: "500m"
+    },
+    min_jdk_version:: 8,
+    max_jdk_version:: max_jdk_version,
+    forks_batches:: 3,
+    bench_forks_per_batch:: 4,
+    forks_timelimit:: "3:30:00"
+  },
+
+  barista: self.barista_template(),
+
   specjbb2015: cc.compiler_benchmark + c.heap.large_with_large_young_gen + bc.bench_max_threads + {
     suite:: "specjbb2015",
     downloads+: {

sdk/mx.sdk/suite.py

@@ -39,7 +39,7 @@
 # SOFTWARE.
 #
 suite = {
-  "mxversion": "7.27.0",
+  "mxversion": "7.27.5.3",
   "name" : "sdk",
   "version" : "24.1.2",
   "release" : False,

substratevm/mx.substratevm/mx_substratevm_benchmark.py

@@ -28,7 +28,9 @@
 import os
 import tempfile
 import zipfile
+import re
 from glob import glob
+from pathlib import Path
 
 import mx
 import mx_benchmark
@@ -264,6 +266,204 @@ def successPatterns(self):
 mx_benchmark.add_bm_suite(RenaissanceNativeImageBenchmarkSuite())
 
 
+class BaristaNativeImageBenchmarkSuite(mx_sdk_benchmark.BaristaBenchmarkSuite, mx_sdk_benchmark.NativeImageBenchmarkMixin, mx_sdk_benchmark.NativeImageBundleBasedBenchmarkMixin):
+    """Native Image variant of the Barista benchmark suite implementation. A collection of microservice workloads running in native execution mode on the Barista harness.
+
+    The run arguments are passed to the Barista harness.
+    If you want to run something like `hwloc-bind` or `taskset` prefixed before the app image, you should use the '--cmd-app-prefix' Barista harness option.
+    If you want to pass options to the app image, you should use the '--app-args' Barista harness option.
+    """
+    def __init__(self, custom_harness_command: mx_benchmark.CustomHarnessCommand = None):
+        if custom_harness_command is None:
+            custom_harness_command = BaristaNativeImageBenchmarkSuite.BaristaNativeImageCommand()
+        super().__init__(custom_harness_command)
+        self._application_nibs = {}
+        # because of an issue in handling image build args in the intended order [GR-58214]
+        # we need the image name that is set inside the nib
+        self._application_fixed_image_names = {}
+
+    def name(self):
+        return "barista-native-image"
+
+    def benchSuiteName(self, bmSuiteArgs=None):
+        return "barista"
+
+    def benchmarkName(self):
+        return self.context.benchmark
+
+    def application_nib(self):
+        if self.benchmarkName() not in self._application_nibs:
+            # Run subprocess retrieving the application nib from the Barista 'build' script
+            out = mx.OutputCapture()
+            mx.run([self.baristaBuilderPath(), "--get-nib", self.baristaHarnessBenchmarkName()], out=out)
+            # Capture the application nib from the Barista 'build' script output
+            nib_pattern = r"application nib file path is: ([^\n]+)\n"
+            nib_match = re.search(nib_pattern, out.data)
+            if not nib_match:
+                raise ValueError(f"Could not extract the nib file path from the command output! Expected to match pattern {repr(nib_pattern)}.")
+            # Cache for future access
+            self._application_nibs[self.benchmarkName()] = nib_match.group(1)
+            # Try to capture the fixed image name from the Barista 'build' script output
+            fixed_image_name_pattern = r"fixed image name is: ([^\n]+)\n"
+            fixed_image_name_match = re.search(fixed_image_name_pattern, out.data)
+            # Cache fixed image name, if present
+            if fixed_image_name_match:
+                self._application_fixed_image_names[self.benchmarkName()] = fixed_image_name_match.group(1)
+        return self._application_nibs[self.benchmarkName()]
+
+    def application_fixed_image_name(self):
+        self.application_nib()
+        return self._application_fixed_image_names.get(self.benchmarkName(), None)
+
+    def applicationDist(self):
+        return Path(self.application_nib()).parent
+
+    def uses_bundles(self):
+        return True
+
+    def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
+        # Pass the VM options, BaristaNativeImageCommand will form the final command.
+        return self.vmArgs(bmSuiteArgs)
+
+    def extra_jvm_arg(self, benchmark, args):
+        # Added by BaristaNativeImageCommand
+        return []
+
+    def extra_agent_run_arg(self, benchmark, args, image_run_args):
+        # Added by BaristaNativeImageCommand
+        return []
+
+    def extra_profile_run_arg(self, benchmark, args, image_run_args, should_strip_run_args):
+        # Added by BaristaNativeImageCommand
+        return []
+
+    def extra_run_arg(self, benchmark, args, image_run_args):
+        # Added by BaristaNativeImageCommand
+        return []
+
+    def run(self, benchmarks, bmSuiteArgs) -> mx_benchmark.DataPoints:
+        return self.intercept_run(super(), benchmarks, bmSuiteArgs)
+
+    def ensure_image_is_at_desired_location(self, bmSuiteArgs):
+        if self.stages_info.requested_stage.is_image() and self.application_fixed_image_name() is not None:
+            # Because of an issue in handling image build args in the intended order [GR-58214]
+            # we need to move the image from the path that is set inside the nib to the path expected by our vm.
+            # This code has no effect if the image is already at the desired location.
+            vm = self.get_vm_registry().get_vm_from_suite_args(bmSuiteArgs)
+            if vm.stages_info.should_produce_datapoints(mx_sdk_benchmark.Stage.INSTRUMENT_IMAGE):
+                desired_image_path = vm.config.instrumented_image_path
+            elif vm.stages_info.should_produce_datapoints(mx_sdk_benchmark.Stage.IMAGE):
+                desired_image_path = vm.config.image_path
+            else:
+                return
+            actual_image_path = desired_image_path.parent / self.application_fixed_image_name()
+            if actual_image_path.is_file() and not desired_image_path.is_file():
+                mx.move(actual_image_path, desired_image_path)
+
+    def runAndReturnStdOut(self, benchmarks, bmSuiteArgs):
+        retcode, out, dims = super().runAndReturnStdOut(benchmarks, bmSuiteArgs)
+        self.ensure_image_is_at_desired_location(bmSuiteArgs)
+        return retcode, out, dims
+
+    class BaristaNativeImageCommand(mx_sdk_benchmark.BaristaBenchmarkSuite.BaristaCommand):
+        """Maps the command produced by NativeImageVM into a command tailored for the Barista harness.
+        """
+        def _short_load_testing_phases(self):
+            """Configures the main barista load-testing phases to be quite short.
+
+            Useful for the `agent` and `instrument-run` stages.
+            """
+            return [
+                "--warmup-iteration-count", "1",
+                "--warmup-duration", "5",
+                "--throughput-iteration-count", "0",
+                "--latency-iteration-count", "0",
+            ]
+
+        def _get_built_app_image(self, suite, stage):
+            """Retrieves the path to the app image built in the previous stage.
+
+            In the case of `instrument-run`, retrieves the image built during `instrument-image`.
+            In the case of `run`, retrieves the image built during `image`.
+            """
+            vm = suite.context.vm
+            if stage == mx_sdk_benchmark.Stage.INSTRUMENT_RUN:
+                return vm.config.instrumented_image_path
+            else:
+                return vm.config.image_path
+
+        def produce_JVM_harness_command(self, cmd, suite):
+            """Maps a JVM command into a command tailored for the Barista harness.
+
+            Utilizes the implementation of the ``mx_sdk_benchmark.BaristaBenchmarkSuite.BaristaCommand`` base class
+            """
+            return super().produceHarnessCommand(cmd, suite)
+
+        def produceHarnessCommand(self, cmd, suite):
+            """Maps a NativeImageVM command into a command tailored for the Barista harness.
+
+            This method is invoked only in the `agent`, `instrument-run` and `run` stages, because hooks are
+            only applied in these stages (defined in ``NativeImageBenchmarkMixin.run_stage``).
+            In the case of the `agent` stage, relies on the parent ``BaristaCommand`` class for the mapping.
+
+            :param list[str] cmd: NativeImageVM command to be mapped.
+            :param BaristaNativeImageBenchmarkSuite suite: Barista benchmark suite running the benchmark on the Barista harness.
+            :return: Command tailored for the Barista harness.
+            :rtype: list[str]
+            """
+            if not isinstance(suite, BaristaNativeImageBenchmarkSuite):
+                raise TypeError(f"Expected an instance of {BaristaNativeImageBenchmarkSuite.__name__}, instead got an instance of {suite.__class__.__name__}")
+
+            stage = suite.stages_info.requested_stage
+            if stage == mx_sdk_benchmark.Stage.AGENT:
+                # BaristaCommand works for agent stage, since it's a JVM stage
+                cmd = self.produce_JVM_harness_command(cmd, suite)
+                # Make agent run short
+                cmd += self._short_load_testing_phases()
+                # Add explicit agent stage args
+                cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.context.bmSuiteArgs)
+                cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-agent-run-arg=", suite.context.bmSuiteArgs)
+                return cmd
+
+            # Extract app image options and command prefix from the NativeImageVM command
+            app_image = str(self._get_built_app_image(suite, stage))
+            try:
+                index_of_app_image = cmd.index(app_image)
+            except:
+                mx.log_error(f"Could not find app image '{app_image}' in {cmd}")
+                raise
+            nivm_cmd_prefix = cmd[:index_of_app_image]
+            nivm_app_options = cmd[index_of_app_image + 1:]
+
+            # Get bench name and workload to use in the barista harness - we might have custom named benchmarks that need to be mapped
+            barista_bench_name = suite.baristaHarnessBenchmarkName()
+            barista_workload = suite.baristaHarnessBenchmarkWorkload()
+
+            # Provide image built in the previous stage to the Barista harnesss using the `--app-executable` option
+            ni_barista_cmd = [suite.baristaHarnessPath(), "--mode", "native", "--app-executable", app_image]
+            if barista_workload is not None:
+                ni_barista_cmd.append(f"--config={barista_workload}")
+            ni_barista_cmd += suite.runArgs(suite.context.bmSuiteArgs)
+            ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.context.bmSuiteArgs)
+            if stage == mx_sdk_benchmark.Stage.INSTRUMENT_RUN:
+                # Make instrument run short
+                ni_barista_cmd += self._short_load_testing_phases()
+                # Add explicit instrument stage args
+                ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-profile-run-arg=", suite.context.bmSuiteArgs) or mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-run-arg=", suite.context.bmSuiteArgs)
+            else:
+                # Add explicit run stage args
+                ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-run-arg=", suite.context.bmSuiteArgs)
+            if nivm_cmd_prefix:
+                self._updateCommandOption(ni_barista_cmd, "--cmd-app-prefix", "-p", " ".join(nivm_cmd_prefix))
+            if nivm_app_options:
+                self._updateCommandOption(ni_barista_cmd, "--app-args", "-a", " ".join(nivm_app_options))
+            ni_barista_cmd += [barista_bench_name]
+            return ni_barista_cmd
+
+
+mx_benchmark.add_bm_suite(BaristaNativeImageBenchmarkSuite())
+
+
 class BaseDaCapoNativeImageBenchmarkSuite():
 
     '''`SetBuildInfo` method in DaCapo source reads from the file nested in daCapo jar.

substratevm/mx.substratevm/suite.py

@@ -1,6 +1,6 @@
 # pylint: disable=line-too-long
 suite = {
-    "mxversion": "7.27.1",
+    "mxversion": "7.27.5.3",
     "name": "substratevm",
     "version" : "24.1.2",
     "release" : False,

vm/mx.vm/mx_vm_benchmark.py

@@ -134,7 +134,7 @@ def __init__(self, vm: NativeImageVM, bm_suite: BenchmarkSuite | NativeImageBenc
         self.bm_suite = bm_suite
         self.benchmark_suite_name = bm_suite.benchSuiteName(args)
         self.benchmark_name = bm_suite.benchmarkName()
-        self.executable, self.classpath_arguments, self.modulepath_arguments, self.system_properties, self.image_vm_args, image_run_args, self.split_run = NativeImageVM.extract_benchmark_arguments(args)
+        self.executable, self.classpath_arguments, self.modulepath_arguments, self.system_properties, self.image_vm_args, image_run_args, self.split_run = NativeImageVM.extract_benchmark_arguments(args, bm_suite.all_command_line_args_are_vm_args())
         self.extra_image_build_arguments: List[str] = bm_suite.extra_image_build_argument(self.benchmark_name, args)
         # use list() to create fresh copies to safeguard against accidental modification
         self.image_run_args = bm_suite.extra_run_arg(self.benchmark_name, args, list(image_run_args))
@@ -720,7 +720,10 @@ def supported_vm_arg_prefixes():
                                     '--patch-module', '--boot-class-path', '--source-path', '-cp', '-classpath', '-p']
 
     @staticmethod
-    def _split_vm_arguments(args):
+    def _split_vm_arguments(args, all_args_are_vm_args):
+        if all_args_are_vm_args:
+            return args, [], []
+
         i = 0
         while i < len(args):
             arg = args[i]
@@ -736,7 +739,7 @@ def _split_vm_arguments(args):
         mx.abort('No executable found in args: ' + str(args))
 
     @staticmethod
-    def extract_benchmark_arguments(args):
+    def extract_benchmark_arguments(args, all_args_are_vm_args):
         i = 0
         clean_args = args[:]
         split_run = None
@@ -750,7 +753,7 @@ def extract_benchmark_arguments(args):
             else:
                 i += 1
         clean_args = [x for x in clean_args if "-Dnative-image" not in x]
-        vm_args, executable, image_run_args = NativeImageVM._split_vm_arguments(clean_args)
+        vm_args, executable, image_run_args = NativeImageVM._split_vm_arguments(clean_args, all_args_are_vm_args)
 
         classpath_arguments = []
         modulepath_arguments = []