benchpark experiment init (#339)

* Add initial framework for experiment init

* saxpy: fix mismatch between variable name and usage

* cmd/experiment: fix interface to Experiment class

* cmd/experiment: implement benchpark experiment list

* cmd/experiment: remove vestigial references to 'system'

* spec: fix bug with ConcreteSpec.satisfies and ConcreteSpec.intersects

* spec: remove vestigial 'autospec' decorator

---------

Co-authored-by: Gregory Becker <[email protected]>

lib/benchpark/cmd/experiment.py

@@ -0,0 +1,71 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import shutil
+import sys
+
+import benchpark.experiment
+import benchpark.spec
+
+
+def experiment_init(args):
+    experiment_spec = benchpark.spec.ExperimentSpec(" ".join(args.spec)).concretize()
+    experiment = experiment_spec.experiment
+
+    if args.basedir:
+        base = args.basedir
+        sysdir = experiment.experiment_id()
+        destdir = os.path.join(base, sysdir)
+    elif args.dest:
+        destdir = args.dest
+    else:
+        raise ValueError("Must specify one of: --dest, --basedir")
+
+    try:
+        os.mkdir(destdir)
+        experiment.write_ramble_dict(f"{destdir}/ramble.yaml")
+    except FileExistsError:
+        print(f"Abort: experiment description dir already exists ({destdir})")
+        sys.exit(1)
+    except Exception:
+        # If there was a failure, remove any partially-generated resources
+        shutil.rmtree(destdir)
+        raise
+
+
+def experiment_list(args):
+    experiments = benchpark.repo.all_object_names(
+        benchpark.repo.ObjectTypes.experiments
+    )
+    # TODO: prettier printing
+    print("    ".join(experiments))
+
+
+def setup_parser(root_parser):
+    system_subparser = root_parser.add_subparsers(dest="experiment_subcommand")
+
+    init_parser = system_subparser.add_parser("init")
+    init_parser.add_argument("--dest", help="Place all system files here directly")
+    init_parser.add_argument(
+        "--basedir", help="Generate a system dir under this, and place all files there"
+    )
+
+    init_parser.add_argument("spec", nargs="+", help="Experiment spec")
+
+    system_subparser.add_parser("list")
+
+
+def command(args):
+    actions = {
+        "init": experiment_init,
+        "list": experiment_list,
+    }
+    if args.experiment_subcommand in actions:
+        actions[args.experiment_subcommand](args)
+    else:
+        raise ValueError(
+            f"Unknown subcommand for 'experiment': {args.experiment_subcommand}"
+        )

lib/benchpark/spec.py

@@ -157,7 +157,7 @@ def _parse(self, string: str):
 
     def intersects(self, other: Union[str, "Spec"]) -> bool:
         if not isinstance(other, Spec):
-            other = type(self)(other)  # keep type from subclass
+            other = Spec(other)  # subclasses do not override intersects behavior
         return (
             (self.name is None or other.name is None or self.name == other.name)
             and (
@@ -170,7 +170,7 @@ def intersects(self, other: Union[str, "Spec"]) -> bool:
 
     def satisfies(self, other: Union[str, "Spec"]) -> bool:
         if not isinstance(other, Spec):
-            other = type(self)(other)  # keep type from subclass
+            other = Spec(other)  # subclasses do not override satisfies behavior
         return (
             (other.name is None or self.name == other.name)
             and (other.namespace is None or self.namespace == other.namespace)
@@ -201,20 +201,6 @@ def concretize(self):
         return ConcreteExperimentSpec(self)
 
 
-def autospec(function):
-    """Decorator that automatically converts the first argument of a
-    function to a Spec.
-    """
-
-    @functools.wraps(function)
-    def converter(self, spec_like, *args, **kwargs):
-        if not isinstance(spec_like, ExperimentSpec):
-            spec_like = ExperimentSpec(spec_like)
-        return function(self, spec_like, *args, **kwargs)
-
-    return converter
-
-
 class ConcreteSpec(Spec):
     def __init__(self, str_or_spec: Union[str, Spec]):
         super().__init__(str_or_spec)

lib/main.py

@@ -13,6 +13,7 @@
 import yaml
 
 import benchpark.cmd.system
+import benchpark.cmd.experiment
 from benchpark.runtime import RuntimeResources
 
 DEBUG = False
@@ -282,7 +283,12 @@ def init_commands(subparsers, actions_dict):
     """
     system_parser = subparsers.add_parser("system", help="Initialize a system config")
     benchpark.cmd.system.setup_parser(system_parser)
+
+    experiment_parser = subparsers.add_parser("experiment", help="Interact with experiments")
+    benchpark.cmd.experiment.setup_parser(experiment_parser)
+
     actions_dict["system"] = benchpark.cmd.system.command
+    actions_dict["experiment"] = benchpark.cmd.experiment.command
 
 
 def run_command(command_str, env=None):

var/exp_repo/experiments/saxpy/experiment.py

@@ -16,7 +16,7 @@ def compute_applications_section(self):
 
         # GPU tests include some smaller sizes
         n = ["512", "1024"]
-        matrix = ["size"]
+        matrix = ["n"]
         if self.spec.satisfies("programming_model=openmp"):
             matrix += ["omp_num_threads"]
             variables["n_nodes"] = ["1", "2"]