1.1.9 Release

.coveragerc

@@ -0,0 +1,6 @@
+[report]
+omit =
+    tests/*
+    setup.py
+    .venv/*
+    .tox/*

.github/workflows/main.yml

@@ -0,0 +1,79 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: ci
+on:
+  push:
+    branches: [main, develop] # proposed addition
+  pull_request:
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      env:
+        POETRY_VIRTUALENVS_CREATE: false
+
+    - name: Install flake8
+      run: |
+        pip install flake8
+
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+  test-install:
+    needs: [ linting ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+    name: Python ${{ matrix.python-version }} with pip
+
+    steps:
+    - name: Check out
+      uses: actions/checkout@v2
+
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+
+    - name: Test editable install
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e .
+
+  tests:
+    needs: [ linting ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10']
+    name: Python ${{ matrix.python-version }} with poetry
+
+    steps:
+    - name: Check out
+      uses: actions/checkout@v2
+
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+
+    - name: Install Poetry 
+      uses: snok/install-poetry@v1
+
+    - name: Install Dependencies using Poetry
+      run: poetry install
+
+    - name: Run pytest
+      run: poetry run pytest

.gitignore

@@ -22,6 +22,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+report.xml
 *.cover
 .hypothesis/
 .pytest_cache/

.pre-commit-config.yaml

@@ -2,11 +2,18 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v4.1.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: check-added-large-files
     -   id: check-toml
+-   repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    - id: black
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
     -   id: flake8

.wci.yml

@@ -0,0 +1,12 @@
+name: Maestro Workflow Conductor
+
+icon: https://raw.githubusercontent.com/LLNL/maestrowf/develop/assets/logo_icon.png
+
+headline: Orchestrate your HPC workflows with ease
+
+description: Maestro is an open-source HPC software tool that defines a YAML-based study specification for defining multistep workflows and automates execution of software flows on HPC resources. The core design tenants of Maestro focus on encouraging clear workflow communication and documentation, while making consistent execution easier to allow users to focus on science. Maestro's study specification helps users think about complex workflows in a step-wise, intent-oriented, manner that encourages modularity and tool reuse. These principles are becoming increasingly important as computational science is continuously more present in scientific fields and has started to require a similar rigor to physical experiment. Maestro is currently in use for multiple projects at Lawrence Livermore National Laboratory and has been used to run existing codes including MFEM, and other simulation codes. It has also been used in other areas including in the training of machine-learned models and more.
+
+documentation:
+  general: https://maestrowf.readthedocs.io/en/latest/
+  installation: https://maestrowf.readthedocs.io/en/latest/installing.html
+  tutorial: https://github.com/LLNL/maestrowf/tree/develop/samples

Dockerfile

@@ -1,6 +1,9 @@
 FROM ubuntu
 LABEL maintainer="Francesco Di Natale [email protected]"
 
-RUN apt-get update && apt-get install -y python python-pip git
+RUN apt-get update && apt-get install -y \
+    python3 \
+    python3-pip \
+    git
 ADD . /maestrowf
-RUN pip install -U /maestrowf
+RUN python3 -m pip install -U /maestrowf

Makefile

@@ -1,21 +1,31 @@
 DOCS = docs
 EGG = $(wildcard *.egg-info)
 
-.PHONY: cleanall clean cleandocs docs
+.PHONY: cleanall clean cleandocs docs help release
 
-all: cleanall release docs
+# Use '#' comments to auto document each target in the help message
+help: # Show this help message
+	@echo 'usage: make [target] ...'
+	@echo
+	@echo 'targets:'
+	@egrep '^(.+)\:\ #\ (.+)' ${MAKEFILE_LIST} | column -t -c 2 -s ':#'
 
-release:
+all: # Clean and then build everything
+	cleanall release docs
+
+release: # Build wheel
 	python setup.py sdist bdist_wheel
 
-docs:
+docs: # Build documentation
 	$(MAKE) -C $(DOCS) html
 
-clean:
+clean: # Clean up release (wheel) build areas
 	rm -rf dist
 	rm -rf build
 
-cleandocs:
+cleandocs: # Clean up docs build areas
 	rm -rf $(DOCS)/build
 
-cleanall: clean cleandocs
+cleanall: # Clean every thing
+	clean
+	cleandocs

README.md

@@ -10,18 +10,18 @@
 [![Stars](https://img.shields.io/github/stars/LLNL/maestrowf.svg)](https://github.com/LLNL/maestrowf/stargazers)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/LLNL/maestrowf/master/LICENSE)
 
-[![Downloads](https://pepy.tech/badge/maestrowf)](https://pepy.tech/project/maestrowf)
-[![Downloads](https://pepy.tech/badge/maestrowf/month)](https://pepy.tech/project/maestrowf/month)
-[![Downloads](https://pepy.tech/badge/maestrowf/week)](https://pepy.tech/project/maestrowf/week)
+[![Downloads](https://static.pepy.tech/personalized-badge/maestrowf?period=total&units=international_system&left_color=grey&right_color=blue&left_text=Downloads)](https://pepy.tech/project/maestrowf)
+[![Downloads](https://static.pepy.tech/personalized-badge/maestrowf?period=month&units=international_system&left_color=grey&right_color=blue&left_text=Downloads/Month)](https://pepy.tech/project/maestrowf)
 
 Maestro can be installed via [pip](https://pip.pypa.io/):
 
     pip install maestrowf
 
 ## Documentation
 
-* [Maestro Documentation](https://maestrowf.readthedocs.io)
-* [Maestro Samples](/samples)
+* [Maestro Documentation](https://maestrowf.readthedocs.io) - Official Maestro documentation.
+* [Maestro Sheetmusic](https://github.com/LLNL/maestro_sheetmusic) - A collection of sample and user contributed Maestro study examples.
+* [Maestro Samples](/samples) - Maestro sample studies.
 
 ## Getting Started is Quick and Easy
 
@@ -131,9 +131,9 @@ There are many definitions of workflow, so we try to keep it simple and define t
 A set of high level tasks to be executed in some order, with or without dependencies on each other.
 ```
 
-We have designed Maestro around the core concept of what we call a "study". A study is defined as a set of steps that are executed (a workflow) over a set of parameters. A study in Maestro's context is analogous to an actual tangible scientific experiment, which has a set of clearly defined and repeatable steps which are repeated over multiple specimen.
+> *PHILOSOPHY*: Maestro believes in the principle of a clearly defined process, specified as a list of tasks, that are self-documenting and clear in their intent.
 
-Maestro's core tenets are defined as follows:
+Running the `hello_world` study is as simple as...
 
 ##### Repeatability
 
@@ -147,7 +147,33 @@ Studies should be consistently documented and able to be run in a consistent fas
 
 Documentation is important in computational studies as much as it is in physical science. The YAML specification defined by Maestro provides a few required key encouraging human-readable documentation. Even further, the specification itself is a documentation of a complete workflow.
 
-----------------
+``` yaml
+description:
+    name: hello_planet
+    description: A simple study to say hello to planets (and Pluto)
+
+study:
+    - name: say-hello
+      description: Say hello to a planet!
+      run:
+          cmd: |
+            echo "Hello, $(PLANET)!" > hello_$(PLANET).txt
+
+global.parameters:
+    PLANET:
+        values: [Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, Pluto]
+        label: PLANET.%%
+```
+
+> *PHILOSOPHY*: Maestro believes that a workflow should be easily parameterized with minimal modifications to the core process.
+
+Maestro will automatically expand each parameter into its own isolated workspace, generate a script for each parameter, and automatically monitor execution of each task.
+
+And, running the study is still as simple as:
+
+``` bash
+    maestro run study.yaml
+```
 
 ## Setting up your Python Environment
 
@@ -163,14 +189,53 @@ To create a new virtual environment:
 
 If you plan to develop on Maestro, install the repository directly using:
 
-    pip install -r requirements.txt
-    pip install -e .
+    pip install poetry
+    poetry install
 
 Once set up, test the environment. The paths should point to a virtual environment folder.
 
     which python
     which pip
 
+----------------
+
+## Using Maestro Dockerfiles
+
+
+Maestro comes packaged with a set of Docker files for testing things out. The two primary files
+are:
+
+- A standard `Dockerfile` in the root of the Maestro repository. This file is a standard install
+  of Maestro meant to try out Maestro on the demo samples provided with this repository. In order
+  to try Maestro locally, with [Docker](https://www.docker.com/) installed run:
+
+  ```
+  docker build -t maestrowf .
+  docker run -ti maestrowf
+  ```
+
+  From within the container run the following:
+
+  ```
+  maestro run ./maestrowf/samples/lulesh/lulesh_sample1_unix.yaml
+  ```
+
+- In order to try out Flux 0.19.0 integration, from the root of the Maestro repository run the
+  following:
+
+  ```
+  docker build -t flux_0190 -f ./docker/flux/0.19.0/Dockerfile .
+  docker run -ti flux_0190
+  ```
+
+  From within the container run the following:
+
+  ```
+  maestro run ./maestrowf/samples/lulesh/lulesh_sample1_unix_flux.yaml
+  ```
+
+----------------
+
 ## Contributors
 
 Many thanks go to MaestroWF's [contributors](https://github.com/LLNL/maestrowf/graphs/contributors).

docker/flux/0.17.0/Dockerfile

@@ -0,0 +1,20 @@
+FROM fluxrm/flux-core:centos7-v0.17.0
+#FROM fluxrm/flux-sched:centos7-v0.9.0
+ENV PATH="/home/fluxuser/.local/bin:$PATH"
+
+RUN curl -sL https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.5.tar.gz | tar xz && \
+    cd ./openmpi-4.0.5 && \
+    ./configure && \
+    make && \
+    sudo make install
+
+COPY . /home/fluxuser/maestrowf
+WORKDIR /home/fluxuser/maestrowf
+RUN echo `which python3` && python3 --version
+RUN echo `which pip3` && pip3 --version
+RUN pip3 install -U --user pip
+RUN echo "$PWD" && ls -la && pip3 install --user .
+RUN pip3 install -U ipython
+WORKDIR /home/fluxuser
+
+LABEL maintainer="Francesco Di Natale [email protected]"