WIP: Ensemble read to 2.9

.github/ISSUE_TEMPLATE/new_release.md

@@ -0,0 +1,53 @@
+<!--
+  Replace the following vars with its corresponding values:
+  - @VERSION@ (example 2.9.0)
+  - @MAJOR@ (example 2)
+  - @MINOR@ (example 9)
+  - @OLD_RELEASE@ (example 2.8.3)
+
+-->
+Instructions for performing an ADIOS2 release:
+
+- [ ] Make sure that the milestone for @VERSION@ has no pending issues/PRs.
+- [ ] Update your remotes
+``
+git fetch origin
+git fetch github #if needed
+``
+- [ ] Create a branch that updates the version
+<!-- If the release_@MAJOR@@MINOR@ already exists -->
+```
+git checkout -b bump-release-version origin/release_@MAJOR@@MINOR@
+```
+<!-- else -->
+```
+git checkout -b bump-release-version origin/master
+```
+<!-- endif -->
+- [ ] Add Commit that updates the version in the repo
+```
+git grep --name-only @OLD_RELEASE@ | xargs -n1 sed -i 's/@OLD_RELEASE@/@VERSION@/g'
+git commit -am 'Bump version to v@VERSION@'
+git push
+```
+- [ ] Create PR (BASE to master if release_@MAJOR@@MINOR@ does not exists; otherwise release_@MAJOR@@MINOR@)
+- [ ] Ask for review
+- [ ] Merge PR
+- [ ] Create Tag commit `git tag -a v@VERSION@ the_merge_commit`
+- [ ] Create Release in GitHub page
+  - Use the following script for getting the PR of this release
+    - `./scripts/developer/create-changelog.sh v@VERSION@ v@OLD_RELEASE@`
+  - Copy its content to the release description
+<!-- If the release_@MAJOR@@MINOR@ does not exists -->
+- [ ] Create the release_@MAJOR@@MINOR@ branch
+```
+git fetch origin
+git checkout -b release_@MAJOR@@MINOR@ origin/master
+# Use the following command with care
+git push origin
+```
+<!-- else -->
+- [ ] Create PR that merges release_@MAJOR@@MINOR@ into master
+- [ ] Submit a PR in Spack that adds this new version of ADIOS (if not RC mark this new version as preferred)
+- [ ] Write an announcement in the ADIOS-ECP mail-list
+  (https://groups.google.com/a/kitware.com/g/adios-ecp)

.github/workflows/everything.yml

@@ -82,6 +82,9 @@ jobs:
       - name: Python
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-flake8.sh
+      - name: Shell
+        working-directory: source
+        run: ../gha/scripts/ci/scripts/run-shellcheck.sh
 
 
 #######################################
@@ -145,7 +148,11 @@ jobs:
 
   linux_emu:
     needs: [format, git_checks]
-    if: false # needs.git_checks.outputs.num_code_changes > 0
+    if: needs.git_checks.outputs.num_code_changes > 0 &&
+        (
+        contains('refs/heads/master', github.ref) ||
+        contains('refs/heads/release_', github.ref)
+        )
 
     runs-on: ubuntu-latest
     container:
@@ -164,20 +171,14 @@ jobs:
         cpu: [power8]
         os: [el7]
         compiler: [xl]
-        parallel: [serial, mpi]
+        parallel: [serial]
         include:
           - cpu: power8
             os: el7
             compiler: xl
             parallel: serial
             container: ornladios/adios2:ci-x86_64-power8-el7-xl
             arch: ppc64le
-          - cpu: power8
-            os: el7
-            compiler: xl
-            parallel: mpi
-            container: ornladios/adios2:ci-x86_64-power8-el7-xl-smpi
-            arch: ppc64le
 
     steps:
       - name: Emulation Setup

.gitlab/config/ccache.cmake

@@ -0,0 +1,61 @@
+##============================================================================
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##============================================================================
+
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+
+set(version 4.6.1)
+set(arch x86_64)
+
+if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
+  set(sha256sum da1e1781bc1c4b019216fa16391af3e1daaee7e7f49a8ec9b0cdc8a1d05c50e2)
+  set(base_url https://github.com/ccache/ccache/releases/download)
+  set(platform linux)
+  set(extension tar.xz)
+elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+  set(sha256sum 3e36ba8c80fbf7f2b95fe0227b9dd1ca6143d721aab052caf0d5729769138059)
+  set(full_url https://gitlab.kitware.com/utils/ci-utilities/-/package_files/534/download)
+  set(filename ccache)
+  set(extension tar.gz)
+elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
+  set(sha256sum a6c6311973aa3d2aae22424895f2f968e5d661be003b25f1bd854a5c0cd57563)
+  set(base_url https://github.com/ccache/ccache/releases/download)
+  set(platform windows)
+  set(extension zip)
+else()
+  message(FATAL_ERROR "Unrecognized platform ${CMAKE_HOST_SYSTEM_NAME}")
+endif()
+
+if(NOT DEFINED filename)
+  set(filename "ccache-${version}-${platform}-${arch}")
+endif()
+
+set(tarball "${filename}.${extension}")
+
+if(NOT DEFINED full_url)
+  set(full_url "${base_url}/v${version}/${tarball}")
+endif()
+
+file(DOWNLOAD
+  "${full_url}" $ENV{CCACHE_INSTALL_DIR}/${tarball}
+  EXPECTED_HASH SHA256=${sha256sum}
+  SHOW_PROGRESS
+  )
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E tar xf ${tarball}
+  WORKING_DIRECTORY $ENV{CCACHE_INSTALL_DIR}
+  RESULT_VARIABLE extract_results
+  )
+
+if(extract_results)
+  message(FATAL_ERROR "Extracting `${tarball}` failed: ${extract_results}.")
+endif()
+
+file(RENAME $ENV{CCACHE_INSTALL_DIR}/${filename} $ENV{CCACHE_INSTALL_DIR}/ccache)

.gitlab/config/dynamic_pipeline.yml.in

@@ -0,0 +1,9 @@
+child_pipeline_{branch}:
+  variables:
+    DOWNSTREAM_COMMIT_SHA: '{commit}'
+    DOWNSTREAM_BRANCH_REF: '{branch}'
+  trigger:
+    include:
+      - project: 'ci/csc303_crusher/dev/adios2'
+        ref: '{branch}'
+        file: '.gitlab/gitlab-ci-crusher.yml'

.gitlab/config/generate_pipelines.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#
+# generate_pipeline.py
+#
+#  Created: May 19, 2023
+#   Author: Vicente Adolfo Bolea Sanchez <[email protected]>
+
+from datetime import datetime
+import argparse
+import requests
+import time
+import re
+import urllib3
+# Remove annoying warning about insecure connection (self-signed cert).
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+def is_date_after(date, days):
+    deadline_sec = int(time.time()) - (days * 86400)
+    utc_dt = datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ')
+    timestamp_sec = (utc_dt - datetime(1970, 1, 1)).total_seconds()
+    return timestamp_sec > deadline_sec
+
+
+def request_dict(url):
+    r = requests.get(url + '?per_page=100', verify=False)
+    return r.json()
+
+
+parser = argparse.ArgumentParser(
+    prog='generate_pipeline.py',
+    description='Generate Dynamic pipelines for Gitlab')
+parser.add_argument(
+    '-u', '--gl-url', required=True,
+    help='Base URL for Gitlab remote. Ex: https://code.olcf.ornl.gov/')
+parser.add_argument(
+    '-n', '--gh-name', required=True,
+    help='Full name of the GitHub project. Ex: ornladios/ADIOS2')
+parser.add_argument(
+    '-c', '--gh-context', default='OLCF Crusher (Frontier)',
+    help='Name of the status in GitHub (A.K.A context)')
+parser.add_argument(
+    '-p', '--project_id', required=True,
+    help='Gitlab internal project ID of the project.')
+parser.add_argument(
+    '-d', '--days', type=int, default=1,
+    help='How many days back to search for commits')
+parser.add_argument(
+    '-m', '--max', type=int, default=2,
+    help='Maximum amount of pipelines computed')
+parser.add_argument(
+    '-f', '--template_file', required=True,
+    help='Template file of the pipeline `{branch}` will be substituted')
+args = parser.parse_args()
+
+
+with open(args.template_file, "r") as fd:
+    template_str = fd.read()
+    gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id)
+    gh_url = 'https://api.github.com/repos/' + args.gh_name
+    branches = request_dict(gl_url + "/repository/branches")
+    num_pipeline = 0
+    for branch in branches:
+        # Convert to ISO 8601 date format.
+        date_stamp = branch['commit']['committed_date'].split('.')[0] + "Z"
+        if num_pipeline < args.max and is_date_after(date_stamp, args.days):
+            commit_sha = branch['commit']['id']
+            # Backported branches use the merge head
+            gh_commit_sha = commit_sha
+            if re.fullmatch(r'^pr\d+_.*$', branch['name']):
+                gh_commit_sha = branch['commit']['parent_ids'][1]
+
+            # Quit if GitHub does not have the commit
+            if 'sha' not in request_dict(gh_url + "/commits/" + gh_commit_sha):
+                continue
+
+            # Query GitHub for the status of this commit
+            commit = request_dict(gh_url + "/commits/" +
+                                  gh_commit_sha + "/status")
+            status_found = False
+            for status in commit['statuses']:
+                if status['context'] == args.gh_context:
+                    status_found = True
+            if not status_found:
+                num_pipeline += 1
+                print(template_str.format(
+                    branch=branch['name'], commit=commit_sha))

.gitlab/config/kokkos.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -x
+
+WORKDIR="$1"
+VERSION="$2"
+
+shift 2
+
+if [ ! -d "$WORKDIR" ] || [ -z "$VERSION" ]
+then
+  echo "[E] missing args: Invoke as .gitlab/ci/config/kokkos.sh <WORKDIR> <VERSION> [extra_args]"
+  exit 1
+fi
+
+# Build and install Kokkos
+curl -L "https://github.com/kokkos/kokkos/archive/refs/tags/$VERSION.tar.gz" \
+  | tar -C "$WORKDIR" -xzf -
+
+cmake -S "$WORKDIR/kokkos-$VERSION" -B "$WORKDIR/kokkos_build" \
+  "-DBUILD_SHARED_LIBS=ON" \
+  "-DCMAKE_BUILD_TYPE:STRING=release" \
+  "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" \
+  "-DCMAKE_CXX_STANDARD:STRING=17" \
+  "-DCMAKE_CXX_EXTENSIONS:BOOL=OFF" \
+  "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" \
+  $*
+
+cmake --build "$WORKDIR/kokkos_build"
+cmake --install "$WORKDIR/kokkos_build"