.gitlab-ci.yml

# Copyright (c) 2010-2021, Lawrence Livermore National Security, LLC. Produced
# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
# LICENSE and NOTICE for details. LLNL-CODE-806117.
#
# This file is part of the MFEM library. For more information and source code
# availability visit https://mfem.org.
#
# MFEM is free software; you can redistribute it and/or modify it under the
# terms of the BSD-3 license. We welcome feedback and contributions, see file
# CONTRIBUTING.md for details.

# General GitLab pipelines configurations for supercomputers and Linux clusters
# at Lawrence Livermore National Laboratory (LLNL). This entire pipeline is
# LLNL-specific!

# We define the following GitLab pipeline variables:
#
# BUILD_ROOT:
# The path to the shared resources between all jobs. For example, external
# repositories like 'tests' and 'tpls' are cloned here. Also, 'tpls' is built
# once for all targets, so that build happen here. The BUILD_ROOT is unique to
# the pipeline, preventing any form of concurrency with other pipelines. This
# also means that the BUILD_ROOT directory will never be cleaned.
# TODO: add a clean-up mechanism
#
# REBASELINE:
# Defines the default choice for updating the saved baseline results. By default
# the baseline can only be updated from the master branch. This variable offers
# the option to manually ask for rebaselining from another branch if necessary.
#
# MFEM_ALLOC_NAME:
# On LLNL's quartz, there is only one allocation shared among jobs in order to
# save time and resources. This allocation has to be uniquely named so that we
# are sure to retrieve it.
#
# TPLS_REPO & TESTS_REPO:
# Git repositories used in the pipeline
#
# ARTIFACTS_DIR:
# Directory used to place artifacts.

variables:
  BUILD_ROOT: ${CI_BUILDS_DIR}/MFEM/${CI_PROJECT_NAME}_${CI_COMMIT_REF_SLUG}_${CI_PIPELINE_ID}
  AUTOTEST_ROOT: ${CI_BUILDS_DIR}/MFEM
  REBASELINE: "NO"
  AUTOTEST: "NO"
  ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID}
  TPLS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tpls.git
  TESTS_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/tests.git
  AUTOTEST_REPO: ssh://git@mybitbucket.llnl.gov:7999/mfem/autotest.git
  ARTIFACTS_DIR: artifacts

# The pipeline is divided into stages. Usually, these are also synchronization
# points, however, we use "needs" keyword to express the DAG of jobs for more
# efficiency.
# - We use setup phase to download content outside of mfem directory.
# - Allocate/Release is where quartz resources are allocated/released once for all.
# - Build and Test is where we build and MFEM for multiple toolchains.
# - Baseline_checks gathers baseline-type test suites execution
# - Baseline_publish, only available on master, allows to update baseline
#   results
stages:
  - q_allocate_resources
  - q_build_and_test
  - q_release_resources
  - l_build_and_test
  - c_build_and_test
  - setup
  - baseline_check
  - baseline_to_autotest
  - baseline_publish

# The setup job in setup stage don't rely on MFEM git repo. It prepares a
# pipeline-wide working directory downloading/updating external repos.
# TODO: updating tests and tpls is not necessary anymore since pipelines are
# now using unique directories so repo are never shared with another pipeline.
# This is not memory efficient (we keep a lot of data), hence this reminder.
# Setup
setup:
  tags:
    - shell
    - quartz
  stage: setup
  variables:
    GIT_STRATEGY: none
  script:
    - mkdir -p ${BUILD_ROOT} && cd ${BUILD_ROOT}
    - if [ ! -d "tpls" ]; then git clone ${TPLS_REPO}; fi
    - if [ ! -d "tests" ]; then git clone ${TESTS_REPO}; fi
    - cd tpls && git pull && cd ..
    - cd tests && git pull && cd ..
    - cd ${AUTOTEST_ROOT}
    - if [ ! -d "autotest" ]; then git clone ${AUTOTEST_REPO}; fi
    - cd autotest && git pull && cd ..
  needs: []

.build_toss_3_x86_64_ib_script:
  script:
    - export THREADS=12
    - echo ${ALLOC_NAME}
    - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
    - echo ${JOBID}
    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test

.build_toss_3_x86_64_ib_corona_script:
  script:
    - srun -p mi60 -t 15 -N 1 tests/gitlab/build_and_test

# Lassen uses a different job scheduler (spectrum lsf) that does not
# allow pre-allocation the same way slurm does.
# We use pdebug queue on lassen to speed-up the allocation.
# However this would not be scalable to multiple builds.
.build_blueos_3_ppc64le_ib_script:
  script:
    - lalloc 1 -W 30 -q pdebug tests/gitlab/build_and_test

# Shared script for baseline and sample-run-baseline, the value of BASELINE_TEST
# differentiates between the two tests.
.baseline_script: &baseline_script |
  # locals
  _glob_err=${BASELINE_TEST}.err
  _base_diff=${BASELINE_TEST}-${SYS_TYPE}.diff
  _base_patch=${BASELINE_TEST}-${SYS_TYPE}.patch
  _base_out=${BASELINE_TEST}-${SYS_TYPE}.out
  # prepare
  cd ${BUILD_ROOT}
  ln -snf ${CI_PROJECT_DIR} mfem
  cd tests
  mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
  # run
  srun --nodes=1 -p pdebug ../runtest ../../mfem "${BASELINE_TEST} ${ADDITIONAL_DIR}"
  # post
  mkdir ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}
  if [[ -s ${_glob_err} ]]
  then
    echo "ERROR during ${BASELINE_TEST} execution";
    echo "Here is the ${_glob_err} file content";
    cat ${_glob_err}
    cp ${_glob_err} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_glob_err}
    exit 1;
  elif [[ ! -f ${_base_patch} && ! -f ${_base_out} ]]
  then
    echo "Something went WRONG in ${BASELINE_TEST}:";
    echo "Either ${_base_patch} or ${_base_out} should exists";
    exit 1;
  elif [[ -f ${_base_patch} ]]
  then
    echo "${BASELINE_TEST}: Differences found, patch generated"
    cp ${_base_patch} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_patch}
  elif [[ -f ${_base_out} ]]
  then
    echo "${BASELINE_TEST}: Differences found, replacement file generated"
    cp ${_base_out} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_out}
  fi
  # _base_diff won't even exist if there is no difference.
  if [[ -f ${_base_diff} ]]
  then
    echo "${BASELINE_TEST}: Relevant differences (filtered diff) ..."
    cat ${_base_diff}
    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/${_base_diff}
    # We create a .err file, because that's how we signal that there was a diff.
    cp ${_base_diff} ${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/gitlab-${BASELINE_TEST}-${SYS_TYPE}.err
  fi
  if [[ ! -s ${_base_diff} ]]
  then
    echo "${BASELINE_TEST}: PASSED"
    true
  else
    echo "${BASELINE_TEST}: FAILED"
    false
  fi

# Actual templates for baseline checks
.baselinecheck_mfem:
  stage: baseline_check
  variables:
    BASELINE_TEST: baseline
    ADDITIONAL_DIR: ${BUILD_ROOT}/tpls
  script:
    - *baseline_script
  artifacts:
    when: always
    paths:
      - ${ARTIFACTS_DIR}
  allow_failure: true

.samplebaselinecheck_mfem:
  stage: baseline_check
  variables:
    BASELINE_TEST: sample-runs-baseline
    ADDITIONAL_DIR: ""
  script:
    - *baseline_script
  timeout: 4h
  artifacts:
    when: always
    paths:
      - ${ARTIFACTS_DIR}
  allow_failure: true

# This job can only be manually triggered on a pipeline for master branch, or if
# the pipeline was triggered with REBASELINE="YES"
.rebaseline_mfem:
  stage: baseline_publish
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" || $REBASELINE == "YES"'
      when: manual
  script:
    - export PATCH_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.patch
    - export FULL_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.out
    - export DIFF_FILE=${CI_PROJECT_DIR}/${ARTIFACTS_DIR}/baseline-${SYS_TYPE}.diff
    - cd ${BUILD_ROOT}/tests
    - |
      if [[ ! -f "${DIFF_FILE}" ]]
      then
          echo "Nothing to be done: no relevant change in baseline"
          exit 0
      elif [[ -f "${PATCH_FILE}" ]]
      then
          patch "./baseline-${SYS_TYPE}.saved" < "${PATCH_FILE}"
      elif [[ -f "${FULL_FILE}t" ]]
      then
          cp "${FULL_FILE}" "./baseline-${SYS_TYPE}.saved"
      else
          echo "File missing: expected ${PATCH_FILE} or ${FULL_FILE}"
          exit 1
      fi
    - git add baseline-${SYS_TYPE}.saved
    - git commit -m "${SYS_TYPE} rebaselined in GitLab pipeline ${CI_PIPELINE_ID}"
    - git push origin master

# The list on jobs is defined in machine-specific files.
include:
  - local: .gitlab/quartz.yml
  - local: .gitlab/lassen.yml