publish.yml

# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "publish" stage

.build-push-docker-image-common:
  extends:
    - .kubernetes-env
  stage:                           publish
  variables:
    CI_IMAGE:                      $BUILDAH_IMAGE
    GIT_STRATEGY:                  none
    DOCKERFILE:                    $PRODUCT.Dockerfile
    IMAGE_NAME:                    docker.io/$IMAGE_PATH
  before_script:
    - !reference [.job-switcher, before_script]
    - cd ./artifacts/$PRODUCT/
    - VERSION="$(cat ./VERSION)"
    - echo "${PRODUCT} version = ${VERSION}"
    - test -z "${VERSION}" && exit 1
  script:
    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
        ( echo "no docker credentials provided"; exit 1 )
    - buildah bud
        --format=docker
        --build-arg VCS_REF="${CI_COMMIT_SHA}"
        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
        --build-arg IMAGE_NAME="${IMAGE_PATH}"
        --tag "$IMAGE_NAME:$VERSION"
        --tag "$IMAGE_NAME:latest"
        --file "$DOCKERFILE" .
    - echo "$DOCKER_PASS" |
        buildah login --username "$DOCKER_USER" --password-stdin docker.io
    - buildah info
    - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
    - buildah push --format=v2s2 "$IMAGE_NAME:latest"
  after_script:
    - buildah logout --all
    - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
    - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
    - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
    - cat ./artifacts/$PRODUCT/build.env

.build-push-docker-image:
  extends:
    - .publish-refs
    - .build-push-docker-image-common
  variables:
    IMAGE_PATH:                    parity/$PRODUCT
    DOCKER_USER:                   $Docker_Hub_User_Parity
    DOCKER_PASS:                   $Docker_Hub_Pass_Parity


# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
.build-push-image-temporary:
  extends:
    - .build-refs
    - .build-push-docker-image-common
  variables:
    IMAGE_PATH:                    paritypr/$PRODUCT
    DOCKER_USER:                   $PARITYPR_USER
    DOCKER_PASS:                   $PARITYPR_PASS

publish-docker-substrate:
  extends:                         .build-push-docker-image
  needs:
    - job:                         build-linux-substrate
      artifacts:                   true
  variables:
    PRODUCT:                       substrate

publish-docker-substrate-temporary:
  extends:                         .build-push-image-temporary
  needs:
    - job:                         build-linux-substrate
      artifacts:                   true
  variables:
    PRODUCT:                       substrate
  artifacts:
    reports:
      # this artifact is used in zombienet-tests job
      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
      dotenv: ./artifacts/$PRODUCT/build.env
    expire_in:                     24h

publish-docker-subkey:
  extends:                         .build-push-docker-image
  needs:
    - job:                         build-subkey-linux
      artifacts:                   true
  variables:
    PRODUCT:                       subkey

publish-s3-release:
  stage:                           publish
  extends:
    - .publish-refs
    - .kubernetes-env
  needs:
    - job:                         build-linux-substrate
      artifacts:                   true
    - job:                         build-subkey-linux
      artifacts:                   true
  image:                           paritytech/awscli:latest
  variables:
    GIT_STRATEGY:                  none
    BUCKET:                        "releases.parity.io"
    PREFIX:                        "substrate/${ARCH}-${DOCKER_OS}"
  script:
    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
    - echo "update objects in latest path"
    - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
  after_script:
    - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
        --recursive --human-readable --summarize

publish-rustdoc:
  stage:                           publish
  extends:                         .kubernetes-env
  variables:
    CI_IMAGE:                      node:16
    GIT_DEPTH:                     100
    RUSTDOCS_DEPLOY_REFS:          "master"
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^monthly-20[0-9]{2}-[0-9]{2}.*$/  # to support: monthly-2021-09+1
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
  # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
  # artifacts from being dowloaded by this job.
  needs:
    - job:                         build-rustdoc
      artifacts:                   true
  script:
    # If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we
    # exit immediately.
    # Putting spaces at the front and back to ensure we are not matching just any substring, but the
    # whole space-separated value.
    - '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0'
    # setup ssh
    - eval $(ssh-agent)
    - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
    - mkdir ~/.ssh && touch ~/.ssh/known_hosts
    - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
    # Set git config
    - git config user.email "devops-team@parity.io"
    - git config user.name "${GITHUB_USER}"
    - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
    - git fetch origin gh-pages
    # Save README and docs
    - cp -r ./crate-docs/ /tmp/doc/
    - cp README.md /tmp/doc/
    # we don't need to commit changes because we copy docs to /tmp
    - git checkout gh-pages --force
    # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS
    - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud
    - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME}
    # Ensure the destination dir doesn't exist.
    - rm -rf ${CI_COMMIT_REF_NAME}
    - mv -f /tmp/doc ${CI_COMMIT_REF_NAME}
    # Upload files
    - git add --all
    # `git commit` has an exit code of > 0 if there is nothing to commit.
    # This causes GitLab to exit immediately and marks this job failed.
    # We don't want to mark the entire job failed if there's nothing to
    # publish though, hence the `|| true`.
    - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
        echo "___Nothing to commit___"
    - git push origin gh-pages --force
  after_script:
    - rm -rf .git/ ./*

publish-draft-release:
  stage:                           publish
  image:                           paritytech/tools:latest
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
  script:
    - ./scripts/ci/gitlab/publish_draft_release.sh
  allow_failure:                   true

# Ref: https://github.com/paritytech/opstooling/issues/111
update-node-template:
  stage:                           publish
  extends:                         .kubernetes-env
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1
  script:
    - git clone --depth=1 --branch="$PIPELINE_SCRIPTS_TAG" https://github.com/paritytech/pipeline-scripts
    - ./pipeline-scripts/update_substrate_template.sh
      --repo-name "substrate-node-template"
      --template-path "bin/node-template"
      --github-api-token "$GITHUB_TOKEN"
      --polkadot-branch "$CI_COMMIT_REF_NAME"

.publish-crates-template:
  stage:                           publish
  extends:                         .crates-publishing-template
  # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite
  # the releases of one another. Use resource_group to ensure that at most one instance of this job
  # is running at any given time.
  resource_group:                  crates-publishing
  variables:
    # crates.io rate limits crates publishing by 1 per minute, so a delay needs to be inserted
    # slightly higher than that after publishing each crate. The value is specified in seconds.
    SPUB_AFTER_PUBLISH_DELAY: 64
  # We might have to publish lots of crates at a time. Given the 1 minute delay introduced above and
  # taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, that would equate
  # to roughly 202 minutes of delay, or 3h and 22 minutes. As such, the job needs to have a much
  # higher timeout than average.
  timeout:                         9h
  # A custom publishing environment is used for us to be able to set up protected secrets
  # specifically for it
  environment: publish-crates
  script:
    - rusty-cachier snapshot create
    - git clone
        --depth 1
        --branch "$RELENG_SCRIPTS_BRANCH"
        https://github.com/paritytech/releng-scripts.git
    - CRATESIO_TARGET_INSTANCE=default ./releng-scripts/publish-crates
    - rusty-cachier cache upload

publish-crates:
  extends:                         .publish-crates-template
  needs:
    - job:                         publish-crates-locally
      artifacts:                   false
  rules:
    - if: $CI_COMMIT_REF_NAME == "master"

publish-crates-manual:
  extends:                         .publish-crates-template
  when:                            manual
  allow_failure:                   true