From fdf829b5f08ee12f7368670fc8e625b13fb1db98 Mon Sep 17 00:00:00 2001 From: galargh Date: Tue, 9 Jul 2024 16:27:40 +0100 Subject: [PATCH] feat: ci: remove gorelease, create drafts and automate release notes --- .github/workflows/docker.yml | 5 +- .github/workflows/release.yml | 278 +++++++++++++------ .gitignore | 3 + .goreleaser.yaml | 111 -------- cmd/release/main.go | 49 +++- documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 24 +- scripts/publish-checksums.sh | 56 ---- 7 files changed, 251 insertions(+), 275 deletions(-) delete mode 100644 .goreleaser.yaml delete mode 100755 scripts/publish-checksums.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6df6d0e4a4b..da17456d22b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,6 +10,7 @@ on: branches: - master - release/v* + - release/miner/v* - releases schedule: - cron: '0 0 * * *' @@ -29,7 +30,7 @@ permissions: jobs: docker: - name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}] + name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ github.event.inputs.publish == 'true' || github.event_name != 'pull_request' }}] runs-on: ubuntu-latest strategy: fail-fast: false @@ -45,7 +46,7 @@ jobs: - image: lotus network: mainnet env: - PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }} + PUBLISH: ${{ github.event.inputs.publish == 'true' || github.event_name != 'pull_request' }} steps: - id: channel env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9777542e0c8..9aee2b8688a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,29 +4,32 @@ on: push: branches: - release/v* + - release/miner/v* - releases paths: - build/version.go pull_request: branches: - release/v* + - release/miner/v* - releases paths: - build/version.go workflow_dispatch: inputs: + projects: + description: 'Projects to release (JSON array)' + required: false + default: '[]' + ref: + description: 'Ref to buuild the binaries from' + required: false + default: '' publish: description: 'Publish the release' required: false default: 'false' - draft: - description: 'Create a draft release' - required: false - default: 'true' - target_commitish: - description: 'The commitish value that determines where the Git tag is created from' - required: false - default: '' + defaults: run: shell: bash @@ -36,22 +39,29 @@ permissions: jobs: check: - name: Check which projects need to be built + name: Check which projects need to be released runs-on: ubuntu-latest outputs: projects: ${{ steps.projects.outputs.projects }} steps: - - uses: actions/checkout@v4 + - name: Check out lotus + uses: actions/checkout@v4 with: fetch-depth: 0 - - id: projects + ref: ${{ github.event.inputs.ref }} + - name: Find projects that need to be released + id: projects env: - TARGET_REF: ${{ github.base_ref || github.ref }} + projects: ${{ github.event.inputs.projects }} run: | go run cmd/release/main.go --json list-projects | jq -r '.msg' | jq 'map(select(.version | endswith("-dev") | not))' | jq 'map(select(.released | not))' | - jq 'map(select(.prerelease != ((env.TARGET_REF | sub("^refs/heads/"; "")) == "releases")))' | + jq 'map(select( + (env.GITHUB_EVENT_NAME == "push" and ((env.GITHUB_REF == "refs/heads/releases" and .prerelease == false) or (env.GITHUB_REF != "refs/heads/releases" and .prerelease == true))) or + (env.GITHUB_EVENT_NAME == "pull_request" and ((env.GITHUB_BASE_REF == "releases" and .prerelease == false) or (env.GITHUB_BASE_REF != "releases" and .prerelease == true))) or + (env.GITHUB_EVENT_NAME == "workflow_dispatch" and (.name as $name | env.projects | fromjson | index($name) != null)) + ))' | jq -c '.' | xargs -I {} -0 echo "projects={}" | tee -a $GITHUB_OUTPUT build: @@ -68,8 +78,10 @@ jobs: - macos-13 # MacOs X64 - macos-14 # MacOS ARM64 steps: - - run: echo "Building on $RUNNER_OS/$RUNNER_ARCH" - - id: project + - name: Print runner information + run: echo "Building on $RUNNER_OS/$RUNNER_ARCH" + - name: Make project config available + id: project env: projects: ${{ needs.check.outputs.projects }} name: ${{ matrix.project }} @@ -77,25 +89,36 @@ jobs: jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' | xargs -I {} -0 echo "config={}" | tee -a $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - name: Check out lotus + uses: actions/checkout@v4 with: submodules: 'recursive' - - uses: ./.github/actions/install-system-dependencies - - uses: ./.github/actions/install-go - - env: - GITHUB_TOKEN: ${{ github.token }} - run: make deps - - if: matrix.project == 'node' - env: - GITHUB_TOKEN: ${{ github.token }} - run: make lotus - - if: matrix.project == 'miner' + - name: Install system dependencies + uses: ./.github/actions/install-system-dependencies + - name: Install Go + uses: ./.github/actions/install-go + - if: github.event.inputs.ref + uses: actions/checkout@v4 + with: + submodules: 'recursive' + ref: ${{ github.event.inputs.ref }} + - name: Build binaries env: GITHUB_TOKEN: ${{ github.token }} - run: make lotus-miner lotus-worker - - if: runner.os == 'macOS' + binaries: ${{ toJSON(fromJSON(steps.project.outputs.config).binaries) }} + run: | + make deps + while read -r binary; do + if [[ -z "$binary" ]]; then + continue + fi + make $binary + done <<< "$(jq -r '.[]' <<< "$binaries")" + - name: Run otool + if: runner.os == 'macOS' run: if [[ -f lotus ]]; then otool -hv lotus; fi - - env: + - name: Verify binary versions + env: LOTUS_VERSION_IGNORE_COMMIT: 1 expected: ${{ fromJSON(steps.project.outputs.config).version }} run: | @@ -109,7 +132,8 @@ jobs: fi fi done - - uses: actions/upload-artifact@v4 + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: name: lotus-${{ matrix.project }}-${{ runner.os }}-${{ runner.arch }} path: | @@ -119,7 +143,7 @@ jobs: release: needs: [check, build] if: needs.check.outputs.projects != '[]' - name: Release ${{ matrix.project }} [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}] + name: Release ${{ matrix.project }} permissions: # This enables the job to create and/or update GitHub releases contents: write @@ -128,11 +152,9 @@ jobs: fail-fast: false matrix: project: ${{ fromJSON(needs.check.outputs.projects).*.name }} - env: - PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }} - IS_DRAFT: ${{ inputs.draft == 'true' || github.event_name == 'pull_request' }} steps: - - id: project + - name: Make project config available + id: project env: projects: ${{ needs.check.outputs.projects }} name: ${{ matrix.project }} @@ -140,79 +162,165 @@ jobs: jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' | xargs -I {} -0 echo "config={}" | tee -a $GITHUB_OUTPUT - - uses: actions/checkout@v4 + - name: Check out lotus + uses: actions/checkout@v4 with: submodules: 'recursive' fetch-depth: 0 - - uses: actions/download-artifact@v4 + - name: Download Linux X64 binaries + uses: actions/download-artifact@v4 with: name: lotus-${{ matrix.project }}-Linux-X64 path: linux_amd64_v1 - - uses: actions/download-artifact@v4 + - name: Download macOS X64 binaries + uses: actions/download-artifact@v4 with: name: lotus-${{ matrix.project }}-macOS-X64 path: darwin_amd64_v1 - - uses: actions/download-artifact@v4 + - name: Download macOS ARM64 binaries + uses: actions/download-artifact@v4 with: name: lotus-${{ matrix.project }}-macOS-ARM64 path: darwin_arm64 - - uses: ./.github/actions/install-go - - uses: ipfs/download-ipfs-distribution-action@v1 + - name: Install Go (release body & archives generation dependency) + uses: ./.github/actions/install-go + - name: Install makefat (archives generation dependency) + run: go install github.com/randall77/makefat@7ddd0e42c8442593c87c1705a5545099604008e5 + - name: Generate archives + env: + prefix: ${{ matrix.project == 'node' && 'lotus' || format('lotus-{0}', matrix.project) }} + version: ${{ fromJSON(steps.project.outputs.config).version }} + binaries: ${{ toJSON(fromJSON(steps.project.outputs.config).binaries) }} + run: | + mkdir darwin_all + while read -r binary; do + if [[ -z "$binary" ]]; then + continue + fi + makefat ./darwin_all/$binary ./darwin_amd64_v1/$binary ./darwin_arm64/$binary + done <<< "$(jq -r '.[]' <<< "$binaries")" + mkdir dist + pushd dist + for directory in darwin_all linux_amd64_v1; do + archive_name="${prefix}_v${version}_${directory}" + cp -r ../$directory $archive_name + tar -czf $archive_name.tar.gz $archive_name + rm -r $archive_name + done + popd + ls -la dist + - name: Install Kubo (checksums generation dependency) + uses: ipfs/download-ipfs-distribution-action@v1 with: name: kubo version: v0.16.0 cache: false - - name: Install yq + - name: Generate checksums + run: | + ./scripts/generate-checksums.sh + ls -la dist + - name: Install zsh (release body generation dependency) + run: sudo apt update && sudo apt install -y zsh + - name: Generate release body env: + tag: ${{ fromJSON(steps.project.outputs.config).tag }} + previous: ${{ fromJSON(steps.project.outputs.config).previous }} + current: ${{ github.event.inputs.ref || github.sha }} + run: | + echo "# $tag" > release_body.md + echo "" >> release_body.md + csplit --digits=4 --quiet --elide-empty-files CHANGELOG.md '/^# /' '{*}' + # Checking the files in reverse order to get to the UNRELEASED section last + for file in $(ls -r xx*); do + if grep -q "^# $tag " $file || grep -q "^# UNRELEASED" $file; then + tail -n +3 $file >> release_body.md + break + fi + done + if [[ "$previous" != '' ]]; then + mkdir -p "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + rm -rf "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + ln -s "$(pwd)" "$(go env GOPATH)/src/github.com/filecoin-project/lotus" + ./scripts/mkreleaselog "$previous" "$current" >> release_body.md + fi + cat release_body.md + - name: Find release + id: before + env: + tag: ${{ fromJSON(steps.project.outputs.config).tag }} GITHUB_TOKEN: ${{ github.token }} run: | - gh release --repo mikefarah/yq download v4.44.2 -p yq_linux_amd64 - sudo mv yq_linux_amd64 /usr/bin/yq - sudo chmod +x /usr/bin/yq - - if: env.IS_DRAFT == 'false' - run: yq -i '.release.draft = false' '.goreleaser.yaml' - - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' && fromJSON(steps.project.outputs.config).latest && !fromJSON(steps.project.outputs.config).prerelease - run: yq -i '.brews.[0].skip_upload = false' '.goreleaser.yaml' - - if: matrix.project == 'node' + echo "release< 0 { + previous = v + } + } + } + } + } + if previous == "" { + return "" + } + return prefix + previous +} + +func getBinaries(name string) []string { + if name == "node" { + return []string{"lotus"} + } + if name == "miner" { + return []string{"lotus-miner", "lotus-worker"} + } + return nil +} + func isReleased(tag string) bool { tags := getTags() for _, t := range tags { @@ -67,12 +100,14 @@ func getPrefix(name string) string { } type project struct { - Name string `json:"name"` - Version string `json:"version"` - Tag string `json:"tag"` - Latest bool `json:"latest"` - Prerelease bool `json:"prerelease"` - Released bool `json:"released"` + Name string `json:"name"` + Version string `json:"version"` + Tag string `json:"tag"` + Previous string `json:"previous"` + Latest bool `json:"latest"` + Prerelease bool `json:"prerelease"` + Released bool `json:"released"` + Binaries []string `json:"binaries"` } func getProject(name, version string) project { @@ -81,9 +116,11 @@ func getProject(name, version string) project { Name: name, Version: version, Tag: getPrefix(name) + version, + Previous: getPrevious(name, version), Latest: isLatest(name, version), Prerelease: isPrerelease(version), Released: isReleased(tag), + Binaries: getBinaries(name), } } diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index b35c26027b8..4fa59b9b4b7 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -13,21 +13,17 @@ **Note for whoever is owning the release:** please capture notes as comments in this issue for anything you noticed that could be improved for future releases. There is a *Post Release* step below for incorporating changes back into the [RELEASE_ISSUE_TEMPLATE](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), and this is easier done by collecting notes from along the way rather than just thinking about it at the end. -- [ ] Fork a new branch (`release/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage. +- [ ] Fork a new branch (`release/vX.Y.Z` or `release/miner/vX.Y.Z`) from `master` and make any further release related changes to this branch. If any "non-trivial" changes get added to the release, uncheck all the checkboxes and return to this stage. - [ ] Bump the version in `build/version.go` in the `master` branch to `vX.Y.(Z+1)-dev` (bump from feature release) or `vX.(Y+1).0-dev` (bump from mandatory release). Run make gen and make docsgen-cli before committing changes **Prepping an RC**: -Perform the following changes to the `release/vX.Y.Z` branch through a PR: +Perform the following changes to the `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch through a PR: - [ ] update the version string in `build/version.go` to one ending with '-rcX' - [ ] run `make gen && make docsgen-cli` to generate documentation -- [ ] run `scripts/mkreleaselog` to generate changelog - - [ ] add the contents of generated changelog to `CHANGELOG.md` - - [ ] add any other details about the release to `CHANGELOG.md` -- [ ] create a **PR** targetting `release/vX.Y.Z` branch - - Opening a PR will trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - - Merging the PR will trigger a CI run that will publish the release to GitHub (including the creation of a GitHub release and a tag) -- [ ] find the newly created [GitHub release](https://github.com/filecoin-project/lotus/releases) and update its' description accordingly +- [ ] create a **PR** targetting `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch + - Opening a PR will trigger a CI run that will build the release and publish it to GitHub as a draft + - Merging the PR will trigger a CI run that will publish the GitHub release (it will also create a git tag) **Testing** @@ -38,14 +34,12 @@ Test the release candidate thoroughly, including automated and manual tests to e Perform the following changes to the `release/vX.Y.Z` branch (optionally, through a PR): - [ ] update the version string in `build/version.go` to one **NOT** ending with '-rcX' - [ ] run `make gen && make docsgen-cli` to generate documentation -- [ ] ensure `CHANGELOG.md` is up to date (run `scripts/mkreleaselog` and update `CHANGELOG.md` if needed) -- [ ] either commit the changes directly or open a PR against the `release/vX.Y.Z` branch +- [ ] either commit the changes directly or open a PR against the `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch Perform the following changes to the `releases` branch through a PR: -- [ ] create a **PR** targetting `releases` branch (base) from `release/vX.Y.Z` branch (head) - - Opening a PR will trigger a CI run that will build the release and run goreleaser in a dry/snapshot mode - - Merging the PR will trigger a CI run that will publish the release to GitHub (including the creation of a GitHub release and a tag) -- [ ] find the newly created [GitHub release](https://github.com/filecoin-project/lotus/releases) and update its' description accordingly +- [ ] create a **PR** targetting `releases` branch (base) from `release/vX.Y.Z` or `release/miner/vX.Y.Z` branch (head) + - Opening a PR will trigger a CI run that will build the release and publish it to GitHub as a draft + - Merging the PR will trigger a CI run that will publish the GitHub release (it will also create a git tag) **Post-Release** diff --git a/scripts/publish-checksums.sh b/scripts/publish-checksums.sh deleted file mode 100755 index 1dc2e41de84..00000000000 --- a/scripts/publish-checksums.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -set -exo - -pushd dist - -# make sure we have a token set, api requests won't work otherwise -if [ -z "${GITHUB_TOKEN}" ]; then - echo "\${GITHUB_TOKEN} not set, publish failed" - exit 1 -fi - -if [ -z "${TAG}" ]; then - echo "\${TAG} not set, publish failed" - exit 1 -fi - -REQUIRED=( - "jq" - "curl" -) -for REQUIRE in "${REQUIRED[@]}" -do - command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" -done - -#see if the release already exists by tag -RELEASE_RESPONSE=` - curl \ - --fail \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" -` -RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` - -if [ "${RELEASE_ID}" = "null" ]; then - echo "https://github.com/repos/${GITHUB_REPOSITORY}/releases/tags/${TAG} does not exist, publish failed" -fi - -RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1` -echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}" - -for CHECKSUM_FILE in *.{cid,sha512} -do - echo "Uploading ${CHECKSUM_FILE}..." - curl \ - --fail \ - --request POST \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - --header "Content-Type: application/octet-stream" \ - --data-binary "@${CHECKSUM_FILE}" \ - "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")" - - echo "Uploaded ${CHECKSUM_FILE}" -done - -popd