Skip to content

Commit

Permalink
Add turbo / improve Rust build caching in GitHub Actions (#31464)
Browse files Browse the repository at this point in the history 
Implements Turborepo for the Next.js repository and leverage it for native build caching.

Co-authored-by: Maia Teegarden <[email protected]>
  • Loading branch information
@jaredpalmer @padmaia
jaredpalmer and padmaia authored Dec 16, 2021
1 parent 6ad8c33 commit ed022ed
Show file tree
Hide file tree
Showing 6 changed files with 164 additions and 72 deletions.
182 changes: 118 additions & 64 deletions .github/workflows/build_test_deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
outputs:
docsChange: ${{ steps.docs-change.outputs.DOCS_CHANGE }}
isRelease: ${{ steps.check-release.outputs.IS_RELEASE }}
weekNum: ${{ steps.get-week.outputs.WEEK }}
steps:
- name: Setup node
uses: actions/setup-node@v2
Expand Down Expand Up @@ -59,6 +60,9 @@ jobs:
else
echo "::set-output name=IS_RELEASE::false"
fi
# We use week in the turbo cache key to keep the cache from infinitely growing
- id: get-week
run: echo ::set-output name=WEEK::$(date +%U)

- uses: actions/cache@v2
id: cache-build
Expand Down Expand Up @@ -655,13 +659,20 @@ jobs:
path: ~/.cargo/git
key: stable-ubuntu-18.04-node@14-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }}

- name: Cache native binary
id: binary-cache
# We use week in the turbo cache key to keep the cache from infinitely growing
- id: get-week
run: echo ::set-output name=WEEK::$(date +%U)

- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
if: ${{ steps.docs-change.outputs.DOCS_CHANGE != 'docs only change' }}
with:
path: packages/next-swc/native/next-swc.linux-x64-gnu.node
key: dev-next-swc-nightly-2021-11-15-linux-x64-gnu-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}-
turbo-${{ github.job }}-canary-${{ steps.get-week.outputs.WEEK }}-
# We use restore-key to pick latest cache.
# We will not get exact match, but doc says
Expand All @@ -678,13 +689,16 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]
- run: npm i -g [email protected]

- name: Build
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' && steps.docs-change.outputs.DOCS_CHANGE != 'docs only change' }}
run: yarn build-native
if: ${{ steps.docs-change.outputs.DOCS_CHANGE != 'docs only change' }}
run: turbo run build-native --cache-dir=".turbo"
env:
MACOSX_DEPLOYMENT_TARGET: '10.13'
working-directory: packages/next-swc
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/[email protected]
Expand Down Expand Up @@ -833,12 +847,15 @@ jobs:
path: ~/.cargo/git
key: stable-${{ matrix.os }}-node@14-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }}

- name: Cache native binary
id: binary-cache
- name: Turbo cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.${{ matrix.name }}.node
key: next-swc-nightly-2021-11-15-${{ matrix.target }}-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ matrix.name }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ matrix.name }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-${{ matrix.name }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Cross build aarch64 setup
if: ${{ matrix.target == 'aarch64-apple-darwin' }}
Expand All @@ -861,11 +878,12 @@ jobs:
next-swc-cargo-cache-${{ matrix.os }}
- name: 'Build'
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
run: yarn build-native --release --target ${{ matrix.target }}
run: turbo run build-native --cache-dir=".turbo" -- --release --target ${{ matrix.target }}
env:
MACOSX_DEPLOYMENT_TARGET: '10.13'
working-directory: packages/next-swc
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/[email protected]
Expand Down Expand Up @@ -907,7 +925,7 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]

- run: npm i -g [email protected]
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
Expand All @@ -916,18 +934,23 @@ jobs:
override: true
target: i686-pc-windows-msvc

- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.win32-ia32-msvc.node
key: next-swc-nightly-2021-11-15-win32-ia32-msvc-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Build
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
shell: bash
run: yarn build-native --release --target i686-pc-windows-msvc
working-directory: packages/next-swc
run: turbo run build-native --cache-dir=".turbo" -- --release --target i686-pc-windows-msvc
env:
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/upload-artifact@v2
Expand All @@ -953,7 +976,7 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]

- run: npm i -g [email protected]
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
Expand All @@ -962,18 +985,23 @@ jobs:
override: true
target: aarch64-pc-windows-msvc

- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.win32-arm64-msvc.node
key: next-swc-nightly-2021-11-15-win32-arm64-msvc-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Build
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
shell: bash
run: yarn build-native --release --target aarch64-pc-windows-msvc
working-directory: packages/next-swc
run: turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-pc-windows-msvc
env:
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}

This comment has been minimized.

Sign in to view

Copy link
@jaredpalmer

jaredpalmer Dec 16, 2021

Author Contributor

@padmaia if we are only caching locally, you can remove the env: stuff
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/upload-artifact@v2
Expand Down Expand Up @@ -1011,17 +1039,19 @@ jobs:
docker pull ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
docker tag ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine builder
- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.linux-x64-musl.node
key: next-swc-nightly-2021-11-15-linux-x64-musl-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: 'Build'
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
run: |
docker run --rm -v ~/.cargo/git:/root/.cargo/git -v ~/.cargo/registry:/root/.cargo/registry -v $(pwd)/packages/next-swc:/build -w /build builder sh -c "npm i -g @napi-rs/[email protected] && yarn build-native --release --target x86_64-unknown-linux-musl"
docker run --rm -v ~/.cargo/git:/root/.cargo/git -v ~/.cargo/registry:/root/.cargo/registry -v $(pwd)/packages/next-swc:/build -w /build builder sh -c "npm i -g @napi-rs/[email protected] && npm i -g [email protected] && turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-musl"
- name: Upload artifact
uses: actions/upload-artifact@v2
Expand Down Expand Up @@ -1049,6 +1079,7 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]
- run: npm i -g [email protected]

- name: Install Rust
uses: actions-rs/toolchain@v1
Expand All @@ -1070,17 +1101,23 @@ jobs:
sudo apt-get update
sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu -y
- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.linux-arm64-gnu.node
key: next-swc-nightly-2021-11-15-linux-arm64-gnu-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Cross build aarch64
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
run: yarn build-native --release --target aarch64-unknown-linux-gnu
working-directory: packages/next-swc
run: turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-gnu
env:
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/upload-artifact@v2
Expand All @@ -1106,7 +1143,7 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]

- run: npm i -g [email protected]
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
Expand All @@ -1127,17 +1164,22 @@ jobs:
sudo apt-get update
sudo apt-get install gcc-aarch64-linux-gnu -y
- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.linux-arm64-musl.node
key: next-swc-nightly-2021-11-15-linux-arm64-musl-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Cross build aarch64
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
run: yarn build-native --release --target aarch64-unknown-linux-musl
working-directory: packages/next-swc
run: turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-musl
env:
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/upload-artifact@v2
Expand Down Expand Up @@ -1165,6 +1207,7 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]
- run: npm i -g [email protected]

- name: Install Rust
uses: actions-rs/toolchain@v1
Expand All @@ -1186,17 +1229,22 @@ jobs:
sudo apt-get update
sudo apt-get install gcc-arm-linux-gnueabihf -y
- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.linux-arm-gnueabihf.node
key: next-swc-nightly-2021-11-15-linux-arm-gnueabihf-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Cross build aarch64
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
run: yarn build-native --release --target armv7-unknown-linux-gnueabihf
working-directory: packages/next-swc
run: turbo run build-native --cache-dir=".turbo" -- --release --target armv7-unknown-linux-gnueabihf
env:
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/upload-artifact@v2
Expand All @@ -1222,6 +1270,7 @@ jobs:
# since the repo's dependencies aren't installed we need
# to install napi globally
- run: npm i -g @napi-rs/[email protected]
- run: npm i -g [email protected]

- name: Install Rust
uses: actions-rs/toolchain@v1
Expand All @@ -1231,20 +1280,25 @@ jobs:
override: true
target: aarch64-linux-android

- name: Cache native binary
id: binary-cache
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v2
with:
path: packages/next-swc/native/next-swc.android-arm64.node
key: next-swc-nightly-2021-11-15-android-arm64-${{ hashFiles('.github/workflows/build_test_deploy.yml', 'packages/next-swc/**') }}
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Build
if: ${{ steps.binary-cache.outputs.cache-hit != 'true' }}
shell: bash
run: |
export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/darwin-x86_64/bin/aarch64-linux-android24-clang"
yarn build-native --release --target aarch64-linux-android
working-directory: packages/next-swc
turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-linux-android
env:
TURBO_TOKEN: ${{secrets.TURBO_TOKEN}}
TURBO_TEAM: nextjs
TURBO_PROJECT: nextjs

- name: Upload artifact
uses: actions/upload-artifact@v2
Expand Down
Loading

0 comments on commit ed022ed

Please sign in to comment.