diff --git a/build/.cachesalt b/build/.cachesalt index d63bdc3118947..26ad5de2bcab8 100644 --- a/build/.cachesalt +++ b/build/.cachesalt @@ -1 +1 @@ -2023-06-12T12:55:48.130Z +2023-07-20T13:31:34.746Z diff --git a/build/azure-pipelines/linux/install.sh b/build/azure-pipelines/linux/install.sh new file mode 100755 index 0000000000000..b8960fc5fd41d --- /dev/null +++ b/build/azure-pipelines/linux/install.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -e + +# To workaround the issue of yarn not respecting the registry value from .npmrc +yarn config set registry "$NPM_REGISTRY" + +if [ -z "$CC" ] || [ -z "$CXX" ]; then + # Download clang based on chromium revision used by vscode + curl -s https://raw.githubusercontent.com/chromium/chromium/108.0.5359.215/tools/clang/scripts/update.py | python - --output-dir=$PWD/.build/CR_Clang --host-os=linux + + # Download libcxx headers and objects from upstream electron releases + DEBUG=libcxx-fetcher \ + VSCODE_LIBCXX_OBJECTS_DIR=$PWD/.build/libcxx-objects \ + VSCODE_LIBCXX_HEADERS_DIR=$PWD/.build/libcxx_headers \ + VSCODE_LIBCXXABI_HEADERS_DIR=$PWD/.build/libcxxabi_headers \ + VSCODE_ARCH="$npm_config_arch" \ + node build/linux/libcxx-fetcher.js + + # Set compiler toolchain + # Flags for the client build are based on + # https://source.chromium.org/chromium/chromium/src/+/refs/tags/108.0.5359.215:build/config/arm.gni + # https://source.chromium.org/chromium/chromium/src/+/refs/tags/108.0.5359.215:build/config/compiler/BUILD.gn + # https://source.chromium.org/chromium/chromium/src/+/refs/tags/108.0.5359.215:build/config/c++/BUILD.gn + export CC=$PWD/.build/CR_Clang/bin/clang + export CXX=$PWD/.build/CR_Clang/bin/clang++ + export CXXFLAGS="-nostdinc++ -D__NO_INLINE__ -I$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit -D_LIBCPP_ABI_NAMESPACE=Cr" + export LDFLAGS="-stdlib=libc++ -fuse-ld=lld -flto=thin -L$PWD/.build/libcxx-objects -lc++abi -Wl,--lto-O0" + export VSCODE_REMOTE_CC=$(which gcc) + export VSCODE_REMOTE_CXX=$(which g++) +fi + +for i in {1..5}; do # try 5 times + yarn --frozen-lockfile --check-files && break + if [ $i -eq 3 ]; then + echo "Yarn failed too many times" >&2 + exit 1 + fi + echo "Yarn failed $i, trying again..." +done diff --git a/build/azure-pipelines/linux/product-build-linux-test.yml b/build/azure-pipelines/linux/product-build-linux-test.yml index a80139473f6d5..504bc0b304ba7 100644 --- a/build/azure-pipelines/linux/product-build-linux-test.yml +++ b/build/azure-pipelines/linux/product-build-linux-test.yml @@ -14,17 +14,6 @@ steps: GITHUB_TOKEN: "$(github-distro-mixin-password)" displayName: Download Electron and Playwright - - ${{ if eq(parameters.VSCODE_QUALITY, 'oss') }}: - - script: | - set -e - sudo apt-get update - sudo apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libkrb5-dev libxss1 dbus xvfb libgtk-3-0 libgbm1 - sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb - sudo chmod +x /etc/init.d/xvfb - sudo update-rc.d xvfb defaults - sudo service xvfb start - displayName: Setup build environment - - ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}: - script: | set -e diff --git a/build/azure-pipelines/linux/product-build-linux.yml b/build/azure-pipelines/linux/product-build-linux.yml index f366cf672f7ea..bf408683d9da9 100644 --- a/build/azure-pipelines/linux/product-build-linux.yml +++ b/build/azure-pipelines/linux/product-build-linux.yml @@ -41,15 +41,28 @@ steps: - script: tar -xzf $(Build.ArtifactStagingDirectory)/compilation.tar.gz displayName: Extract compilation output - - script: | - set -e - # Start X server - /etc/init.d/xvfb start - # Start dbus session - DBUS_LAUNCH_RESULT=$(sudo dbus-daemon --config-file=/usr/share/dbus-1/system.conf --print-address) - echo "##vso[task.setvariable variable=DBUS_SESSION_BUS_ADDRESS]$DBUS_LAUNCH_RESULT" - displayName: Setup system services - condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64')) + - script: | + set -e + # Start X server + sudo apt-get update + sudo apt-get install -y pkg-config \ + libxss1 \ + dbus \ + xvfb \ + libgtk-3-0 \ + libgbm1 \ + libxkbfile-dev \ + libsecret-1-dev \ + libkrb5-dev + sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb + sudo chmod +x /etc/init.d/xvfb + sudo update-rc.d xvfb defaults + sudo service xvfb start + # Start dbus session + sudo mkdir -p /var/run/dbus + DBUS_LAUNCH_RESULT=$(sudo dbus-daemon --config-file=/usr/share/dbus-1/system.conf --print-address) + echo "##vso[task.setvariable variable=DBUS_SESSION_BUS_ADDRESS]$DBUS_LAUNCH_RESULT" + displayName: Setup system services - script: node build/setup-npm-registry.js $NPM_REGISTRY condition: and(succeeded(), ne(variables['NPM_REGISTRY'], 'none')) @@ -72,7 +85,10 @@ steps: - script: | set -e npm config set registry "$NPM_REGISTRY" --location=project - npm config set always-auth=true --location=project + # npm >v7 deprecated the `always-auth` config option, refs npm/cli@72a7eeb + # following is a workaround for yarn to send authorization header + # for GET requests to the registry. + echo "always-auth=true" >> .npmrc yarn config set registry "$NPM_REGISTRY" condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM & Yarn @@ -83,17 +99,6 @@ steps: condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), ne(variables['NPM_REGISTRY'], 'none')) displayName: Setup NPM Authentication - # TODO@joaomoreno TODO@deepak1556 this should be part of the base image - - ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}: - - script: | - sudo apt-get update && sudo apt-get install -y ca-certificates curl gnupg - sudo mkdir -m 0755 -p /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt update && sudo apt install -y docker-ce-cli - displayName: Install Docker client - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - ${{ if and(ne(parameters.VSCODE_QUALITY, 'oss'), or(eq(parameters.VSCODE_ARCH, 'x64'), eq(parameters.VSCODE_ARCH, 'arm64'))) }}: - task: Docker@1 displayName: "Pull Docker image" @@ -105,74 +110,65 @@ steps: containerCommand: uname condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - ${{ if and(ne(parameters.VSCODE_QUALITY, 'oss'), eq(parameters.VSCODE_ARCH, 'arm64')) }}: - - script: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: Register Docker QEMU - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true'), eq(variables['VSCODE_ARCH'], 'arm64')) - - - script: | - sudo apt-get update && sudo apt-get install -y libkrb5-dev - displayName: Install libkrb5-dev - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - - script: | - set -e + - ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}: + - script: | + set -e - for i in {1..5}; do # try 5 times - yarn --cwd build --frozen-lockfile --check-files && break - if [ $i -eq 3 ]; then - echo "Yarn failed too many times" >&2 - exit 1 - fi - echo "Yarn failed $i, trying again..." - done - - if [ -z "$CC" ] || [ -z "$CXX" ]; then - # Download clang based on chromium revision used by vscode - curl -s https://raw.githubusercontent.com/chromium/chromium/108.0.5359.215/tools/clang/scripts/update.py | python - --output-dir=$PWD/.build/CR_Clang --host-os=linux - # Download libcxx headers and objects from upstream electron releases - DEBUG=libcxx-fetcher \ - VSCODE_LIBCXX_OBJECTS_DIR=$PWD/.build/libcxx-objects \ - VSCODE_LIBCXX_HEADERS_DIR=$PWD/.build/libcxx_headers \ - VSCODE_LIBCXXABI_HEADERS_DIR=$PWD/.build/libcxxabi_headers \ - VSCODE_ARCH="$(NPM_ARCH)" \ - node build/linux/libcxx-fetcher.js - # Set compiler toolchain - # Flags for the client build are based on - # https://source.chromium.org/chromium/chromium/src/+/refs/tags/108.0.5359.215:build/config/arm.gni - # https://source.chromium.org/chromium/chromium/src/+/refs/tags/108.0.5359.215:build/config/compiler/BUILD.gn - # https://source.chromium.org/chromium/chromium/src/+/refs/tags/108.0.5359.215:build/config/c++/BUILD.gn - export CC=$PWD/.build/CR_Clang/bin/clang - export CXX=$PWD/.build/CR_Clang/bin/clang++ - export CXXFLAGS="-nostdinc++ -D__NO_INLINE__ -I$PWD/.build/libcxx_headers -isystem$PWD/.build/libcxx_headers/include -isystem$PWD/.build/libcxxabi_headers/include -fPIC -flto=thin -fsplit-lto-unit -D_LIBCPP_ABI_NAMESPACE=Cr" - export LDFLAGS="-stdlib=libc++ -fuse-ld=lld -flto=thin -L$PWD/.build/libcxx-objects -lc++abi -Wl,--lto-O0" - export VSCODE_REMOTE_CC=$(which gcc) - export VSCODE_REMOTE_CXX=$(which g++) - fi - - for i in {1..5}; do # try 5 times - yarn --frozen-lockfile --check-files && break - if [ $i -eq 3 ]; then - echo "Yarn failed too many times" >&2 - exit 1 - fi - echo "Yarn failed $i, trying again..." - done - env: - npm_config_arch: $(NPM_ARCH) - ELECTRON_SKIP_BINARY_DOWNLOAD: 1 - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 - GITHUB_TOKEN: "$(github-distro-mixin-password)" - ${{ if and(ne(parameters.VSCODE_QUALITY, 'oss'), or(eq(parameters.VSCODE_ARCH, 'x64'), eq(parameters.VSCODE_ARCH, 'arm64'))) }}: - VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME: vscodehub.azurecr.io/vscode-linux-build-agent:centos7-devtoolset8-$(VSCODE_ARCH) - displayName: Install dependencies - condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + for i in {1..5}; do # try 5 times + yarn --cwd build --frozen-lockfile --check-files && break + if [ $i -eq 3 ]; then + echo "Yarn failed too many times" >&2 + exit 1 + fi + echo "Yarn failed $i, trying again..." + done + + docker run -e GITHUB_TOKEN -e npm_config_arch -e NPM_REGISTRY \ + -e VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME -e VSCODE_HOST_MOUNT \ + -e ELECTRON_SKIP_BINARY_DOWNLOAD -e PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD \ + -v /mnt/vss/_work/1/s:/home/builduser/vscode -v /mnt/vss/_work/1/s/.build/.netrc:/home/builduser/.netrc \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -u 1000:1000 \ + -w /home/builduser/vscode vscodehub.azurecr.io/vscode-linux-build-agent:bionic-$(VSCODE_ARCH) \ + /bin/bash -c "./build/azure-pipelines/linux/install.sh" + + sudo chown -R $USER:$USER /mnt/vss/_work/1/s + env: + npm_config_arch: $(NPM_ARCH) + NPM_REGISTRY: "$(NPM_REGISTRY)" + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + GITHUB_TOKEN: "$(github-distro-mixin-password)" + VSCODE_HOST_MOUNT: "/mnt/vss/_work/1/s" + ${{ if or(eq(parameters.VSCODE_ARCH, 'x64'), eq(parameters.VSCODE_ARCH, 'arm64')) }}: + VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME: vscodehub.azurecr.io/vscode-linux-build-agent:centos7-devtoolset8-$(VSCODE_ARCH) + displayName: Install dependencies + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) - - ${{ if ne(parameters.VSCODE_QUALITY, 'oss') }}: - script: node build/azure-pipelines/distro/mixin-npm condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) displayName: Mixin distro node modules + - ${{ else }}: + - script: | + set -e + + for i in {1..5}; do # try 5 times + yarn --frozen-lockfile --check-files && break + if [ $i -eq 3 ]; then + echo "Yarn failed too many times" >&2 + exit 1 + fi + echo "Yarn failed $i, trying again..." + done + env: + npm_config_arch: $(NPM_ARCH) + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + GITHUB_TOKEN: "$(github-distro-mixin-password)" + displayName: Install dependencies + condition: and(succeeded(), ne(variables.NODE_MODULES_RESTORED, 'true')) + - script: | set -e node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt @@ -253,12 +249,32 @@ steps: mv $(Build.ArtifactStagingDirectory)/cli/$APP_NAME $(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)/bin/$CLI_APP_NAME displayName: Make CLI executable + - script: | + set -e + docker run -v /mnt/vss/_work/1/s:/home/builduser/vscode \ + -v /mnt/vss/_work/1/s/.build/.netrc:/home/builduser/.netrc \ + -v /mnt/vss/_work/1/VSCode-linux-$(VSCODE_ARCH):/home/builduser/VSCode-linux-$(VSCODE_ARCH) \ + -u 1000:1000 \ + -w /home/builduser/vscode vscodehub.azurecr.io/vscode-linux-build-agent:bionic-$(VSCODE_ARCH) \ + yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-deb" + displayName: Prepare deb package + - script: | set -e yarn gulp "vscode-linux-$(VSCODE_ARCH)-build-deb" echo "##vso[task.setvariable variable=DEB_PATH]$(ls .build/linux/deb/*/deb/*.deb)" displayName: Build deb package + - script: | + set -e + docker run -v /mnt/vss/_work/1/s:/home/builduser/vscode \ + -v /mnt/vss/_work/1/s/.build/.netrc:/home/builduser/.netrc \ + -v /mnt/vss/_work/1/VSCode-linux-$(VSCODE_ARCH):/home/builduser/VSCode-linux-$(VSCODE_ARCH) \ + -u 1000:1000 \ + -w /home/builduser/vscode vscodehub.azurecr.io/vscode-linux-build-agent:bionic-$(VSCODE_ARCH) \ + yarn gulp "vscode-linux-$(VSCODE_ARCH)-prepare-rpm" + displayName: Prepare rpm package + - script: | set -e yarn gulp "vscode-linux-$(VSCODE_ARCH)-build-rpm" diff --git a/build/azure-pipelines/product-build.yml b/build/azure-pipelines/product-build.yml index 1be2db3b5fa3c..9ab1d7ce5d3b3 100644 --- a/build/azure-pipelines/product-build.yml +++ b/build/azure-pipelines/product-build.yml @@ -143,18 +143,6 @@ name: "$(Date:yyyyMMdd).$(Rev:r) (${{ parameters.VSCODE_QUALITY }})" resources: containers: - - container: vscode-bionic-x64 - image: vscodehub.azurecr.io/vscode-linux-build-agent:bionic-x64 - endpoint: VSCodeHub - options: --user 0:0 --cap-add SYS_ADMIN - - container: vscode-arm64 - image: vscodehub.azurecr.io/vscode-linux-build-agent:bionic-arm64 - endpoint: VSCodeHub - options: --user 0:0 --cap-add SYS_ADMIN - - container: vscode-armhf - image: vscodehub.azurecr.io/vscode-linux-build-agent:bionic-armhf - endpoint: VSCodeHub - options: --user 0:0 --cap-add SYS_ADMIN - container: snapcraft image: vscodehub.azurecr.io/vscode-linux-build-agent:snapcraft-x64 endpoint: VSCodeHub @@ -382,7 +370,6 @@ stages: - ${{ if eq(variables['VSCODE_CIBUILD'], true) }}: - job: Linuxx64UnitTest displayName: Unit Tests - container: vscode-bionic-x64 variables: VSCODE_ARCH: x64 NPM_ARCH: x64 @@ -398,7 +385,6 @@ stages: VSCODE_RUN_SMOKE_TESTS: false - job: Linuxx64IntegrationTest displayName: Integration Tests - container: vscode-bionic-x64 variables: VSCODE_ARCH: x64 NPM_ARCH: x64 @@ -414,7 +400,6 @@ stages: VSCODE_RUN_SMOKE_TESTS: false - job: Linuxx64SmokeTest displayName: Smoke Tests - container: vscode-bionic-x64 variables: VSCODE_ARCH: x64 NPM_ARCH: x64 @@ -431,7 +416,6 @@ stages: - ${{ if and(eq(variables['VSCODE_CIBUILD'], false), eq(parameters.VSCODE_BUILD_LINUX, true)) }}: - job: Linuxx64 - container: vscode-bionic-x64 variables: VSCODE_ARCH: x64 NPM_ARCH: x64 @@ -458,7 +442,6 @@ stages: - ${{ if and(eq(variables['VSCODE_CIBUILD'], false), eq(parameters.VSCODE_BUILD_LINUX_ARMHF, true)) }}: - job: LinuxArmhf - container: vscode-armhf variables: VSCODE_ARCH: armhf NPM_ARCH: arm @@ -474,7 +457,6 @@ stages: - ${{ if and(eq(variables['VSCODE_CIBUILD'], false), eq(parameters.VSCODE_BUILD_LINUX_ARM64, true)) }}: - job: LinuxArm64 - container: vscode-arm64 variables: VSCODE_ARCH: arm64 NPM_ARCH: arm64 diff --git a/build/gulpfile.vscode.linux.js b/build/gulpfile.vscode.linux.js index 90f75ccfabd7f..0d7d3c5b7f80c 100644 --- a/build/gulpfile.vscode.linux.js +++ b/build/gulpfile.vscode.linux.js @@ -297,12 +297,14 @@ const BUILD_TARGETS = [ BUILD_TARGETS.forEach(({ arch }) => { const debArch = getDebPackageArch(arch); const prepareDebTask = task.define(`vscode-linux-${arch}-prepare-deb`, task.series(util.rimraf(`.build/linux/deb/${debArch}`), prepareDebPackage(arch))); - const buildDebTask = task.define(`vscode-linux-${arch}-build-deb`, task.series(prepareDebTask, buildDebPackage(arch))); + gulp.task(prepareDebTask); + const buildDebTask = task.define(`vscode-linux-${arch}-build-deb`, buildDebPackage(arch)); gulp.task(buildDebTask); const rpmArch = getRpmPackageArch(arch); const prepareRpmTask = task.define(`vscode-linux-${arch}-prepare-rpm`, task.series(util.rimraf(`.build/linux/rpm/${rpmArch}`), prepareRpmPackage(arch))); - const buildRpmTask = task.define(`vscode-linux-${arch}-build-rpm`, task.series(prepareRpmTask, buildRpmPackage(arch))); + gulp.task(prepareRpmTask); + const buildRpmTask = task.define(`vscode-linux-${arch}-build-rpm`, buildRpmPackage(arch)); gulp.task(buildRpmTask); const prepareSnapTask = task.define(`vscode-linux-${arch}-prepare-snap`, task.series(util.rimraf(`.build/linux/snap/${arch}`), prepareSnapPackage(arch))); diff --git a/build/npm/postinstall.js b/build/npm/postinstall.js index d9280ffb1eb87..09df602a3bf5d 100644 --- a/build/npm/postinstall.js +++ b/build/npm/postinstall.js @@ -53,7 +53,10 @@ function yarnInstall(dir, opts) { console.log(`Installing dependencies in ${dir} inside container ${process.env['VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME']}...`); opts.cwd = root; - run('docker', ['run', '-e', 'GITHUB_TOKEN', '-e', 'npm_config_arch', '-v', `/mnt/vss/_work/1/s:/root/vscode`, '-v', `/mnt/vss/_work/1/s/.build/.netrc:/root/.netrc`, process.env['VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME'], 'yarn', '--cwd', dir, ...args], opts); + if (process.env['npm_config_arch'] === 'arm64') { + run('sudo', ['docker', 'run', '--rm', '--privileged', 'multiarch/qemu-user-static', '--reset', '-p', 'yes'], opts); + } + run('sudo', ['docker', 'run', '-e', 'GITHUB_TOKEN', '-e', 'npm_config_arch', '-v', `${process.env['VSCODE_HOST_MOUNT']}:/root/vscode`, '-v', `${process.env['VSCODE_HOST_MOUNT']}/.build/.netrc:/root/.netrc`, process.env['VSCODE_REMOTE_DEPENDENCIES_CONTAINER_NAME'], 'yarn', '--cwd', dir, ...args], opts); run('sudo', ['chown', '-R', `${userinfo.uid}:${userinfo.gid}`, `${dir}/node_modules`], opts); } else { console.log(`Installing dependencies in ${dir}...`);