fix: fix the builds for public projects that use yarn 2 (or above) with workspaces

run-build-functions.sh

@@ -88,18 +88,39 @@ install_deps() {
 
 restore_node_modules() {
   local workspace_output
-  local workspace_exit_code
+  local package_manager_version
+  local package_locations
   local installer=$1
-  # YARN_IGNORE_PATH will ignore the presence of a local yarn executable (i.e. yarn 2) and default
-  # to using the global one (which, for now, is always yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath
-  # we can actually use this command for npm workspaces as well
-  workspace_output="$(YARN_IGNORE_PATH=1 yarn workspaces --json info 2>/dev/null)"
-  workspace_exit_code=$?
-  if [ $workspace_exit_code -eq 0 ]
-  then
-    echo "$installer workspaces detected"
-    local package_locations
-    # Extract all the packages and respective locations. .data will be a JSON object like
+
+  # Workspaces do not work for public repos in yarn 1.x,
+  # but this limitation has been removed in later major versions.
+  # (see https://yarnpkg.com/features/workspaces#how-to-declare-a-worktree).
+  # Due to this difference we cannot use the global installation
+  # of yarn (which is always 1.x) to get the list of workspaces.
+  # Thus, we need to get the list of workspaces for yarn 1 and
+  # later versions separately.
+
+  package_manager_version="$(yarn --version)"
+
+  if [ "$installer" = "yarn" ] && [ "${package_manager_version:0:1}" -gt 1 ]
+  then
+    # From yarn 2 there's no way to output information about repo's workspaces
+    # in pure json. Here's how the output of `yarn workspaces list --json` looks like
+    # (stringified json objects separated with new lines):
+    #
+    # ```
+    # {"location":".","name":"the-monorepo"}
+    # {"location":"packages/blog-1","name":"dev blog"}
+    # {"location":"packages/blog-2","name":"personal blog"}
+    # ```
+    workspace_output=[$(paste -s -d ',' <(yarn workspaces list --json 2>/dev/null))]
+
+    mapfile -t package_locations <<<"$(jq -r '.[].location | select(. != ".")' <<<"$workspace_output")"
+  else
+    local workspace_exit_code
+    # We can actually use this command for npm workspaces as well
+    workspace_output="$(yarn workspaces --json info 2>/dev/null)"
+    # .data will be a JSON object like
     # {
     #   "my-package-1": {
     #     "location": "packages/blog-1",
@@ -109,7 +130,17 @@ restore_node_modules() {
     #   (...)
     # }
     # We need to cache all the node_module dirs, or we'll always be installing them on each run
-    mapfile -t package_locations <<< "$(echo "$workspace_output" | jq -r '.data | fromjson | to_entries | .[].value.location')"
+    workspace_exit_code=$?
+    if [ "$workspace_exit_code" -eq 0 ]
+    then
+      mapfile -t package_locations <<<"$(echo "$workspace_output" | jq -r '.data | fromjson | to_entries | .[].value.location')"
+    fi
+  fi
+
+  if [ ${#package_locations[@]} -ne 0 ]
+  then
+    echo "$installer workspaces detected"
+    # Extract all the packages and respective locations.
     restore_js_workspaces_cache "${package_locations[@]}"
   else
     echo "No $installer workspaces detected"

tests/node/fixtures/yarn-v1-workspaces/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "yarn-v1-workspaces",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "UNLICENSED",
+  "dependencies": {
+    "@netlify/plugins-list": "^3.6.0"
+  },
+  "private": true,
+  "workspaces": [
+    "workspace-a",
+    "workspace-b"
+  ]
+}

tests/node/fixtures/yarn-v1-workspaces/workspace-a/package.json

@@ -0,0 +1,7 @@
+{
+"name": "workspace-a",
+"version": "1.0.0",
+"dependencies": {
+  "@netlify/plugins-list": "^3.6.0"
+}
+}

tests/node/fixtures/yarn-v1-workspaces/workspace-b/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "workspace-b",
+  "version": "1.0.0",
+  "dependencies": {
+    "@netlify/plugins-list": "^3.6.0"
+  }
+}

tests/node/fixtures/yarn-v2-workspaces/.yarnrc.yml

@@ -0,0 +1,3 @@
+nodeLinker: node-modules
+
+yarnPath: .yarn/releases/yarn-3.2.0.cjs

tests/node/fixtures/yarn-v2-workspaces/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "yarn-v2-workspaces",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "UNLICENSED",
+  "dependencies": {
+    "@netlify/plugins-list": "^3.6.0"
+  },
+  "workspaces": [
+    "workspace-a",
+    "workspace-b"
+  ]
+}

tests/node/fixtures/yarn-v2-workspaces/workspace-a/package.json

@@ -0,0 +1,7 @@
+{
+"name": "workspace-a",
+"version": "1.0.0",
+"dependencies": {
+  "@netlify/plugins-list": "^3.6.0"
+}
+}

tests/node/fixtures/yarn-v2-workspaces/workspace-b/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "workspace-b",
+  "version": "1.0.0",
+  "dependencies": {
+    "@netlify/plugins-list": "^3.6.0"
+  }
+}

tests/node/yarn-v1-workspaces.bats

@@ -0,0 +1,58 @@
+#!/usr/bin/env bats
+
+load '../helpers.sh'
+
+load '../../node_modules/bats-support/load'
+load '../../node_modules/bats-assert/load'
+load '../../node_modules/bats-file/load'
+
+setup() {
+  TMP_DIR=$(setup_tmp_dir)
+  REPO_DIR="${TMP_DIR}/repo"
+  DEPENDENCY_NAME="some-dependency"
+  set_fixture_as_repo 'yarn-v1-workspaces' "$TMP_DIR"
+
+
+  # Load functions
+  load '../../run-build-functions.sh'
+
+  local js_workspaces=$TMP_DIR/cache/js-workspaces
+
+  # Create the directories that should be restored
+  # from cache
+  mkdir -p $js_workspaces/node_modules/$DEPENDENCY_NAME
+  mkdir -p $js_workspaces/workspace-a/node_modules/$DEPENDENCY_NAME
+  mkdir -p $js_workspaces/workspace-b/node_modules/$DEPENDENCY_NAME
+
+  source_nvm
+}
+
+teardown() {
+  rm -rf "$TMP_DIR"
+
+  # Return to original dir
+  cd - || return
+}
+
+@test 'project uses yarn v1' {
+ run yarn --version
+ assert_output '1.22.10'
+}
+
+@test 'restore_node_modules correctly restores workspace dependencies' {
+  # Make sure that the directories we'll restore from the cache
+  # do not exist yet
+  assert_dir_not_exist $REPO_DIR/node_modules/$DEPENDENCY_NAME
+  assert_dir_not_exist $REPO_DIR/workspace-a/node_modules/$DEPENDENCY_NAME
+  assert_dir_not_exist $REPO_DIR/workspace-b/node_modules/$DEPENDENCY_NAME
+
+  # I couldn't find a way to use multiline string assertions
+  # in bats, that's why I can't use `assert_output` here
+  logs=$(restore_node_modules 'yarn' | paste -s -d ',')
+  assert_equal "$logs" "yarn workspaces detected,Started restoring workspace workspace-a node modules,Finished restoring workspace workspace-a node modules,Started restoring workspace workspace-b node modules,Finished restoring workspace workspace-b node modules,Started restoring workspace root node modules,Finished restoring workspace root node modules"
+
+  # Check if the all the dependencies have been restored
+  assert_dir_exist $REPO_DIR/node_modules/$DEPENDENCY_NAME
+  assert_dir_exist $REPO_DIR/workspace-a/node_modules/$DEPENDENCY_NAME
+  assert_dir_exist $REPO_DIR/workspace-b/node_modules/$DEPENDENCY_NAME
+}

tests/node/yarn-v1.bats

@@ -0,0 +1,49 @@
+#!/usr/bin/env bats
+
+load '../helpers.sh'
+
+load '../../node_modules/bats-support/load'
+load '../../node_modules/bats-assert/load'
+load '../../node_modules/bats-file/load'
+
+setup() {
+  TMP_DIR=$(setup_tmp_dir)
+  REPO_DIR="${TMP_DIR}/repo"
+  DEPENDENCY_NAME="some-dependency"
+  set_fixture_as_repo 'simple-node' "$TMP_DIR"
+
+
+  # Load functions
+  load '../../run-build-functions.sh'
+
+  # Create a fake dependency that should be restored
+  # from cache
+  mkdir $TMP_DIR/cache/node_modules/$DEPENDENCY_NAME
+
+  source_nvm
+}
+
+teardown() {
+  rm -rf "$TMP_DIR"
+
+  # Return to original dir
+  cd - || return
+}
+
+@test 'project uses yarn 1.x' {
+  run yarn --version
+  assert_output '1.22.10'
+}
+
+@test 'restore_node_modules correctly restores dependencies' {
+  # Make sure that that directory we'll restore from the cache
+  # does not exist
+  assert_dir_not_exist $REPO_DIR/node_modules/$DEPENDENCY_NAME
+
+  # I couldn't find a way to use multiline string assertions
+  # in bats, that's why I can't use `assert_output` here
+  logs=$(restore_node_modules 'yarn' | paste -s -d ',')
+  assert_equal "$logs" "No yarn workspaces detected,Started restoring cached node modules,Finished restoring cached node modules"
+
+  assert_dir_exist $REPO_DIR/node_modules/$DEPENDENCY_NAME
+}

tests/node/yarn-v2-workspaces.bats

@@ -0,0 +1,57 @@
+#!/usr/bin/env bats
+
+load '../helpers.sh'
+
+load '../../node_modules/bats-support/load'
+load '../../node_modules/bats-assert/load'
+load '../../node_modules/bats-file/load'
+
+setup() {
+  TMP_DIR=$(setup_tmp_dir)
+  REPO_DIR="${TMP_DIR}/repo"
+  DEPENDENCY_NAME="some-dependency"
+  set_fixture_as_repo 'yarn-v2-workspaces' "$TMP_DIR"
+
+  # Load functions
+  load '../../run-build-functions.sh'
+
+  local js_workspaces=$TMP_DIR/cache/js-workspaces
+
+  # Create the directories that should be restored
+  # from cache
+  mkdir -p $js_workspaces/node_modules/$DEPENDENCY_NAME
+  mkdir -p $js_workspaces/workspace-a/node_modules/$DEPENDENCY_NAME
+  mkdir -p $js_workspaces/workspace-b/node_modules/$DEPENDENCY_NAME
+
+  source_nvm
+}
+
+teardown() {
+  rm -rf "$TMP_DIR"
+
+  # Return to original dir
+  cd - || return
+}
+
+@test 'project uses yarn 2+' {
+  run yarn --version
+  assert_output '3.2.0'
+}
+
+@test 'restore_node_modules correctly restores workspace dependencies' {
+  # Make sure that the directories we'll restore from the cache
+  # do not exist yet
+  assert_dir_not_exist $REPO_DIR/node_modules/$DEPENDENCY_NAME
+  assert_dir_not_exist $REPO_DIR/workspace-a/node_modules/$DEPENDENCY_NAME
+  assert_dir_not_exist $REPO_DIR/workspace-b/node_modules/$DEPENDENCY_NAME
+
+  # I couldn't find a way to use multiline string assertions
+  # in bats, that's why I can't use `assert_output` here
+  logs=$(restore_node_modules 'yarn' | paste -s -d ',')
+  assert_equal "$logs" "yarn workspaces detected,Started restoring workspace workspace-a node modules,Finished restoring workspace workspace-a node modules,Started restoring workspace workspace-b node modules,Finished restoring workspace workspace-b node modules,Started restoring workspace root node modules,Finished restoring workspace root node modules"
+
+  # Check if the all the dependencies have been restored
+  assert_dir_exist $REPO_DIR/node_modules/$DEPENDENCY_NAME
+  assert_dir_exist $REPO_DIR/workspace-a/node_modules/$DEPENDENCY_NAME
+  assert_dir_exist $REPO_DIR/workspace-b/node_modules/$DEPENDENCY_NAME
+}

tests/node/yarn.bats

@@ -11,6 +11,25 @@ YARN_CACHE_DIR=/opt/buildhome/.yarn_cache
 # So that we can speed up the `run_yarn` function and not require new yarn installs for tests
 YARN_DEFAULT_VERSION=1.22.10
 
+# So that we can backup the default installation of yarn
+YARN_DEFAULT_INSTALLATION_DIR=/opt/buildhome/.yarn
+YARN_DEFAULT_INSTALLATION_BACKUP=/opt/buildhome/yarn-default-installation
+
+setup_file() {
+  # We have to backup the default installation of yarn, because some of
+  # the tests in this file execute the `run_yarn` function, which
+  # removes the default installation.
+  mkdir $YARN_DEFAULT_INSTALLATION_BACKUP
+  cp -r $YARN_DEFAULT_INSTALLATION_DIR/*  $YARN_DEFAULT_INSTALLATION_BACKUP
+}
+
+teardown_file() {
+  # Restore the default yarn installation
+  assert_dir_not_exist $YARN_DEFAULT_INSTALLATION_DIR
+  assert_dir_exist $YARN_DEFAULT_INSTALLATION_BACKUP
+  mv $YARN_DEFAULT_INSTALLATION_BACKUP $YARN_DEFAULT_INSTALLATION_DIR
+}
+
 setup() {
   TMP_DIR=$(setup_tmp_dir)
   set_fixture_as_repo 'simple-node' "$TMP_DIR"