GODRIVER-3338 Remove usage of PREPARE_SHELL in Evergreen Config (mong…

…odb#1843)

.evergreen/run-task.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Source the env.sh file and run the given task
+set -eu
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+PROJECT_DIRECTORY=$(dirname $SCRIPT_DIR)
+pushd ${PROJECT_DIRECTORY} > /dev/null
+
+source env.sh
+task "$@"
+
+popd > /dev/null

.evergreen/setup-system.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+#
+# Set up environment and write env.sh and expansion.yml files.
+set -eu
+
+# Set up default environment variables.
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+PROJECT_DIRECTORY=$(dirname $SCRIPT_DIR)
+ROOT_DIR=$(dirname $PROJECT_DIRECTORY)
+DRIVERS_TOOLS=${DRIVERS_TOOLS:-${ROOT_DIR}/drivers-evergreen-tools}
+MONGO_ORCHESTRATION_HOME="${DRIVERS_TOOLS}/.evergreen/orchestration"
+MONGODB_BINARIES="${DRIVERS_TOOLS}/mongodb/bin"
+OS="${OS:-""}"
+
+# Set Golang environment vars. GOROOT is wherever current Go distribution is, and is set in evergreen config.
+# GOPATH is always 3 directories up from pwd on EVG; GOCACHE is under .cache in the pwd.
+GOROOT=${GOROOT:-$(dirname "$(dirname "$(which go)")")}
+export GOPATH=${GOPATH:-$ROOT_DIR}
+export GOCACHE="${GO_CACHE:-$PROJECT_DIRECTORY/.cache}"
+
+# Handle paths on Windows.
+if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
+  GOPATH=$(cygpath -m $GOPATH)
+  GOCACHE=$(cygpath -w $GOCACHE)
+  DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
+  PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
+  EXTRA_PATH=/cygdrive/c/libmongocrypt/bin
+  MONGO_ORCHESTRATION_HOME=$(cygpath -m $MONGO_ORCHESTRATION_HOME)
+  MONGODB_BINARIES=$(cygpath -m $MONGODB_BINARIES)
+  # Set home variables for Windows, too.
+  USERPROFILE=$(cygpath -w "$ROOT_DIR")
+  HOME=$USERPROFILE
+else
+  EXTRA_PATH=${GCC:-}
+fi
+
+# Add binaries to the path.
+PATH="${GOROOT}/bin:${GOPATH}/bin:${MONGODB_BINARIES}:${EXTRA_PATH}:${PATH}"
+
+# Get the current unique version of this checkout.
+if [ "${IS_PATCH:-}" = "true" ]; then
+    CURRENT_VERSION=$(git describe)-patch-${VERSION_ID}
+else
+    CURRENT_VERSION=latest
+fi
+
+# Ensure a checkout of drivers-tools.
+if [ ! -d "$DRIVERS_TOOLS" ]; then
+  git clone https://github.com/mongodb-labs/drivers-evergreen-tools $DRIVERS_TOOLS
+fi
+
+# Write the .env file for drivers-tools.
+cat <<EOT > ${DRIVERS_TOOLS}/.env
+SKIP_LEGACY_SHELL=1
+DRIVERS_TOOLS="$DRIVERS_TOOLS"
+MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
+MONGODB_BINARIES="$MONGODB_BINARIES"
+TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
+EOT
+
+# Check Go installation.
+go version
+go env
+
+# Install taskfile.
+go install github.com/go-task/task/v3/cmd/[email protected]
+
+# Write our own env file.
+cat <<EOT > env.sh
+export GOROOT="$GOROOT"
+export GOPATH="$GOPATH"
+export GOCACHE="$GOCACHE"
+export DRIVERS_TOOLS="$DRIVERS_TOOLS"
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+export PATH="$PATH"
+EOT
+
+if [ "Windows_NT" = "$OS" ]; then
+    echo "export USERPROFILE=$USERPROFILE" >> env.sh
+    echo "export HOME=$HOME" >> env.sh
+fi
+
+# source the env.sh file and write the expansion file.
+cat <<EOT > expansion.yml
+CURRENT_VERSION: "$CURRENT_VERSION"
+DRIVERS_TOOLS: "$DRIVERS_TOOLS"
+PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
+RUN_TASK: "$PROJECT_DIRECTORY/.evergreen/run-task.sh"
+EOT
+
+cat env.sh

.gitignore

@@ -20,6 +20,9 @@ libmongocrypt
 venv
 test.suite
 go.work.sum
+.task
+env.sh
+expansion.yml
 
 # AWS SAM-generated files
 internal/cmd/faas/awslambda/.aws-sam
@@ -36,3 +39,4 @@ api-report.txt
 # Ignore secrets files
 secrets-expansion.yml
 secrets-export.sh
+.test.env

.pre-commit-config.yaml

@@ -34,7 +34,7 @@ repos:
   hooks:
     - id: shellcheck
       name: shellcheck
-      args: ["--severity=error"]
+      args: ["--severity=warning"]
 
 - repo: https://github.com/codespell-project/codespell
   rev: "v2.2.6"

Taskfile.yml

@@ -5,6 +5,8 @@ env:
   TEST_TIMEOUT: 1800
   LONG_TEST_TIMEOUT: 3600
 
+dotenv: ['.test.env']
+
 tasks:
 
   ### Utility tasks. ###
@@ -16,7 +18,7 @@ tasks:
   check-license: bash etc/check_license.sh
 
   build:
-    deps: [cross-compile, build-tests, build-compile-check]
+    deps: [cross-compile, build-tests, build-compile-check, install-libmongocrypt]
     cmds:
       - go build ./...
       - go build ${BUILD_TAGS} ./...
@@ -25,6 +27,8 @@ tasks:
 
   build-compile-check: bash etc/compile_check.sh
 
+  build-aws-ecs-test: go build ${BUILD_TAGS} ./internal/cmd/testaws/main.go
+
   cross-compile: 
     - GOOS=linux GOARCH=386 go build ./...
     - GOOS=linux GOARCH=arm go build ./...
@@ -46,10 +50,15 @@ tasks:
 
   api-report: bash etc/api_report.sh
 
-  install-libmongocrypt: bash etc/install-libmongocrypt.sh
+  install-libmongocrypt: 
+    cmds: [bash etc/install-libmongocrypt.sh]
+    status: 
+      - test -d install || test -d /cygdrive/c/libmongocrypt/bin
 
   run-docker: bash etc/run_docker.sh
 
+  run-fuzz: bash etc/run-fuzz.sh
+
   cherry-picker: bash etc/cherry-picker.sh
 
   # Lint with various GOOS and GOARCH tasks to catch static analysis failures that may only affect
@@ -80,7 +89,8 @@ tasks:
 
   test-oidc: bash etc/run-oidc-test.sh 'task --silent evg-test-oidc-auth'
 
-  test-atlas: bash etc/run-atlas-test.sh
+  test-atlas-connect: 
+    - go test -v -run ^TestAtlas$ go.mongodb.org/mongo-driver/v2/internal/cmd/testatlas -args "$ATLAS_REPL" "$ATLAS_SHRD" "$ATLAS_FREE" "$ATLAS_TLS11" "$ATLAS_TLS12" "$ATLAS_SERVERLESS" "$ATLAS_SRV_REPL" "$ATLAS_SRV_SHRD" "$ATLAS_SRV_FREE" "$ATLAS_SRV_TLS11" "$ATLAS_SRV_TLS12" "$ATLAS_SRV_SERVERLESS" >> test.suite
 
   test-awskms: bash etc/run-awskms-test.sh
 
@@ -92,7 +102,7 @@ tasks:
       - make -c internal/cmd/faas/awslambda
 
   ### Evergreen specific tasks. ###
-  build-aws-ecs-test: go build ${BUILD_TAGS} ./internal/cmd/testaws/main.go
+  setup-test: bash etc/setup-test.sh {{.CLI_ARGS}}
 
   evg-test:
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=$MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s -p 1 ./... >> test.suite
@@ -101,7 +111,8 @@ tasks:
     - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestUnifiedSpecs/atlas-data-lake-testing >> spec_test.suite
     - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestAtlasDataLake >> spec_test.suite
 
-  evg-test-enterprise-auth: go run -tags gssapi ./internal/cmd/testentauth/main.go
+  evg-test-enterprise-auth: 
+    - go run -tags gssapi ./internal/cmd/testentauth/main.go
 
   evg-test-oidc-auth:
     - go run ./internal/cmd/testoidcauth/main.go
@@ -164,6 +175,12 @@ tasks:
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=${MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s ./internal/integration >> test.suite
     - go test -exec "env PKG_CONFIG_PATH=${PKG_CONFIG_PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DYLD_LIBRARY_PATH=${MACOS_LIBRARY_PATH}" ${BUILD_TAGS} -v -timeout {{.TEST_TIMEOUT}}s ./internal/integration/unified >> test.suite
 
+  evg-test-aws: bash etc/run-mongodb-aws-test.sh {{.CLI_ARGS}}
+
+  evg-test-aws-ecs: bash etc/run-mongodb-aws-ecs-test.sh
+
+  evg-test-deployed-lambda-aws: bash ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh
+
   build-kms-test: go build ${BUILD_TAGS} ./internal/cmd/testkms
 
   ### Benchmark specific tasks and support. ###

docs/CONTRIBUTING.md

@@ -183,7 +183,9 @@ However, some of the tests require secrets handling.  Please see the team [Wiki]
 The test suite can be run with or without the secrets as follows:
 
 ```bash
-TASKFILE_TARGET=evg-test-versioned-api bash .evergreen/run-tests.sh
+task setup-env
+task setup-test
+task evg-test-versioned-api
 ```
 
 ### Load Balancer

etc/check_license.sh

@@ -12,7 +12,8 @@ add_copyright() {
     file=$1
 
     # Check if first 24 bytes match first 24 bytes of copyright notice.
-    local line=$(head -c 24 $file)
+    local line
+    line=$(head -c 24 $file)
     if [ "$line" == "// Copyright (C) MongoDB" ]; then
         if [ ! -z "$verbose" ]; then
             echo "$file already has copyright notice" >&2
@@ -21,7 +22,7 @@ add_copyright() {
     fi
 
     # Check if first 14 bytes matches the prefix "// Copied from"
-    local line=$(head -c 14 $file)
+    line=$(head -c 14 $file)
     if [ "$line" == "// Copied from" ]; then
         if [ ! -z "$verbose" ]; then
             echo "$file has a third-party copyright notice" >&2
@@ -46,6 +47,10 @@ do
     case "${flag}" in
         a) add=1;;
         v) verbose=1;;
+        *)
+            echo "flag not recognized"
+            exit 1
+            ;;
     esac
 done
 

etc/check_modules.sh

@@ -8,7 +8,7 @@ set -eu
 mods=$(find . -name go.mod)
 exit_code=0
 for mod in $mods; do
-  pushd $(dirname $mod) > /dev/null
+  pushd "$(dirname $mod)" > /dev/null
   echo "Checking $mod..."
   go mod tidy -v
   git diff --exit-code go.mod go.sum || {