From cb904135dc62c969ae4b61cbbec0c915681355c9 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:10:46 -0700 Subject: [PATCH] CSHARP-4765: CI script to generate Driver's dev-package (#1169) --- .editorconfig | 1 + .gitignore | 1 + evergreen/evergreen.yml | 143 +++++++++++++++++- evergreen/install-dotnet.sh | 17 +++ evergreen/packages-pack.sh | 12 ++ evergreen/packages-push.sh | 26 ++++ evergreen/packages-version.sh | 21 +++ src/MongoDB.Bson/MongoDB.Bson.csproj | 2 + .../MongoDB.Driver.Core.csproj | 2 + .../MongoDB.Driver.GridFS.csproj | 2 + .../MongoDB.Driver.Legacy.csproj | 1 + src/MongoDB.Driver/MongoDB.Driver.csproj | 2 + 12 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 evergreen/install-dotnet.sh create mode 100644 evergreen/packages-pack.sh create mode 100644 evergreen/packages-push.sh create mode 100644 evergreen/packages-version.sh diff --git a/.editorconfig b/.editorconfig index ca2ed214681..1ab98bee463 100644 --- a/.editorconfig +++ b/.editorconfig @@ -187,5 +187,6 @@ indent_size = 2 # Shell scripts [*.sh] end_of_line = lf +indent_size = 2 [*.{cmd, bat}] end_of_line = crlf diff --git a/.gitignore b/.gitignore index 0e99a62229c..a853777a90c 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ src/packages # Nupkg artifacts *.nupkg +*.snupkg # NCrunch artifacts *.ncrunch* diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index cfbc6895f9a..213cfe2ffe1 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -31,10 +31,10 @@ functions: - command: git.get_project params: directory: mongo-csharp-driver - # Applies the subitted patch, if any + # Applies the submitted patch, if any # Deprecated. Should be removed. But still needed for certain agents (ZAP) - command: git.apply_patch - # Make an evergreen exapanstion file with dynamic values + # Make an evergreen expansion file with dynamic values - command: shell.exec params: working_dir: mongo-csharp-driver @@ -46,6 +46,11 @@ functions: CURRENT_VERSION=latest fi + if [ "${PACKAGE_TARGET}" != "" ]; then + PACKAGE_VERSION=$(PACKAGE_TARGET="${PACKAGE_TARGET}" sh ./evergreen/packages-version.sh) + fi + + export DOTNET_SDK_PATH="$(pwd)/../.dotnet" export DRIVERS_TOOLS="$(pwd)/../drivers-tools" if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin @@ -68,6 +73,8 @@ functions: MONGODB_BINARIES: "$MONGODB_BINARIES" UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + PACKAGE_VERSION: "$PACKAGE_VERSION" + DOTNET_SDK_PATH: "$DOTNET_SDK_PATH" PREPARE_SHELL: | set -o errexit set -o xtrace @@ -76,14 +83,16 @@ functions: export MONGODB_BINARIES="$MONGODB_BINARIES" export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - + export PACKAGE_VERSION="$PACKAGE_VERSION" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$MONGODB_BINARIES:$PATH" + export PATH="$DOTNET_SDK_PATH:$MONGODB_BINARIES:$PATH" export PROJECT="${project}" EOT # See what we've done cat expansion.yml + # Do not output expansion.yml contents after this point + # Add CSFLE variables that shouldn't be output to the logs cat <> expansion.yml PREPARE_CSFLE: | @@ -97,13 +106,19 @@ functions: export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY} set -o xtrace # Enable tracing. EOT - # Do not output expansion.yml contents after this point # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update params: file: mongo-csharp-driver/expansion.yml + install-dotnet: + - command: shell.exec + params: + script: | + ${PREPARE_SHELL} + OS=${OS} sh ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh + prepare-resources: - command: shell.exec params: @@ -1023,6 +1038,61 @@ functions: ${PREPARE_SHELL} echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json + packages-pack: + - command: shell.exec + params: + working_dir: mongo-csharp-driver + script: | + ${PREPARE_SHELL} + . ./evergreen/packages-pack.sh + + packages-push: + - command: shell.exec + params: + working_dir: mongo-csharp-driver + env: + PACKAGES_SOURCE: ${PACKAGES_SOURCE} + PACKAGES_SOURCE_KEY: ${PACKAGES_SOURCE_KEY} + script: | + ${PREPARE_SHELL} + . ./evergreen/packages-push.sh + + upload-package: + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/octet-stream} + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/octet-stream} + + download-package: + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg + bucket: mciuploads + pre: - func: fetch-source - func: prepare-resources @@ -1040,7 +1110,6 @@ post: - func: cleanup tasks: - - name: test-net472 commands: - func: bootstrap-mongo-orchestration @@ -1676,6 +1745,47 @@ tasks: export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' ./evergreen/run-csfle-gcp-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + - name: packages-create + commands: + - func: packages-pack + - func: upload-package + vars: + PACKAGE_ID: "MongoDB.Bson" + - func: upload-package + vars: + PACKAGE_ID: "MongoDB.Driver" + - func: upload-package + vars: + PACKAGE_ID: "MongoDB.Driver.Core" + - func: upload-package + vars: + PACKAGE_ID: "MongoDB.Driver.GridFS" + - func: upload-package + vars: + PACKAGE_ID: "mongocsharpdriver" + + - name: packages-push + commands: + - func: download-package + vars: + PACKAGE_ID: "MongoDB.Bson" + - func: download-package + vars: + PACKAGE_ID: "MongoDB.Driver" + - func: download-package + vars: + PACKAGE_ID: "MongoDB.Driver.Core" + - func: download-package + vars: + PACKAGE_ID: "MongoDB.Driver.GridFS" + - func: download-package + vars: + PACKAGE_ID: "mongocsharpdriver" + - func: packages-push + vars: + PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json" + PACKAGES_SOURCE_KEY: ${nuget_api_key} + axes: - id: version display_name: MongoDB Version @@ -1897,6 +2007,19 @@ task_groups: tasks: - test-csfle-with-gcp-kms + - name: dev-package + setup_task: + - func: fetch-source + vars: + PACKAGE_TARGET: "dev" + - func: prepare-resources + - func: fix-absolute-paths + - func: make-files-executable + - func: install-dotnet + tasks: + - packages-create + - packages-push + buildvariants: - matrix_name: "secure-tests" @@ -2200,3 +2323,11 @@ buildvariants: - name: test-smoke-tests-netcoreapp31 - name: test-smoke-tests-net50 - name: test-smoke-tests-net60 + +- name: dev-package-pack + git_tag_only: true + display_name: "Dev Package Pack" + run_on: ubuntu1804-small + tags: ["dev-package"] + tasks: + - name: dev-package diff --git a/evergreen/install-dotnet.sh b/evergreen/install-dotnet.sh new file mode 100644 index 00000000000..14fb49216ed --- /dev/null +++ b/evergreen/install-dotnet.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail + +DOTNET_SDK_PATH=./.dotnet +mkdir -p "$DOTNET_SDK_PATH" + +if [[ $OS =~ [Ww]indows.* ]]; then + echo "Downloading Windows .NET SDK installer into $DOTNET_SDK_PATH folder..." + curl -Lfo "$DOTNET_SDK_PATH"/dotnet-install.ps1 https://dot.net/v1/dotnet-install.ps1 + echo "Installing .NET LTS SDK..." + powershell.exe "$DOTNET_SDK_PATH"/dotnet-install.ps1 -Channel 6.0 -InstallDir "$DOTNET_SDK_PATH" -NoPath +else + echo "Downloading .NET SDK installer into $DOTNET_SDK_PATH folder..." + curl -Lfo "$DOTNET_SDK_PATH"/dotnet-install.sh https://dot.net/v1/dotnet-install.sh + echo "Installing .NET LTS SDK..." + bash "$DOTNET_SDK_PATH"/dotnet-install.sh --channel 6.0 --install-dir "$DOTNET_SDK_PATH" --no-path +fi diff --git a/evergreen/packages-pack.sh b/evergreen/packages-pack.sh new file mode 100644 index 00000000000..e013d079097 --- /dev/null +++ b/evergreen/packages-pack.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail + +if [ -z "$PACKAGE_VERSION" ]; then + PACKAGE_VERSION=$(sh ./evergreen/packages-version.sh) + echo Calculated PACKAGE_VERSION value: "$PACKAGE_VERSION" +fi + +echo Creating nuget package... + +dotnet clean ./CSharpDriver.sln +dotnet pack ./CSharpDriver.sln -o ./build/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg diff --git a/evergreen/packages-push.sh b/evergreen/packages-push.sh new file mode 100644 index 00000000000..2190e35e0c7 --- /dev/null +++ b/evergreen/packages-push.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail +set +o xtrace # Disable tracing. + +if [ -z "$PACKAGES_SOURCE" ]; then + echo "PACKAGES_SOURCE variable should be set" + exit 1 +fi + +if [ -z "$PACKAGES_SOURCE_KEY" ]; then + echo "PACKAGES_SOURCE_KEY variable should be set" + exit 1 +fi + +if [ -z "$PACKAGE_VERSION" ]; then + echo "PACKAGE_VERSION variable should be set" + exit 1 +fi + +dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Bson."$PACKAGE_VERSION".nupkg +dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver.Core."$PACKAGE_VERSION".nupkg +dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver."$PACKAGE_VERSION".nupkg +dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver.GridFS."$PACKAGE_VERSION".nupkg +dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/mongocsharpdriver."$PACKAGE_VERSION".nupkg + + diff --git a/evergreen/packages-version.sh b/evergreen/packages-version.sh new file mode 100644 index 00000000000..1aba0776a47 --- /dev/null +++ b/evergreen/packages-version.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail + +if [ -z "$PACKAGE_TARGET" ]; then + # Use production release tag if nothing was passed + PACKAGE_TARGET="release" +fi + +if [ "${PACKAGE_TARGET}" = "dev" ]; then + PACKAGE_VERSION_MATCH="v[0-9]*.[0-9]*.[0-9]*-dev[0-9]*" + PACKAGE_VERSION_EXCLUDE="" +elif [ "${PACKAGE_TARGET}" = "release" ]; then + PACKAGE_VERSION_MATCH="v[0-9]*.[0-9]*.[0-9]*" + PACKAGE_VERSION_EXCLUDE="*-dev[0-9]*" +else + echo "Unexpected value of PACKAGE_TARGET: ${PACKAGE_TARGET}" +fi + +PACKAGE_VERSION=$(git describe --tags --abbrev=0 --match="${PACKAGE_VERSION_MATCH}" --exclude="${PACKAGE_VERSION_EXCLUDE}") +PACKAGE_VERSION=$(echo $PACKAGE_VERSION | cut -c 2-) +echo "$PACKAGE_VERSION" diff --git a/src/MongoDB.Bson/MongoDB.Bson.csproj b/src/MongoDB.Bson/MongoDB.Bson.csproj index 62f3ff6d73a..6b88210e189 100644 --- a/src/MongoDB.Bson/MongoDB.Bson.csproj +++ b/src/MongoDB.Bson/MongoDB.Bson.csproj @@ -7,9 +7,11 @@ MongoDB.Bson MongoDB.Bson + MongoDB.Bson Official MongoDB supported BSON library. See https://www.mongodb.com/docs/drivers/csharp/ for more details. MongoDB's Official Bson Library. $(PackageTags);bson + true diff --git a/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj b/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj index 028209b155d..19bddd29c05 100644 --- a/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj +++ b/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj @@ -6,8 +6,10 @@ MongoDB.Driver.Core MongoDB.Driver.Core + MongoDB.Driver.Core Official MongoDB supported Driver Core library. See https://www.mongodb.com/docs/drivers/csharp/ for more details. Core Component of the Official MongoDB .NET Driver. + true diff --git a/src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj b/src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj index 3ad6a159c19..c6676e8c4f7 100644 --- a/src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj +++ b/src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj @@ -7,9 +7,11 @@ MongoDB.Driver.GridFS MongoDB.Driver.GridFS + MongoDB.Driver.GridFS Official MongoDB supported driver for MongoDB GridFS implementation. See https://www.mongodb.com/docs/drivers/csharp/ for more details. GridFS Component of the Official MongoDB .NET Driver. $(PackageTags);gridfs + true diff --git a/src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj b/src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj index 7d38736a63b..a45d21890ab 100644 --- a/src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj +++ b/src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj @@ -11,6 +11,7 @@ mongocsharpdriver This package contains the legacy driver. The new driver's package name is MongoDB.Driver + true diff --git a/src/MongoDB.Driver/MongoDB.Driver.csproj b/src/MongoDB.Driver/MongoDB.Driver.csproj index 65d07aadc53..ca7da983b89 100644 --- a/src/MongoDB.Driver/MongoDB.Driver.csproj +++ b/src/MongoDB.Driver/MongoDB.Driver.csproj @@ -7,8 +7,10 @@ MongoDB.Driver MongoDB.Driver + MongoDB.Driver Official MongoDB supported driver for MongoDB. See https://www.mongodb.com/docs/drivers/csharp/ for more details. Official .NET driver for MongoDB. + true