Add publishing of recipes for pull requests (#5511)

# Description

This change adds a publish step to our builds that will upload the
recipes we use in tests. Right now these recipes are published manually,
and so changes we make to recipes cannot be tested in PRs.

This step adds publishing of recipes to our existing `radiusdev` ACR.
The next change will start to consume these recipes from their new paths
in our functional tests.

.github/scripts/publish-recipes.sh

@@ -0,0 +1,57 @@
+#! /bin/bash
+
+# Fail immedietly if any command fails
+set -e
+
+# Get command line arguments
+BICEP_PATH=$1
+DIRECTORY=$2
+REGISTRY_PATH=$3
+RECIPE_VERSION=$4
+
+BICEP_EXECUTABLE="$BICEP_PATH/rad-bicep"
+
+# Print usage information
+function print_usage() {
+    echo "Usage: $0 <BICEP_PATH> <DIRECTORY> <REGISTRY_PATH> <RECIPE_VERSION>"
+    echo ""
+    echo "  Publishes all recipes in the repository to the Azure Container Registry. Requires you to be logged into Azure via az login."
+    echo ""
+    echo "  BICEP_PATH: Path to directory containing the bicep executable. For example, ~/.rad/bin"
+    echo "  DIRECTORY: Directory containing the recipes to publish. For example, ./test/functional/testdata/recipes"
+    echo "  REGISTRY_PATH: Registry hostname and path prefix. For example, myregistry.azurecr.io/tests/recipes."
+    echo "  RECIPE_VERSION: Version of the recipe to publish. For example, pr-19293"
+    echo ""
+}
+
+# Verify that the required arguments are present
+if [[ $# -ne 4 ]]; then
+    echo "Error: Missing required arguments"
+    echo ""
+    print_usage
+    exit 1
+fi
+
+# We create output that's intended to be consumed by the GitHub Action summary. If we're
+# not running in a GitHub Action, we'll just silence the output.
+if [[ -z "$GITHUB_STEP_SUMMARY" ]]; then
+    GITHUB_STEP_SUMMARY=/dev/null
+fi
+
+echo "## Recipes published to $REGISTRY_PATH" >> $GITHUB_STEP_SUMMARY
+for RECIPE in $(find "$DIRECTORY" -type f -name "*.bicep")
+do
+    FILENAME=$(basename $RECIPE)
+    PUBLISH_REF="$REGISTRY_PATH/${FILENAME%.*}:$RECIPE_VERSION"
+
+    # Skip files that start with _. These are not recipes, they are modules that are
+    # used by the recipes.
+    if [[ $RECIPE = _* ]]; then
+        echo "Skipping $RECIPE"
+        continue
+    fi
+
+    echo "Publishing $RECIPE to $PUBLISH_REF"
+    echo "- $PUBLISH_REF" >> $GITHUB_STEP_SUMMARY
+    $BICEP_EXECUTABLE publish $RECIPE --target "br:$PUBLISH_REF"
+done

.github/workflows/functional-test.yaml

@@ -132,6 +132,7 @@ jobs:
             * Dapr: ${{ env.DAPR_VER }}
             * Azure KeyVault CSI driver: ${{ env.AZURE_KEYVAULT_CSI_DRIVER_VER }}
             * Azure Workload identity webhook: ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }}
+            * recipe location `${{ env.CACHE_REGISTRY}}/test/functional/corerp/recipes/<name>:${{ env.REL_VERSION }}`
             * appcore-rp test image location: `${{ env.CACHE_REGISTRY }}/appcore-rp:${{ env.REL_VERSION }}`
             * ucp test image location: `${{ env.CACHE_REGISTRY }}/ucpd:${{ env.REL_VERSION }}`
 
@@ -186,6 +187,36 @@ jobs:
           append: true
           message: |
             :x: Test image build failed
+      - name: Download rad-bicep
+        run: |
+          mkdir -p dist
+          ./.github/scripts/curl-with-retries.sh https://get.radapp.dev/tools/bicep-extensibility/edge/linux-x64/rad-bicep --output dist/rad-bicep
+          chmod +x dist/rad-bicep
+      - name: Publish Bicep Test Recipes
+        run: |
+          make publish-test-recipes
+        env:
+          RECIPE_REGISTRY: ${{ env.CACHE_REGISTRY }}
+          RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
+          RAD_BICEP_PATH: dist
+      - uses: marocchino/sticky-pull-request-comment@v2
+        if: success() && env.PR_NUMBER != ''
+        continue-on-error: true
+        with:
+          header: teststatus-${{ github.run_id }}
+          number: ${{ env.PR_NUMBER }}
+          append: true
+          message: |
+            :white_check_mark: Test recipe publishing succeeded
+      - uses: marocchino/sticky-pull-request-comment@v2
+        if: failure() && env.PR_NUMBER != ''
+        continue-on-error: true
+        with:
+          header: teststatus-${{ github.run_id }}
+          number: ${{ env.PR_NUMBER }}
+          append: true
+          message: |
+            :x: Test recipe publishing failed
   tests:
     name: Run ${{ matrix.name }} functional tests
     needs: build

Makefile

@@ -6,4 +6,4 @@
 ARROW := \033[34;1m=>\033[0m
 
 # order matters for these
-include build/help.mk build/version.mk build/build.mk build/util.mk build/generate.mk build/test.mk build/controller.mk build/docker.mk build/install.mk build/debug.mk
+include build/help.mk build/version.mk build/build.mk build/util.mk build/generate.mk build/test.mk build/controller.mk build/docker.mk build/recipes.mk build/install.mk build/debug.mk

build/recipes.mk

@@ -0,0 +1,20 @@
+# ------------------------------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------------------------------
+
+RECIPE_TAG_VERSION?=latest
+RAD_BICEP_PATH?=$${HOME}/.rad/bin
+
+##@ Recipes
+
+.PHONY: publish-test-recipes
+publish-test-recipes: ## Publishes test recipes to <RECIPE_REGISTRY> with version <RECIPE_TAG_VERSION>
+	@if [ -z "$(RECIPE_REGISTRY)" ]; then echo "Error: RECIPE_REGISTRY must be set to a valid OCI registry"; exit 1; fi
+
+	@echo "$(ARROW) Publishing recipes from ./test/functional/corerp/resources/testdata/recipes/test-recipes..."
+	./.github/scripts/publish-recipes.sh \
+		${RAD_BICEP_PATH} \
+		./test/functional/corerp/resources/testdata/recipes/test-recipes \
+		${RECIPE_REGISTRY}/test/functional/corerp/recipes \
+		${RECIPE_TAG_VERSION}

docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md

@@ -1,43 +1,54 @@
 # Running Radius functional tests
 
-You can find the functional tests under `./test/functional`. A functional tests (in our terminology) is a test that interacts with real hosting enviroments (Azure, Kubernetes), deploys real applications and resources, and covers realistic or simulated user scenarios.
+You can find the functional tests under `./test/functional`. A functional test (in our terminology) is a test that interacts with real hosting enviroments (Kubernetes), deploys real applications and resources, and covers realistic or simulated user scenarios.
 
 These tests verify whether:
 
-- That Radius environments can be created successfully.
+- That Radius environment can be created successfully.
 - That Bicep templates of sample applications ca be deployed to the Radius environment. 
 
-These run on Azure Radius environments (also called Radius test clusters) that are managed dynamically as part of the test process.
-
-Note that these tests require the Radius environment to be associated with "default" kubernetes namespace. 
-Since an environment "env-name"'s default namespace is "env-name", we should explicitly supply  --namespace "default" flag during the test Radius environment creation.
 
 ## Running via GitHub workflow
 
-These tests automatically run for every PR in the `azure-pipelines.yml` github workflow.
+These tests automatically run for every PR in the `functional-tests.yml` github workflow.
 
 We do not run these tests for commits to `main` or tags since they might block the build if they fail.
 
 ### How this works 
 
 For each PR we run the following set of steps:
 
-- Create a new test environment.
-- Run deployment tests.
-- Delete the environment.
+- Build Radius and publish test assets
+- For each group of tests:
+  - Create a Kubernetes cluster and install the build
+  - Run tests
+  - Delete any cloud resources that were created
+
+We have a separate scheduled job (`purge-test-resources.yaml`) that will delete cloud resources that are left behind. This can happen when the test run is cancelled or times out.
 
 ## Configuration
 
-These tests use your local Kubernetes credentials, and Radius environment for testing. In a GitHub workflow, our automation makes the CI environment resemble a dev scenario.
+These tests use your local Kubernetes credentials, and Radius environment for testing. In a GitHub workflow, our automation makes the CI environment resemble a real user scenario. This way we test a setup that is close to what users will have in the real world.
 
-The tests use our product functionality (the Radius config file) to configure the environment.
+As much as possible, the tests use product functionality such as the Radius CLI configuration and local KubeConfig to detect settings.
 
 ## Running the tests locally
 
-1. Create an environment
-2. Place `rad` on your path
-3. Make sure `rad-bicep` is downloaded (`rad bicep download`)
-4. Run:
+### Prerequisites
+
+1. Place `rad` on your path
+2. Make sure `rad-bicep` is downloaded (`rad bicep download`)
+3. Create a container registry (must be Azure Container Registry for now)
+4. Log-in to the container registry `az acr login -n <registry-name>`
+5. Publish test recipes by running `RECIPE_REGISTRY=<registry-name>.azurecr.io make publish-test-recipes`
+6. Create a Radius environment with `rad init` and specify the default namespace
+
+> ⚠️ The tests assume the Kubernetes namespace in use is `default`. If your environment is set up differently you will see
+> test failures.
+
+### Run 
+
+1. Run:
 
     ```sh
         .{workspace}/radius/test/executeCoreRPFunctionalTest.sh
@@ -48,7 +59,13 @@ When you're running locally with this configuration, the tests will use your loc
     make test-functional-corerp
  ```
 
-You can also run/debug individual tests from VSCode
+You can also run/debug individual tests from VSCode.
+
+### Tips
+
+> 💡 If you make changes to recipes, make sure to re-run the *publish test recipe* step from prerequisites.
+
+> 💡 If you make changes to the `rad` CLI make sure to copy it to your path. 
 
 ### Seeing log output
 

test/functional/corerp/resources/testdata/recipes/test-recipes/README.md

@@ -0,0 +1,11 @@
+# Test Recipes
+
+The recipes in this folder are published as part of the PR process to:
+
+> `radiusdev.azurecr.io/test/functional/corerp/<filename>:pr-<pr-number>`
+
+This is important because it allows us to make changes to the recipes, and test them in the same PR that contains the change.
+
+## Non-recipes bicep files
+
+Any Bicep file starting with `_` will be skipped during publishing. Use this as a convention to create shared modules that are not published as recipes. For example `_redis_kubernetes.bicep` would not be published.