Getting long-haul test cluster to a working state

Signed-off-by: ytimocin <[email protected]>

.github/scripts/cleanup-cluster.sh

@@ -6,7 +6,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#    
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
@@ -18,13 +18,18 @@
 
 set -e
 
-# Delete all test resources in queuemessages
+echo "cleaning up cluster"
+
+# Delete all test resources in queuemessages.
+echo "delete all resources in queuemessages.ucp.dev"
 kubectl delete queuemessages.ucp.dev -n radius-system --all
 
+# Testing deletion of deployment.apps.
+
 # Delete all test resources in resources without proxy resource.
+echo "delete all resources in resources.ucp.dev"
 resources=$(kubectl get resources.ucp.dev -n radius-system --no-headers -o custom-columns=":metadata.name")
-for r in $resources
-do
+for r in $resources; do
     if [[ $r == scope.local.* || $r == scope.aws.* || -z "$r" ]]; then
         echo "skip deletion: $r"
     else
@@ -34,9 +39,11 @@ do
 done
 
 # Delete all test namespaces.
-namespaces=$(kubectl get namespace | grep -E '^corerp.*|^default-.*|^radiusfunctionaltestbucket.*|^radius-test.*|^kubernetes-cli.*|^dpsb-.*|^azstorage-workload.*|^dapr-serviceinvocation|^ms.+' | awk '{print $1}')
-for ns in $namespaces
-do
+echo "delete all test namespaces"
+namespaces=$(kubectl get namespace |
+    grep -E '^kubernetes-interop-tutorial.*|^corerp.*|^default-.*|^radiusfunctionaltestbucket.*|^radius-test.*|^kubernetes-cli.*|^dpsb-.*|^dsrp-.*|^azstorage-workload.*|^dapr-serviceinvocation|^daprrp-rs-.*|^mynamespace.*|^ms.+' |
+    awk '{print $1}')
+for ns in $namespaces; do
     if [ -z "$ns" ]; then
         break
     fi

.github/workflows/long-running-azure.yaml

@@ -45,11 +45,11 @@ on:
     branches:
       - main
     paths:
-      - ".github/workflows/long-running-azure.yaml"
+      - '.github/workflows/long-running-azure.yaml'
 
 env:
   # Go version
-  GOVER: "^1.21"
+  GOVER: '^1.21'
   GOPROXY: https://proxy.golang.org
 
   # gotestsum version - see: https://github.com/gotestyourself/gotestsum
@@ -68,21 +68,23 @@ env:
   # The Radius helm chart location.
   RADIUS_CHART_LOCATION: deploy/Chart/
   # The region for AWS resources
-  AWS_REGION: "us-west-2"
+  AWS_REGION: us-west-2
   # The AWS account ID
-  AWS_ACCOUNT_ID: "${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}"
+  AWS_ACCOUNT_ID: '${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}'
 
   # The valid radius build time window in seconds to rebuild radius. 24 hours = 24 * 60 * 60 = 86400
   VALID_RADIUS_BUILD_WINDOW: 86400
 
   # The AKS cluster name
-  AKS_CLUSTER_NAME: "radiuse2e00-aks"
+  AKS_CLUSTER_NAME: 'radiuse2e00-aks'
   # The resource group for AKS_CLUSTER_NAME resource.
-  AKS_RESOURCE_GROUP:
-    "radiuse2e00"
+  AKS_RESOURCE_GROUP: 'radiuse2e00'
 
-    # Server where terraform test modules are deployed
-  TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local"
+  # Server where terraform test modules are deployed
+  TF_RECIPE_MODULE_SERVER_URL: 'http://tf-module-server.radius-test-tf-module-server.svc.cluster.local'
+
+  # Radius test environment name
+  RADIUS_TEST_ENVIRONMENT_NAME: 'kind-radius'
 
 jobs:
   build:
@@ -277,7 +279,7 @@ jobs:
   tests:
     name: Run functional tests
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     if: github.repository == 'radius-project/radius'
     env:
       SKIP_BUILD: ${{ needs.build.outputs.SKIP_BUILD }}
@@ -295,6 +297,13 @@ jobs:
         with:
           repository: ${{ env.CHECKOUT_REPO }}
           ref: ${{ env.CHECKOUT_REF }}
+      - name: Checkout samples repo
+        uses: actions/checkout@v3
+        with:
+          repository: radius-project/samples
+          ref: refs/heads/edge
+          token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
+          path: samples
       - name: Set up Go ${{ env.GOVER }}
         uses: actions/setup-go@v4
         with:
@@ -371,25 +380,27 @@ jobs:
           which rad || { echo "cannot find rad"; exit 1; }
 
           echo "*** Create workspace, group and environment for test ***"
-          rad workspace create kubernetes
-          rad workspace list
-          rad group create radius-e2e
-          rad group switch radius-e2e
+          rad workspace create kubernetes --force
+          rad group create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
+          rad group switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
 
           # The functional test is designed to use default namespace. So you must create the environment for default namespace.
-          rad env create radius-e2e --namespace default
-          rad env switch radius-e2e
-          rad workspace list
+          rad env create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --namespace default
+          rad env switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
+
+          # Temporary workaround to fix the x509 certificate error in the controller.
+          # https://github.com/radius-project/radius/issues/6989
+          kubectl delete secrets controller-cert -n radius-system
 
           echo "*** Configuring Azure provider ***"
-          rad env update radius-e2e --azure-subscription-id ${{ secrets.INTEGRATION_TEST_SUBSCRIPTION_ID }} \
+          rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --azure-subscription-id ${{ secrets.INTEGRATION_TEST_SUBSCRIPTION_ID }} \
             --azure-resource-group ${{ env.AZURE_TEST_RESOURCE_GROUP }}
           rad credential register azure --client-id ${{ secrets.INTEGRATION_TEST_SP_APP_ID }} \
             --client-secret ${{ secrets.INTEGRATION_TEST_SP_PASSWORD }} \
             --tenant-id ${{ secrets.INTEGRATION_TEST_TENANT_ID }}
 
           echo "*** Configuring AWS provider ***"
-          rad env update radius-e2e --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
+          rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
           rad credential register aws \
             --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
       - name: Log radius installation status (failure)
@@ -432,6 +443,24 @@ jobs:
           # Test_MongoDB_Recipe_Parameters is using the following environment variable.
           INTEGRATION_TEST_RESOURCE_GROUP_NAME: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
           FUNC_TEST_OIDC_ISSUER: ${{ env.FUNCTEST_OIDC_ISSUER }}
+          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
+          BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
+          BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
+      - name: Collect Pod details
+        if: always()
+        run: |
+          POD_STATE_LOG_FILENAME='${{ env.RADIUS_CONTAINER_LOG_BASE }}/all-tests-pod-states.log'
+          mkdir -p $(dirname $POD_STATE_LOG_FILENAME)
+          echo "kubectl get pods -A" >> $POD_STATE_LOG_FILENAME
+          kubectl get pods -A >> $POD_STATE_LOG_FILENAME
+          echo "kubectl describe pods -A" >> $POD_STATE_LOG_FILENAME
+          kubectl describe pods -A >> $POD_STATE_LOG_FILENAME
+      - name: Upload container logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: all_container_logs
+          path: ./${{ env.RADIUS_CONTAINER_LOG_BASE }}
       - name: Log radius e2e test status (success)
         if: success()
         run: |

build/test.mk

@@ -58,7 +58,10 @@ test-get-envtools:
 test-validate-cli: ## Run cli integration tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) -coverpkg= ./pkg/cli/cmd/... ./cmd/rad/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
-test-functional-all: test-functional-ucp test-functional-kubernetes test-functional-shared test-functional-msgrp test-functional-daprrp ## Runs all functional tests
+test-functional-all: test-functional-ucp test-functional-kubernetes test-functional-shared test-functional-msgrp test-functional-daprrp test-functional-datastoresrp test-functional-samples ## Runs all functional tests
+
+test-functional-ucp: ## Runs UCP functional tests
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/ucp/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
 test-functional-kubernetes: ## Runs Kubernetes functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/kubernetes/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
@@ -78,9 +81,6 @@ test-functional-datastoresrp: ## Runs Datastores RP functional tests
 test-functional-samples: ## Runs Samples functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/samples/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
-test-functional-ucp: ## Runs UCP functional tests
-	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/ucp/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
-
 test-validate-bicep: ## Validates that all .bicep files compile cleanly
 	BICEP_PATH="${HOME}/.rad/bin/rad-bicep" ./build/validate-bicep.sh
 

pkg/cli/cmd/recipe/list/list.go

@@ -136,7 +136,10 @@ func (r *Runner) Run(ctx context.Context) error {
 					ResourceType: resourceType,
 					TemplatePath: *c.TemplatePath,
 					TemplateKind: *c.TemplateKind,
-					PlainHTTP:    *c.PlainHTTP,
+				}
+
+				if c.PlainHTTP != nil {
+					recipe.PlainHTTP = *c.PlainHTTP
 				}
 			}
 			envRecipes = append(envRecipes, recipe)

test/functional/datastoresrp/resources/testdata/datastoresrp-resources-sqldb-recipe.bicep

@@ -1,4 +1,5 @@
 import radius as radius
+
 @description('Specifies the location for resources.')
 param location string = 'global'
 

test/functional/kubernetes/kubernetes_test.go

@@ -48,7 +48,6 @@ import (
 )
 
 func Test_TutorialApplication_KubernetesManifests(t *testing.T) {
-	t.Skip("Skipping for now, will reenable before merging.")
 	ctx := testcontext.New(t)
 	opts := shared.NewRPTestOptions(t)
 
@@ -79,11 +78,11 @@ func Test_TutorialApplication_KubernetesManifests(t *testing.T) {
 
 	t.Run("Deploy", func(t *testing.T) {
 		t.Log("Creating recipe")
-		err = opts.Client.Create(ctx, deployment)
+		err = opts.Client.Create(ctx, recipe)
 		require.NoError(t, err)
 
 		t.Log("Creating deployment")
-		err = opts.Client.Create(ctx, recipe)
+		err = opts.Client.Create(ctx, deployment)
 		require.NoError(t, err)
 	})
 

test/functional/shared/resources/testdata/recipes/test-bicep-recipes/README.md

@@ -8,4 +8,4 @@ This is important because it allows us to make changes to the recipes, and test
 
 ## Non-recipes bicep files
 
-Any Bicep file starting with `_` will be skipped during publishing. Use this as a convention to create shared modules that are not published as recipes. For example `_redis_kubernetes.bicep` would not be published.
+Any Bicep file starting with `_` will be skipped during publishing. Use this as a convention to create shared modules that are not published as recipes. For example `_redis_kubernetes.bicep` would not be published.

test/functional/shared/resources/testdata/recipes/test-bicep-recipes/sqldb-recipe.bicep

@@ -52,12 +52,12 @@ resource sql 'apps/Deployment@v1' = {
             ]
             resources: {
               requests: {
-                cpu: '600m'
-                memory: '1024Mi'
+                cpu: '1200m'
+                memory: '2048Mi'
               }
               limits: {
-                cpu: '900m'
-                memory: '1024Mi'
+                cpu: '1500m'
+                memory: '4096Mi'
               }
             }
             ports: [

test/functional/testUtil.go

@@ -73,7 +73,7 @@ func SetDefault() (string, string) {
 	defaultDockerReg := os.Getenv("DOCKER_REGISTRY")
 	imageTag := os.Getenv("REL_VERSION")
 	if defaultDockerReg == "" {
-		defaultDockerReg = "ghcr.io/radius-project/dev"
+		defaultDockerReg = "ghcr.io/radius-project"
 	}
 	if imageTag == "" {
 		imageTag = "latest"
@@ -93,6 +93,7 @@ func GetBicepRecipeRegistry() string {
 	if defaultRecipeRegistry == "" {
 		defaultRecipeRegistry = "ghcr.io/radius-project"
 	}
+
 	return "registry=" + defaultRecipeRegistry
 }
 
@@ -103,6 +104,7 @@ func GetBicepRecipeVersion() string {
 	if defaultVersion == "" {
 		defaultVersion = "latest"
 	}
+
 	return "version=" + defaultVersion
 }
 

test/infra/README.md

@@ -2,4 +2,4 @@
 
 This directory includes the infrastructure templates to deploy test environments for Radius.
 
-* [azure](./azure/): includes the bicep template for Radius test infrastructure on Azure.
+- [azure](./azure/): includes the bicep template for Radius test infrastructure on Azure.