Lay groundwork for cluster MSI usage (#3757)

* Add a parameter for enabling Entra ID RBAC on key vaults
* Add an RP-level feature flag for determining whether to use the mock MSI RP
* Tweak the mock identity URL to play nicely with the mock MSI RP
* Add Azure SDK client wrappers for new clients (federated identity credentials control plane and key vault data plane)
* Vendor in new Azure SDK clients and update msi-dataplane

* Lay groundwork for use of cluster MSI...
- Initialize the MSI dataplane client, using the mock MSI RP/stub if
  appropriate
- Initialize key vault store client (for MSI certificates; functionality
  is implemented in MSI dataplane module)
- Create a cluster MSI certificate and store it in the key vault during
  cluster bootstrap
- Instantiate an Azure SDK FederatedIdentityCredential client using the
  cluster MSI certificate
- Delete the cluster MSI certificate as needed during cluster deletion

* Don't fail during cluster deletion if the cluster MSI certificate is
already gone from the key vault (or was potentially never created)

* Establish an RP-Config variable for the MSI RP endpoint

- Update doc comment for ensureClusterMsiCertificate
- Simplify conditional logic in MSI cert deletion

* Use pointer conversion functions that aren't deprecated

* Respond to PR comments (and fix some other things along the way)

- Move `clusterMsiResourceId` function to `OpenShiftCluster` type
- When persisting the MSI cert to KV, use the `NotAfter` returned by the MSI RP (for the stub, just use an arbitrary value)
- Move `getClientOptions` functionality to `AROEnvironment` type
- Move logic for determining cluster MSI key vault name to `pkg/env`
- Pull cloud name mapping stuff out to `AROEnvironment` type
- Update msi-dataplane module to include new changes and use `UserAssignedIdentities` type to get Azure credential in `pkg/cluster/clustermsi.go`
- Fix typo in https URL in comment in `pkg/cluster/delete.go`
- Implement suggestion to use `errors.As` instead of a type assertion in `pkg/cluster/delete.go`

* Update documentation with info about new feature flag

- Move new cluster MSI steps forward in bootstrap step order
- Move MSI dataplane client options stuff to pkg/env
- Explicitly check for a single cluster MSI in `ClusterMsiResourceId`
- Other small tweaks

* Vendor in msi-dataplane update that prevents a potential nil pointer dereference

* Add missing method to internal key vault client

* Make error messages more specific in ClusterMsiResourceId

* Add missing env vars to run-rp make target and uncomment dynamic validation bootstrap step

- In newly added Azure clients, return struct types instead of interface
  types
- Move cluster MSI certificate deletion to be after Azure resource
  deletion for safety just in case cx continues to use cluster that is
  in Failed/Deleting provisioning state

* Add new env vars for MIWI to env.example for clarity/completeness

* Turn check for nonzero number of user assigned identities into a utility function

* Use existing constant for key vault dns suffix

Makefile

@@ -531,6 +531,9 @@ run-rp: ci-rp podman-secrets
 		-e ARO_CHECKOUT_PATH="/app" \
 		-e ARO_INSTALL_VIA_HIVE="true" \
 		-e ARO_ADOPT_BY_HIVE="true" \
+		-e MOCK_MSI_TENANT_ID \
+		-e MOCK_MSI_CLIENT_ID \
+		-e MOCK_MSI_CERT \
 		--secret aks.kubeconfig,target=/app/secrets/aks.kubeconfig \
 		--secret proxy-client.key,target=/app/secrets/proxy-client.key \
 		--secret proxy-client.crt,target=/app/secrets/proxy-client.crt \

cmd/aro/rp.go

@@ -62,6 +62,7 @@ func rp(ctx context.Context, log, audit *logrus.Entry) error {
 			"CLUSTER_MDM_NAMESPACE",
 			"MDM_ACCOUNT",
 			"MDM_NAMESPACE",
+			"MSI_RP_ENDPOINT",
 			env.OIDCStorageAccountName,
 		}
 

docs/feature-flags.md

@@ -47,4 +47,9 @@ feature flags defined in pkg/env/env.go.  At the time of writing these include:
 
 * RequireOIDCStorageWebEndpoint: Since Azure Front Door is only present for INT and PROD, there is a need to determine the web endpoint of the OIDC Storage Account after its creation.
 Format of web endpoint(It uses Azure DNS Zone endpoint):- **https://[storage-account].z[00-99].web.storage.azure.net** .
-Used in development only.
+Used in development only.
+
+* UseMockMsiRp: The MSI RP is only present in PROD, so this feature flag is used
+  in local development, full service development, and INT to tell the RP to use a
+  mocked version of the MSI dataplane for the cluster MSI. Only relevant to
+  clusters that have a cluster MSI.

env.example

@@ -9,4 +9,12 @@ export CLUSTER_NAME="${AZURE_PREFIX}-aro-cluster"
 export CLUSTER_VNET="${AZURE_PREFIX}-aro-vnet"
 export ARO_IMAGE=arointsvc.azurecr.io/aro:latest 
 
-. secrets/env
+# You'll need these to create MIWI clusters with your local RP, but you can comment them
+# out or remove them from your env file if you're only going to be creating service principal
+# clusters.
+export MOCK_MSI_CLIENT_ID="replace_with_value_output_by_hack/devtools/msi.sh"
+export MOCK_MSI_CERT="replace_with_value_output_by_hack/devtools/msi.sh"
+export MOCK_MSI_TENANT_ID="replace_with_value_output_by_hack/devtools/msi.sh"
+export PLATFORM_WORKLOAD_IDENTITY_ROLE_SETS="replace_with_value_output_by_hack/devtools/msi.sh"
+
+. secrets/env

go.mod

@@ -5,21 +5,23 @@ go 1.21
 require (
 	github.com/Azure/azure-sdk-for-go v63.1.0+incompatible
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmos/armcosmos/v2 v2.5.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0
+	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
 	github.com/Azure/go-autorest/autorest v0.11.29
 	github.com/Azure/go-autorest/autorest/adal v0.9.23
 	github.com/Azure/go-autorest/autorest/date v0.3.0
 	github.com/Azure/go-autorest/autorest/to v0.4.0
 	github.com/Azure/go-autorest/autorest/validation v0.3.1
 	github.com/Azure/go-autorest/tracing v0.6.0
-	github.com/Azure/msi-dataplane v0.0.3
+	github.com/Azure/msi-dataplane v0.0.6
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/codahale/etm v0.0.0-20141003032925-c00c9e6fb4c9
 	github.com/containers/image/v5 v5.30.1
@@ -74,6 +76,7 @@ require (
 	github.com/tebeka/selenium v0.9.9
 	github.com/ugorji/go/codec v1.2.12
 	github.com/vincent-petithory/dataurl v1.0.0
+	go.uber.org/mock v0.4.0
 	golang.org/x/crypto v0.26.0
 	golang.org/x/net v0.28.0
 	golang.org/x/oauth2 v0.18.0
@@ -96,6 +99,7 @@ require (
 require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect

go.sum

@@ -6,8 +6,8 @@ github.com/Azure/azure-sdk-for-go v63.1.0+incompatible h1:yNC7qlSUWVF8p0TzxdmWW1
 github.com/Azure/azure-sdk-for-go v63.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3 v3.0.0-beta.2 h1:qiir/pptnHqp6hV8QwV+IExYIf6cPsXBfUDUXQ27t2Y=
@@ -22,12 +22,18 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFG
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 h1:z4YeiSXxnUI+PqB46Yj6MZA3nwb1CcJIkEMDrzUd8Cs=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0/go.mod h1:rko9SzMxcMk0NJsNAxALEGaTYyy79bNRwxgJfrH0Spw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0 h1:h4Zxgmi9oyZL2l8jeg1iRTqPloHktywWcu0nlJmo1tA=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0/go.mod h1:LgLGXawqSreJz135Elog0ywTJDsm0Hz2k+N+6ZK35u8=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1 h1:9fXQS/0TtQmKXp8SureKouF+idbQvp7cPUxykiohnBs=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1/go.mod h1:f+OaoSg0VQYPMqB0Jp2D54j1VHzITYcJaCNwV+k00ts=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
@@ -50,8 +56,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/Azure/msi-dataplane v0.0.3 h1:xeKVAv5vI6vU2fSTjvfQmKC8TmQAoM2jnOGVZtN/7Js=
-github.com/Azure/msi-dataplane v0.0.3/go.mod h1:OhTIdOVMVLF7f4miqOTfVYyAVojDNp8aS4hiF34F5wo=
+github.com/Azure/msi-dataplane v0.0.6 h1:+IhGETRF9lLNlbs6793xWFKMcbborBtaR2ops1XWlPo=
+github.com/Azure/msi-dataplane v0.0.6/go.mod h1:/fXvAsxSogxoT7If0xfaeyaIQ7Q/0xAY9ISn7lOpA4o=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=

pkg/api/openshiftcluster.go

@@ -4,8 +4,11 @@ package api
 // Licensed under the Apache License 2.0.
 
 import (
+	"errors"
 	"sync"
 	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 )
 
 // OpenShiftCluster represents an OpenShift cluster
@@ -32,6 +35,31 @@ func (oc *OpenShiftCluster) UsesWorkloadIdentity() bool {
 	return oc.Properties.PlatformWorkloadIdentityProfile != nil && oc.Properties.ServicePrincipalProfile == nil
 }
 
+// ClusterMsiResourceId returns the resource ID of the cluster MSI or an error
+// if it encounters an issue while grabbing the resource ID from the cluster
+// doc. It is written under the assumption that there is only one cluster MSI
+// and will have to be refactored if we ever use more than one.
+func (oc *OpenShiftCluster) ClusterMsiResourceId() (*arm.ResourceID, error) {
+	if !oc.HasUserAssignedIdentities() {
+		return nil, errors.New("could not find cluster MSI in cluster doc")
+	} else if len(oc.Identity.UserAssignedIdentities) > 1 {
+		return nil, errors.New("unexpectedly found more than one cluster MSI in cluster doc")
+	}
+
+	var msiResourceId string
+	for resourceId := range oc.Identity.UserAssignedIdentities {
+		msiResourceId = resourceId
+	}
+
+	return arm.ParseResourceID(msiResourceId)
+}
+
+// HasUserAssignedIdentities returns true if and only if the cluster doc's
+// Identity.UserAssignedIdentities is non-nil and non-empty.
+func (oc *OpenShiftCluster) HasUserAssignedIdentities() bool {
+	return oc.Identity != nil && oc.Identity.UserAssignedIdentities != nil && len(oc.Identity.UserAssignedIdentities) > 0
+}
+
 // CreatedByType by defines user type, which executed the request
 // This field should match common-types field names for swagger and sdk generation
 type CreatedByType string

pkg/api/openshiftcluster_test.go

@@ -6,6 +6,10 @@ package api
 import (
 	"fmt"
 	"testing"
+
+	"go.uber.org/mock/gomock"
+
+	utilerror "github.com/Azure/ARO-RP/test/util/error"
 )
 
 func TestIsTerminal(t *testing.T) {
@@ -110,3 +114,139 @@ func TestIsWorkloadIdentity(t *testing.T) {
 		})
 	}
 }
+
+func TestClusterMsiResourceId(t *testing.T) {
+	mockGuid := "00000000-0000-0000-0000-000000000000"
+	clusterRGName := "aro-cluster"
+	miName := "aro-cluster-msi"
+	miResourceId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ManagedIdentity/userAssignedIdentities/%s", mockGuid, clusterRGName, miName)
+
+	tests := []struct {
+		name    string
+		oc      *OpenShiftCluster
+		wantErr string
+	}{
+		{
+			name:    "error - cluster doc has nil Identity",
+			oc:      &OpenShiftCluster{},
+			wantErr: "could not find cluster MSI in cluster doc",
+		},
+		{
+			name: "error - cluster doc has non-nil Identity but nil Identity.UserAssignedIdentities",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{},
+			},
+			wantErr: "could not find cluster MSI in cluster doc",
+		},
+		{
+			name: "error - cluster doc has non-nil Identity but empty Identity.UserAssignedIdentities",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{
+					UserAssignedIdentities: UserAssignedIdentities{},
+				},
+			},
+			wantErr: "could not find cluster MSI in cluster doc",
+		},
+		{
+			name: "error - cluster doc has non-nil Identity but two MSIs in Identity.UserAssignedIdentities",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{
+					UserAssignedIdentities: UserAssignedIdentities{
+						miResourceId:  ClusterUserAssignedIdentity{},
+						"secondEntry": ClusterUserAssignedIdentity{},
+					},
+				},
+			},
+			wantErr: "unexpectedly found more than one cluster MSI in cluster doc",
+		},
+		{
+			name: "error - invalid resource ID (theoretically not possible, but still)",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{
+					UserAssignedIdentities: UserAssignedIdentities{
+						"Hi hello I'm not a valid resource ID": ClusterUserAssignedIdentity{},
+					},
+				},
+			},
+			wantErr: "invalid resource ID: resource id 'Hi hello I'm not a valid resource ID' must start with '/'",
+		},
+		{
+			name: "success",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{
+					UserAssignedIdentities: UserAssignedIdentities{
+						miResourceId: ClusterUserAssignedIdentity{},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			_, err := tt.oc.ClusterMsiResourceId()
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
+		})
+	}
+}
+
+func TestHasUserAssignedIdentities(t *testing.T) {
+	mockGuid := "00000000-0000-0000-0000-000000000000"
+	clusterRGName := "aro-cluster"
+	miName := "aro-cluster-msi"
+	miResourceId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ManagedIdentity/userAssignedIdentities/%s", mockGuid, clusterRGName, miName)
+
+	tests := []struct {
+		name       string
+		oc         *OpenShiftCluster
+		wantResult bool
+	}{
+		{
+			name:       "false - cluster doc has nil Identity",
+			oc:         &OpenShiftCluster{},
+			wantResult: false,
+		},
+		{
+			name: "false - cluster doc has non-nil Identity but nil Identity.UserAssignedIdentities",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{},
+			},
+			wantResult: false,
+		},
+		{
+			name: "false - cluster doc has non-nil Identity but empty Identity.UserAssignedIdentities",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{
+					UserAssignedIdentities: UserAssignedIdentities{},
+				},
+			},
+			wantResult: false,
+		},
+		{
+			name: "true",
+			oc: &OpenShiftCluster{
+				Identity: &Identity{
+					UserAssignedIdentities: UserAssignedIdentities{
+						miResourceId: ClusterUserAssignedIdentity{},
+					},
+				},
+			},
+			wantResult: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := gomock.NewController(t)
+			defer controller.Finish()
+
+			got := tt.oc.HasUserAssignedIdentities()
+			if got != tt.wantResult {
+				t.Error(fmt.Errorf("got != want: %v != %v", got, tt.wantResult))
+			}
+		})
+	}
+}