Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add application profile annotation support #313

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

add application profile annotation support #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions pkg/ccm/loadbalancer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ package ccm
import (
"context"
"fmt"
"strconv"
"strings"

"github.com/vmware/cloud-provider-for-cloud-director/pkg/cpisdk"
"github.com/vmware/cloud-provider-for-cloud-director/pkg/util"
"github.com/vmware/cloud-provider-for-cloud-director/pkg/vcdsdk"
Expand All @@ -22,13 +25,12 @@ import (
"k8s.io/client-go/kubernetes"
cloudProvider "k8s.io/cloud-provider"
"k8s.io/klog"
"strconv"
"strings"
)

const (
sslPortsAnnotation = `service.beta.kubernetes.io/vcloud-avi-ssl-ports`
sslCertAliasAnnotation = `service.beta.kubernetes.io/vcloud-avi-ssl-cert-alias`
applicationProfileAnnotation = `service.beta.kubernetes.io/vcloud-avi-application-profile`
skipAviSSLTerminationAnnotation = `service.beta.kubernetes.io/vcloud-avi-ssl-no-termination`
// TODO: Update controlPlaneLabel to use default K8s constants if available
controlPlaneLabel = `node-role.kubernetes.io/control-plane`
Expand Down Expand Up @@ -228,8 +230,9 @@ func (lb *LBManager) UpdateLoadBalancer(ctx context.Context, clusterName string,
klog.Infof("Updating pool [%s] with port [%s:%d]", lbPoolName, portName, internalPort)
protocol, _ := nameToProtocol[portName]
resourcesAllocated := &util.AllocatedResourcesMap{}
applicationProfile := getApplicationProfile(service)
vip, err := gm.UpdateLoadBalancer(ctx, lbPoolName, virtualServiceName, nodeIps, userSpecifiedLBIP, internalPort,
externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated)
externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated, applicationProfile)
// TODO: Should we record this error as well?
if rdeErr := lb.addLBResourcesToRDE(ctx, resourcesAllocated, vip); rdeErr != nil {
return fmt.Errorf("failed to add load balancer resources to RDE [%s]: [%v]", lb.clusterID, err)
Expand Down Expand Up @@ -504,6 +507,15 @@ func getSSLCertAlias(service *v1.Service) string {
return sslCertAlias
}

func getApplicationProfile(service *v1.Service) string {
applicationProfile, ok := service.Annotations[applicationProfileAnnotation]
if !ok {
return ""
}

return applicationProfile
}

func shouldSkipAviSSLTermination(service *v1.Service) bool {
shouldSkipAviSSLTerminationStr, ok := service.Annotations[skipAviSSLTerminationAnnotation]
if !ok {
Expand Down Expand Up @@ -559,6 +571,8 @@ func (lb *LBManager) createLoadBalancer(ctx context.Context, service *v1.Service
userSpecifiedLBIP := getUserSpecifiedLoadBalancerIP(service)
klog.Infof("createLoadBalancer called with loadBalancerIP [%s] for service [%s]", userSpecifiedLBIP, service.Name)

applicationProfile := getApplicationProfile(service)

if lbExists {
// Update load balancer if there are changes in service properties
typeToInternalPortMap, typeToExternalPortMap, nameToProtocol := lb.getServicePortMap(service)
Expand All @@ -570,7 +584,7 @@ func (lb *LBManager) createLoadBalancer(ctx context.Context, service *v1.Service
klog.Infof("Updating pool [%s] with port [%s:%d:%d]", lbPoolName, portName, internalPort, externalPort)
resourcesAllocated := &util.AllocatedResourcesMap{}
vip, err := gm.UpdateLoadBalancer(ctx, lbPoolName, virtualServiceName, nodeIPs, userSpecifiedLBIP, internalPort,
externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated)
externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated, applicationProfile)
if rdeErr := lb.addLBResourcesToRDE(ctx, resourcesAllocated, vip); rdeErr != nil {
return nil, fmt.Errorf("failed to update RDE [%s] with load balancer resources: [%v]", lb.clusterID, err)
}
Expand Down Expand Up @@ -676,7 +690,7 @@ func (lb *LBManager) createLoadBalancer(ctx context.Context, service *v1.Service
// Create using VCD API
resourcesAllocated := &util.AllocatedResourcesMap{}
lbIP, err := gm.CreateLoadBalancer(ctx, virtualServiceNamePrefix, lbPoolNamePrefix, nodeIPs, portDetailsList,
lb.OneArm, lb.EnableVirtualServiceSharedIP, portNameToIPMap, userSpecifiedLBIP, resourcesAllocated)
lb.OneArm, lb.EnableVirtualServiceSharedIP, portNameToIPMap, userSpecifiedLBIP, resourcesAllocated, applicationProfile)
if rdeErr := lb.addLBResourcesToRDE(ctx, resourcesAllocated, lbIP); rdeErr != nil {
return nil, fmt.Errorf("unable to add load balancer pool resources to RDE [%s]: [%v]", lb.clusterID, err)
}
Expand Down
34 changes: 21 additions & 13 deletions pkg/vcdsdk/gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,18 @@ package vcdsdk
import (
"context"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"

"github.com/antihax/optional"
"github.com/peterhellberg/link"
"github.com/vmware/cloud-provider-for-cloud-director/pkg/util"
swaggerClient "github.com/vmware/cloud-provider-for-cloud-director/pkg/vcdswaggerclient_36_0"
"github.com/vmware/go-vcloud-director/v2/govcd"
"github.com/vmware/go-vcloud-director/v2/types/v56"
"k8s.io/klog"
"net/http"
"net/url"
"strconv"
"strings"
)

type OneArm struct {
Expand Down Expand Up @@ -1104,7 +1105,7 @@ func (gatewayManager *GatewayManager) checkIfGatewayIsReady(ctx context.Context)
}

func (gatewayManager *GatewayManager) UpdateVirtualService(ctx context.Context, virtualServiceName string,
virtualServiceIP string, externalPort int32, oneArmEnabled bool) (*swaggerClient.EntityReference, error) {
virtualServiceIP string, externalPort int32, oneArmEnabled bool, applicationProfile string) (*swaggerClient.EntityReference, error) {
client := gatewayManager.Client
vsSummary, err := gatewayManager.GetVirtualService(ctx, virtualServiceName)
if err != nil {
Expand Down Expand Up @@ -1147,6 +1148,11 @@ func (gatewayManager *GatewayManager) UpdateVirtualService(ctx context.Context,
// update the virtual IP address of the virtual service when one arm is nil
vs.VirtualIpAddress = virtualServiceIP
}

if applicationProfile != "" && vs.ApplicationProfile.Name != applicationProfile {
vs.ApplicationProfile.Name = applicationProfile
}

resp, err := client.APIClient.EdgeGatewayLoadBalancerVirtualServiceApi.UpdateVirtualService(ctx, vs, vsSummary.Id, org.Org.ID)
if resp != nil && resp.StatusCode != http.StatusAccepted {
var responseMessageBytes []byte
Expand Down Expand Up @@ -1187,7 +1193,7 @@ func (gatewayManager *GatewayManager) UpdateVirtualService(ctx context.Context,
func (gatewayManager *GatewayManager) CreateVirtualService(ctx context.Context, virtualServiceName string,
lbPoolRef *swaggerClient.EntityReference, segRef *swaggerClient.EntityReference,
freeIP string, vsType string, externalPort int32,
useSSL bool, certificateAlias string) (*swaggerClient.EntityReference, error) {
useSSL bool, certificateAlias, applicationProfile string) (*swaggerClient.EntityReference, error) {

client := gatewayManager.Client
if gatewayManager.GatewayRef == nil {
Expand Down Expand Up @@ -1231,9 +1237,7 @@ func (gatewayManager *GatewayManager) CreateVirtualService(ctx context.Context,
SslEnabled: useSSL,
},
},
ApplicationProfile: &swaggerClient.EdgeLoadBalancerApplicationProfile{
SystemDefined: true,
},
ApplicationProfile: &swaggerClient.EdgeLoadBalancerApplicationProfile{},
}
switch vsType {
case "TCP":
Expand All @@ -1259,6 +1263,10 @@ func (gatewayManager *GatewayManager) CreateVirtualService(ctx context.Context,
return nil, fmt.Errorf("unhandled virtual service type [%s]", vsType)
}

if applicationProfile != "" {
virtualServiceConfig.ApplicationProfile.Name = applicationProfile
}

clusterOrg, err := client.VCDClient.GetOrgByName(client.ClusterOrgName)
if err != nil {
return nil, fmt.Errorf("unable to get org for org [%s]: [%v]", client.ClusterOrgName, err)
Expand Down Expand Up @@ -1502,7 +1510,7 @@ func (gatewayManager *GatewayManager) GetLoadBalancerPoolMemberIPs(ctx context.C

func (gm *GatewayManager) CreateLoadBalancer(ctx context.Context, virtualServiceNamePrefix string, lbPoolNamePrefix string,
ips []string, portDetailsList []PortDetails, oneArm *OneArm, enableVirtualServiceSharedIP bool,
portNameToIP map[string]string, providedIP string, resourcesAllocated *util.AllocatedResourcesMap) (string, error) {
portNameToIP map[string]string, providedIP string, resourcesAllocated *util.AllocatedResourcesMap, applicationProfile string) (string, error) {
if len(portDetailsList) == 0 {
// nothing to do here
klog.Infof("There is no port specified. Hence nothing to do.")
Expand Down Expand Up @@ -1707,7 +1715,7 @@ func (gm *GatewayManager) CreateLoadBalancer(ctx context.Context, virtualService

virtualServiceRef, err := gm.CreateVirtualService(ctx, virtualServiceName, lbPoolRef, segRef,
virtualServiceIP, portDetails.Protocol, portDetails.ExternalPort,
portDetails.UseSSL, portDetails.CertAlias)
portDetails.UseSSL, portDetails.CertAlias, applicationProfile)
if err != nil {
// return plain error if vcdsdk.VirtualServicePendingError is returned. Helps the caller recognize that the
// error is because VirtualService is still in Pending state.
Expand Down Expand Up @@ -1833,7 +1841,7 @@ func (gm *GatewayManager) DeleteLoadBalancer(ctx context.Context, virtualService

func (gm *GatewayManager) UpdateLoadBalancer(ctx context.Context, lbPoolName string, virtualServiceName string,
ips []string, externalIP string, internalPort int32, externalPort int32, oneArm *OneArm, enableVirtualServiceSharedIP bool, protocol string,
resourcesAllocated *util.AllocatedResourcesMap) (string, error) {
resourcesAllocated *util.AllocatedResourcesMap, applicationProfile string) (string, error) {

if gm == nil {
return "", fmt.Errorf("GatewayManager cannot be nil")
Expand All @@ -1852,7 +1860,7 @@ func (gm *GatewayManager) UpdateLoadBalancer(ctx context.Context, lbPoolName str
return "", fmt.Errorf("unable to update load balancer pool [%s]: [%v]", lbPoolName, err)
}
resourcesAllocated.Insert(VcdResourceLoadBalancerPool, lbPoolRef)
vsRef, err := gm.UpdateVirtualService(ctx, virtualServiceName, externalIP, externalPort, oneArm != nil)
vsRef, err := gm.UpdateVirtualService(ctx, virtualServiceName, externalIP, externalPort, oneArm != nil, applicationProfile)
if vsRef != nil {
resourcesAllocated.Insert(VcdResourceVirtualService, vsRef)
}
Expand Down
Loading