Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARO-13380 - metrics: cwp status #4002

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

ARO-13380 - metrics: cwp status #4002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions pkg/monitor/cluster/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ import (
configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
machineclient "github.com/openshift/client-go/machine/clientset/versioned"
operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned"

"github.com/Azure/ARO-RP/pkg/api"
"github.com/Azure/ARO-RP/pkg/env"
"github.com/Azure/ARO-RP/pkg/hive"
"github.com/Azure/ARO-RP/pkg/metrics"
"github.com/Azure/ARO-RP/pkg/monitor/dimension"
Expand All @@ -43,13 +45,16 @@ type Monitor struct {
oc *api.OpenShiftCluster
dims map[string]string

restconfig *rest.Config
cli kubernetes.Interface
configcli configclient.Interface
maocli machineclient.Interface
mcocli mcoclient.Interface
m metrics.Emitter
arocli aroclient.Interface
restconfig *rest.Config
cli kubernetes.Interface
configcli configclient.Interface
operatorcli operatorclient.Interface
maocli machineclient.Interface
mcocli mcoclient.Interface
m metrics.Emitter
arocli aroclient.Interface
env env.Interface
tenantID string

ocpclientset client.Client
hiveclientset client.Client
Expand All @@ -68,7 +73,7 @@ type Monitor struct {
doc *api.OpenShiftClusterDocument
}

func NewMonitor(log *logrus.Entry, restConfig *rest.Config, oc *api.OpenShiftCluster, doc *api.OpenShiftClusterDocument, m metrics.Emitter, hiveRestConfig *rest.Config, hourlyRun bool, wg *sync.WaitGroup, hiveClusterManager hive.ClusterManager) (*Monitor, error) {
func NewMonitor(log *logrus.Entry, restConfig *rest.Config, oc *api.OpenShiftCluster, doc *api.OpenShiftClusterDocument, env env.Interface, tenantID string, m metrics.Emitter, hiveRestConfig *rest.Config, hourlyRun bool, wg *sync.WaitGroup, hiveClusterManager hive.ClusterManager) (*Monitor, error) {
r, err := azure.ParseResourceID(oc.ID)
if err != nil {
return nil, err
Expand Down Expand Up @@ -105,6 +110,10 @@ func NewMonitor(log *logrus.Entry, restConfig *rest.Config, oc *api.OpenShiftClu
if err != nil {
return nil, err
}
operatorcli, err := operatorclient.NewForConfig(restConfig)
if err != nil {
return nil, err
}

// lazy discovery will not attempt to reach out to the apiserver immediately
mapper, err := apiutil.NewDynamicRESTMapper(restConfig, apiutil.WithLazyDiscovery)
Expand Down Expand Up @@ -134,9 +143,12 @@ func NewMonitor(log *logrus.Entry, restConfig *rest.Config, oc *api.OpenShiftClu
restconfig: restConfig,
cli: cli,
configcli: configcli,
operatorcli: operatorcli,
maocli: maocli,
mcocli: mcocli,
arocli: arocli,
env: env,
tenantID: tenantID,
m: m,
ocpclientset: ocpclientset,
hiveclientset: hiveclientset,
Expand Down Expand Up @@ -221,6 +233,7 @@ func (mon *Monitor) Monitor(ctx context.Context) (errs []error) {
mon.emitCertificateExpirationStatuses,
mon.emitEtcdCertificateExpiry,
mon.emitPrometheusAlerts, // at the end for now because it's the slowest/least reliable
mon.emitCWPStatus,
} {
err = f(ctx)
if err != nil {
Expand Down
229 changes: 229 additions & 0 deletions pkg/monitor/cluster/clusterwideproxystatus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
package cluster

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

import (
"context"
"net/url"
"strconv"
"strings"

"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/sirupsen/logrus"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

apisubnet "github.com/Azure/ARO-RP/pkg/api/util/subnet"
arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
)

const (
cwp = "clusterWideProxy.status"
cwpErrorMessage = "NoProxy entries are incorrect"
cluster = "cluster"
mandatory_no_proxies = "localhost,127.0.0.1,.svc,.cluster.local,169.254.169.254"
AzureDNS = "168.63.129.16"
//169.254.169.254 (the IMDS IP)
//168.63.129.16 (Azure DNS, if no custom DNS exists)
//localhost, 127.0.0.1, .svc, .cluster.local
)

// Main function to emit CWP status
func (mon *Monitor) emitCWPStatus(ctx context.Context) error {
proxyConfig, err := mon.configcli.ConfigV1().Proxies().Get(ctx, cluster, metav1.GetOptions{})
if err != nil {
mon.log.Errorf("Error in getting the cluster wide proxy: %v", err)
return err
}
if proxyConfig.Spec.HTTPProxy == "" && proxyConfig.Spec.HTTPSProxy == "" && proxyConfig.Spec.NoProxy == "" {
mon.emitGauge(cwp, 1, map[string]string{
"status": strconv.FormatBool(false),
LiniSusan marked this conversation as resolved.
Show resolved Hide resolved
"Message": "CWP Not Enabled",
})
} else {
// Create the noProxy map for efficient lookups
no_proxy_list := strings.Split(proxyConfig.Spec.NoProxy, ",")
LiniSusan marked this conversation as resolved.
Show resolved Hide resolved
noProxyMap := make(map[string]bool)
var missing_no_proxy_list []string
for _, proxy := range no_proxy_list {
noProxyMap[proxy] = true
}

// Check mandatory no_proxy entries
for _, mandatory_no_proxy := range strings.Split(mandatory_no_proxies, ",") {
if !noProxyMap[mandatory_no_proxy] {
missing_no_proxy_list = append(missing_no_proxy_list, mandatory_no_proxy)
}
}
if !noProxyMap[AzureDNS] {
dnsConfigcluster, err := mon.operatorcli.OperatorV1().DNSes().Get(ctx, "default", metav1.GetOptions{})
if err != nil {
mon.log.Errorf("Error in getting DNS configuration: %v", err)
return err
}
if len(dnsConfigcluster.Spec.Servers) == 0 {
kimorris27 marked this conversation as resolved.
Show resolved Hide resolved
missing_no_proxy_list = append(missing_no_proxy_list, AzureDNS)
}
}

mastersubnetID, err := azure.ParseResourceID(mon.oc.Properties.MasterProfile.SubnetID)
if err != nil {
mon.log.Errorf("failed to parse the mastersubnetID: %v", err)
return err
}
token, err := mon.env.FPNewClientCertificateCredential(mon.tenantID, nil)
if err != nil {
mon.log.Errorf("failed to obtain FP Client Credentials: %v", err)
return err
}

// Create client factory
clientFactory, err := armnetwork.NewClientFactory(mastersubnetID.SubscriptionID, token, nil)
kimorris27 marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
mon.log.Errorf("failed to create client: %v", err)
return err
}

// Check master subnet
LiniSusan marked this conversation as resolved.
Show resolved Hide resolved
masterVnetID, _, err := apisubnet.Split(mon.oc.Properties.MasterProfile.SubnetID)
if err != nil {
mon.log.Errorf("failed to get the masterVnetID: %v", err)
return err
}
mastervnetId, err := azure.ParseResourceID(masterVnetID)
if err != nil {
mon.log.Errorf("failed to parse the masterVnetID: %v", err)
return err
}
res, err := clientFactory.NewSubnetsClient().Get(ctx, mastersubnetID.ResourceGroup, mastervnetId.ResourceName, mastersubnetID.ResourceName, &armnetwork.SubnetsClientGetOptions{Expand: nil})
if err != nil {
mon.log.Errorf("failed to finish the NewSubnetsClient request: %v", err)
return err
}

if res.Properties.AddressPrefix != nil {
if !noProxyMap[*res.Properties.AddressPrefix] {
missing_no_proxy_list = append(missing_no_proxy_list, *res.Properties.AddressPrefix)
}
}

// Check worker profiles
for _, workerProfile := range mon.oc.Properties.WorkerProfiles {
workersubnetID, err := azure.ParseResourceID(workerProfile.SubnetID)
if err != nil {
mon.log.Errorf("failed to parse the workersubnetID: %v", err)
return err
}
workerVnetID, _, err := apisubnet.Split(workerProfile.SubnetID)
if err != nil {
mon.log.Errorf("failed to feth the workerVnetID: %v", err)
return err
}
workervnetId, err := azure.ParseResourceID(workerVnetID)
if err != nil {
mon.log.Errorf("failed to parse the workerVnetID: %v", err)
return err
LiniSusan marked this conversation as resolved.
Show resolved Hide resolved
}
workerres, err := clientFactory.NewSubnetsClient().Get(ctx, workersubnetID.ResourceGroup, workervnetId.ResourceName, workersubnetID.ResourceName, &armnetwork.SubnetsClientGetOptions{Expand: nil})
if err != nil {
mon.log.Errorf("failed to finish the request: %v", err)
}
LiniSusan marked this conversation as resolved.
Show resolved Hide resolved
if workerres.Properties.AddressPrefix != nil {
workermachinesCIDR := *workerres.Properties.AddressPrefix
if !noProxyMap[workermachinesCIDR] {
missing_no_proxy_list = append(missing_no_proxy_list, workermachinesCIDR)
}
}
}

// Network Configuration Check
networkConfig, err := mon.configcli.ConfigV1().Networks().Get(ctx, cluster, metav1.GetOptions{})
if err != nil {
mon.log.Errorf("Error in getting network info: %v", err)
return err
}
for _, network := range networkConfig.Spec.ClusterNetwork {
if !noProxyMap[network.CIDR] {
missing_no_proxy_list = append(missing_no_proxy_list, network.CIDR)
}
}
for _, network := range networkConfig.Spec.ServiceNetwork {
if !noProxyMap[network] {
missing_no_proxy_list = append(missing_no_proxy_list, network)
}
}

// Gateway Domains Check
clusterdetails, err := mon.arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{})
if err != nil {
mon.log.Errorf("Error in getting cluster information: %v", err)
return err
}
clusterDomain := clusterdetails.Spec.Domain
if !(noProxyMap[clusterDomain] || noProxyMap[".apps."+clusterDomain] || noProxyMap["."+clusterDomain]) {
kimorris27 marked this conversation as resolved.
Show resolved Hide resolved
missing_no_proxy_list = append(missing_no_proxy_list, clusterDomain)
}
for _, gatewayDomain := range clusterdetails.Spec.GatewayDomains {
gatewayDomain = strings.ToLower(gatewayDomain)
if !noProxyMap[gatewayDomain] {
missing_no_proxy_list = append(missing_no_proxy_list, gatewayDomain)
}
}

// Infrastructure Configuration Check
infraConfig, err := mon.configcli.ConfigV1().Infrastructures().Get(ctx, cluster, metav1.GetOptions{})
if err != nil {
mon.log.Errorf("Error in getting Infrasturcture info: %v", err)
return err
}

// APIServerInternal URL Check
apiServerIntURL, err := url.Parse(infraConfig.Status.APIServerInternalURL)
if err != nil {
mon.log.Errorf("Error in parsing APIServerProfile: %v", err)
return err
}
apiServerIntdomain := strings.Split(apiServerIntURL.Host, ":")[0]
if !noProxyMap[apiServerIntdomain] {
missing_no_proxy_list = append(missing_no_proxy_list, apiServerIntdomain)
}

// APIServerProfile URL Check
apiServerProfileURL, err := url.Parse(mon.oc.Properties.APIServerProfile.URL)
if err != nil {
mon.log.Errorf("Error in parsing APIServerProfile: %v", err)
return err
}
apiServerProfiledomain := strings.Split(apiServerProfileURL.Host, ":")[0]
if !noProxyMap[apiServerProfiledomain] {
missing_no_proxy_list = append(missing_no_proxy_list, apiServerProfiledomain)
}

if len(missing_no_proxy_list) > 0 {
status := true
message := "CWP enabled but missing " + strings.Join(missing_no_proxy_list, ",") + " in the no_proxy list"
mon.emitGauge(cwp, 1, map[string]string{
"status": strconv.FormatBool(status),
"Message": message,
})
mon.log.Infof(message)
if mon.hourlyRun {
mon.log.WithFields(logrus.Fields{
"metric": cwp,
"status": strconv.FormatBool(status),
"Message": message,
}).Print()
}
} else {
mon.emitGauge(cwp, 1, map[string]string{
"status": strconv.FormatBool(false),
"Message": "CWP enabled successfully",
})
mon.log.Infof("CWP enabled successfully")
}
}

return nil
}
Loading
Loading