Add metrics for ec2 api calls made by CNI and expose via prometheus

aws · Nov 15, 2022 · bc04a28 · bc04a28
1 parent f8bc3b8
commit bc04a28
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 0 deletions.
diff --git a/cmd/cni-metrics-helper/metrics/cni_metrics.go b/cmd/cni-metrics-helper/metrics/cni_metrics.go
@@ -147,6 +147,13 @@ var InterestingCNIMetrics = map[string]metricsConvert{
 				actionFunc: metricsAdd,
 				data:       &dataPoints{},
 				logToFile:  true}}},
+	"awscni_ec2api_req_count": {
+		actions: []metricsAction{
+			{cwMetricName: "ec2ApiReqCount",
+				matchFunc:  matchAny,
+				actionFunc: metricsAdd,
+				data:       &dataPoints{},
+				logToFile:  true}}},
 }
 
 // CNIMetricsTarget defines data structure for kube-state-metric target

diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go
@@ -105,6 +105,13 @@ var (
 		},
 		[]string{"fn", "error"},
 	)
+	ec2ApiReq = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "awscni_ec2api_req_count",
+			Help: "The number of requests made to EC2 APIs by CNI",
+		},
+		[]string{"fn"},
+	)
 	prometheusRegistered = false
 )
 
@@ -306,6 +313,7 @@ func prometheusRegister() {
 		prometheus.MustRegister(awsAPILatency)
 		prometheus.MustRegister(awsAPIErr)
 		prometheus.MustRegister(awsUtilsErr)
+		prometheus.MustRegister(ec2ApiReq)
 		prometheusRegistered = true
 	}
 }
@@ -555,6 +563,7 @@ func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string) error {
 			}
 			start := time.Now()
 			_, err = cache.ec2SVC.ModifyNetworkInterfaceAttributeWithContext(context.Background(), attributeInput)
+			ec2ApiReq.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc()
 			awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 			if err != nil {
 				if aerr, ok := err.(awserr.Error); ok {
@@ -701,6 +710,7 @@ func (cache *EC2InstanceMetadataCache) awsGetFreeDeviceNumber() (int, error) {
 
 	start := time.Now()
 	result, err := cache.ec2SVC.DescribeInstancesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("DescribeInstances").Inc()
 	awsAPILatency.WithLabelValues("DescribeInstances", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:DescribeInstances")
@@ -765,6 +775,7 @@ func (cache *EC2InstanceMetadataCache) AllocENI(useCustomCfg bool, sg []*string,
 
 	start := time.Now()
 	_, err = cache.ec2SVC.ModifyNetworkInterfaceAttributeWithContext(context.Background(), attributeInput)
+	ec2ApiReq.WithLabelValues("ModifyNetworkInterfaceAttribute").Inc()
 	awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:ModifyNetworkInterfaceAttribute")
@@ -795,6 +806,7 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) {
 	}
 	start := time.Now()
 	attachOutput, err := cache.ec2SVC.AttachNetworkInterfaceWithContext(context.Background(), attachInput)
+	ec2ApiReq.WithLabelValues("AttachNetworkInterface").Inc()
 	awsAPILatency.WithLabelValues("AttachNetworkInterface", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:AttachNetworkInterface")
@@ -844,6 +856,7 @@ func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string
 
 	start := time.Now()
 	result, err := cache.ec2SVC.CreateNetworkInterfaceWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("CreateNetworkInterface").Inc()
 	awsAPILatency.WithLabelValues("CreateNetworkInterface", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:CreateNetworkInterface")
@@ -894,6 +907,7 @@ func (cache *EC2InstanceMetadataCache) TagENI(eniID string, currentTags map[stri
 	return retry.NWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxENIBackoffDelay, 0.3, 2), 5, func() error {
 		start := time.Now()
 		_, err := cache.ec2SVC.CreateTagsWithContext(context.Background(), input)
+		ec2ApiReq.WithLabelValues("CreateTags").Inc()
 		awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 		if err != nil {
 			CheckAPIErrorAndBroadcastEvent(err, "ec2:CreateTags")
@@ -953,6 +967,7 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe
 	err = retry.NWithBackoff(retry.NewSimpleBackoff(time.Millisecond*200, maxBackoffDelay, 0.15, 2.0), maxENIEC2APIRetries, func() error {
 		start := time.Now()
 		_, ec2Err := cache.ec2SVC.DetachNetworkInterfaceWithContext(context.Background(), detachInput)
+		ec2ApiReq.WithLabelValues("DetachNetworkInterface").Inc()
 		awsAPILatency.WithLabelValues("DetachNetworkInterface", fmt.Sprint(ec2Err != nil), awsReqStatus(ec2Err)).Observe(msSince(start))
 		if ec2Err != nil {
 			CheckAPIErrorAndBroadcastEvent(err, "ec2:DetachNetworkInterface")
@@ -989,6 +1004,7 @@ func (cache *EC2InstanceMetadataCache) getENIAttachmentID(eniID string) (*string
 
 	start := time.Now()
 	result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc()
 	awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -1025,6 +1041,7 @@ func (cache *EC2InstanceMetadataCache) deleteENI(eniName string, maxBackoffDelay
 	err := retry.NWithBackoff(retry.NewSimpleBackoff(time.Millisecond*500, maxBackoffDelay, 0.15, 2.0), maxENIEC2APIRetries, func() error {
 		start := time.Now()
 		_, ec2Err := cache.ec2SVC.DeleteNetworkInterfaceWithContext(context.Background(), deleteInput)
+		ec2ApiReq.WithLabelValues("DeleteNetworkInterface").Inc()
 		awsAPILatency.WithLabelValues("DeleteNetworkInterface", fmt.Sprint(ec2Err != nil), awsReqStatus(ec2Err)).Observe(msSince(start))
 		if ec2Err != nil {
 			if aerr, ok := ec2Err.(awserr.Error); ok {
@@ -1053,6 +1070,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [
 
 	start := time.Now()
 	result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc()
 	awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -1082,6 +1100,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (add
 
 	start := time.Now()
 	result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc()
 	awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -1112,6 +1131,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(eniID string) (add
 
 	start := time.Now()
 	result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc()
 	awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -1155,6 +1175,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult,
 		input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: aws.StringSlice(eniIDs)}
 		start := time.Now()
 		ec2Response, err = cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input)
+		ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc()
 		awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 		if err == nil {
 			// No error, exit the loop
@@ -1347,6 +1368,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddress(eniID string) error {
 
 	start := time.Now()
 	output, err := cache.ec2SVC.AssignPrivateIpAddressesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("AssignPrivateIpAddresses").Inc()
 	awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:AssignPrivateIpAddresses")
@@ -1369,6 +1391,7 @@ func (cache *EC2InstanceMetadataCache) FetchInstanceTypeLimits() error {
 	var eniLimits InstanceTypeLimits
 	describeInstanceTypesInput := &ec2.DescribeInstanceTypesInput{InstanceTypes: []*string{aws.String(cache.instanceType)}}
 	output, err := cache.ec2SVC.DescribeInstanceTypesWithContext(context.Background(), describeInstanceTypesInput)
+	ec2ApiReq.WithLabelValues("DescribeInstanceTypes").Inc()
 	if err != nil || len(output.InstanceTypes) != 1 {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:DescribeInstanceTypes")
 		return errors.New(fmt.Sprintf("Failed calling DescribeInstanceTypes for `%s`: %v", cache.instanceType, err))
@@ -1473,6 +1496,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int
 
 	start := time.Now()
 	output, err := cache.ec2SVC.AssignPrivateIpAddressesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("AssignPrivateIpAddresses").Inc()
 	awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:AssignPrivateIpAddresses")
@@ -1503,6 +1527,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPv6Prefixes(eniID string) ([]*strin
 	}
 	start := time.Now()
 	output, err := cache.ec2SVC.AssignIpv6AddressesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("AssignIpv6Addresses").Inc()
 	awsAPILatency.WithLabelValues("AssignIpv6AddressesWithContext", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:AssignPrivateIpv6Addresses")
@@ -1602,6 +1627,7 @@ func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []st
 
 	start := time.Now()
 	_, err := cache.ec2SVC.UnassignPrivateIpAddressesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("UnassignPrivateIpAddresses").Inc()
 	awsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:UnassignPrivateIpAddresses")
@@ -1628,6 +1654,7 @@ func (cache *EC2InstanceMetadataCache) DeallocPrefixAddresses(eniID string, pref
 
 	start := time.Now()
 	_, err := cache.ec2SVC.UnassignPrivateIpAddressesWithContext(context.Background(), input)
+	ec2ApiReq.WithLabelValues("UnassignPrivateIpAddresses").Inc()
 	awsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 	if err != nil {
 		CheckAPIErrorAndBroadcastEvent(err, "ec2:UnassignPrivateIpAddresses")
@@ -1688,6 +1715,7 @@ func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDe
 	_ = retry.NWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxBackoffDelay, 0.3, 2), 5, func() error {
 		start := time.Now()
 		_, err := cache.ec2SVC.CreateTagsWithContext(context.Background(), input)
+		ec2ApiReq.WithLabelValues("CreateTags").Inc()
 		awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 		if err != nil {
 			CheckAPIErrorAndBroadcastEvent(err, "ec2:CreateTags")
@@ -1869,6 +1897,7 @@ func (cache *EC2InstanceMetadataCache) getENIsFromPaginatedDescribeNetworkInterf
 		awsAPIErrInc("DescribeNetworkInterfaces", err)
 		return err
 	}
+	ec2ApiReq.WithLabelValues("DescribeNetworkInterfaces").Inc()
 	return innerErr
 }