From 9ff25af064c2bd841f28ab9f6df1e1b3fac2104c Mon Sep 17 00:00:00 2001 From: Raghul Christus Date: Wed, 7 Oct 2020 12:28:48 +0530 Subject: [PATCH 1/4] Enhancements and bug fixes --- stats_lambda/Makefile | 5 +- stats_lambda/stats.py | 212 +++++++++++++++++++++++++++--------------- 2 files changed, 139 insertions(+), 78 deletions(-) diff --git a/stats_lambda/Makefile b/stats_lambda/Makefile index 8e51360..957c548 100644 --- a/stats_lambda/Makefile +++ b/stats_lambda/Makefile @@ -10,7 +10,10 @@ help: stats.zip: stats.py - zip -r9 stats.zip stats.py + # AWS Lambda NOW expects the script to be named as lambda_function.py + cp stats.py lambda_function.py + zip -r9 stats.zip lambda_function.py + rm -f lambda_function.py package-lambda: stats.zip @echo "create/update lambda deployment package (stats.zip)" diff --git a/stats_lambda/stats.py b/stats_lambda/stats.py index fe151a1..36ef250 100644 --- a/stats_lambda/stats.py +++ b/stats_lambda/stats.py @@ -3,8 +3,9 @@ import sys import urllib2 import os -from datetime import datetime +from datetime import datetime, timedelta import json +import copy logging.basicConfig() @@ -16,12 +17,96 @@ ec2_client = boto3.client('ec2') cw_client = boto3.client('cloudwatch') asg_client = boto3.client('autoscaling') - - -def get_stats(vpx_instance_info): - ns_password = vpx_instance_info['instance-id'] - url = 'http://{}/nitro/v1/stat/lbvserver/'.format(vpx_instance_info['nsip']) - headers = {'Content-Type': 'application/json', 'X-NITRO-USER': 'nsroot', 'X-NITRO-PASS': ns_password} +METRICS_TEMPLATE = "https://raw.githubusercontent.com/christus02/citrix-adc-metrics-exporter-serverless/master/utils/metrics-template-creator/citrix-adc-cloudwatch-metrics-template.json" +CLOUDWATCH_NAMESPACE = 'CITRIXADC' +''' + Use the INCLUDE_FEATURES list to specify what features stats + to pull and push to CloudWatch + INCLUDE_FEATURES = [] (empty list) would enable stats to be pulled + for all the features specified in the Metrics JSON +''' +INCLUDE_FEATURES = ['system', 'protocolhttp', 'lbvserver', 'csvserver', 'service'] +#INCLUDE_FEATURES = [] + +def get_metrics_template(): + ''' + Method to fetch the Metrics Template from a URL + ''' + headers = {'Content-Type': 'application/json'} + r = urllib2.Request(METRICS_TEMPLATE, headers=headers) + try: + resp = urllib2.urlopen(r) + return json.loads(resp.read()) + except urllib2.HTTPError as hte: + logger.info("Error Fetching the metrics template : Error code: " + + str(hte.code) + ", reason=" + hte.reason) + except: + logger.warn("Caught exception: " + str(sys.exc_info()[:2])) + return {} + +def get_all_stats(vpx_instance_info, feature_list): + ''' + Method to fetch all Nitro Stats based on the provided feature list + ''' + stats = {} + for feature in feature_list: + stats[feature] = get_feature_stats(vpx_instance_info, feature) + return stats + +def parse_stats(vpx_instance_info, metrics_json, stats): + ''' + Method to update the metrics json template with the + Nitro Stats got from the VPX + ''' + filled_metrics = [] + for feature in stats.keys(): + if feature not in stats[feature]: + continue + for counter in metrics_json[feature]['counters']: + filled_counter = counter + if filled_counter['MetricName'] in stats[feature][feature]: + filled_counter['Value'] = int(stats[feature][feature].get(filled_counter['MetricName'], 0)) + filled_counter['Timestamp'] = datetime.now() - timedelta(days=2) + filled_counter['Dimensions'][1]['Value'] = vpx_instance_info['asg-name'] # AutoScale Group + filled_counter['Dimensions'][2]['Value'] = vpx_instance_info['instance-id'] # Instance ID + filled_metrics.append(filled_counter) + return filled_metrics + +def fill_up_metrics(vpx_instance_info, metrics_json): + ''' + Method to fetch all the Nitro stats from the VPX + and fill the JSON Metrics accordingly with the Nitro Stats + Value + ''' + features = metrics_json.keys() + selected_features = [] + if len(INCLUDE_FEATURES) != 0: + # Get Selective Stats only + for feature in features: + if feature in INCLUDE_FEATURES: + selected_features.append(feature) + else: + selected_features = features + + all_stats = get_all_stats(vpx_instance_info, selected_features) + filled_metrics = parse_stats(vpx_instance_info, metrics_json, all_stats) + return filled_metrics + +def get_feature_stats(vpx_instance_info,feature): + ''' + Method to fetch feature specific Nitro stats from VPX + ''' + REQUEST_METHOD = "http" # Choose protocol as http or https + CITRIX_ADC_USERNAME = "nsroot" + CITRIX_ADC_PASSWORD = vpx_instance_info['instance-id'] + NSIP = vpx_instance_info['nsip'] + + if vpx_instance_info['nsip-public'] is not "": + logger.info("Getting Stats from VPX over it's public NSIP " + vpx_instance_info['nsip-public']) + NSIP = vpx_instance_info['nsip-public'] + + url = '{}://{}/nitro/v1/stat/{}/'.format(REQUEST_METHOD, NSIP, feature) + headers = {'Content-Type': 'application/json', 'X-NITRO-USER': CITRIX_ADC_USERNAME, 'X-NITRO-PASS': CITRIX_ADC_PASSWORD} r = urllib2.Request(url, headers=headers) try: resp = urllib2.urlopen(r) @@ -31,27 +116,12 @@ def get_stats(vpx_instance_info): str(hte.code) + ", reason=" + hte.reason) except: logger.warn("Caught exception: " + str(sys.exc_info()[:2])) - return "{}" - - -def make_metric(metricname, dimensions, value, unit): - metric = {'MetricName': metricname, - 'Dimensions': dimensions, - 'Timestamp': datetime.now(), - 'Value': value, - 'Unit': unit - } - return metric - - -def make_dimensions(dims): - dimensions = [] - for d in dims.keys(): - dimensions.append({'Name': d, 'Value': dims[d]}) - return dimensions - + return {} def get_vpx_instances(vpx_asg_name): + ''' + Get all the VPX instances in the provided AutoScale Group + ''' result = [] logger.info("Looking for instances in ASG:" + vpx_asg_name) groups = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[vpx_asg_name]) @@ -77,56 +147,41 @@ def get_vpx_instances(vpx_asg_name): logger.info("Found net interface for " + ec2_instance_id + ", state=" + net_if['Status']) if net_if['Status'] == 'in-use': + nsip_public = net_if['PrivateIpAddresses'][0].get('Association',{}) + nsip_public = nsip_public.get('PublicIp', "") nsip = net_if['PrivateIpAddresses'][0]['PrivateIpAddress'] - logger.info("Found NSIP ip for " + ec2_instance_id + ": " + nsip) + logger.info("Found Private NSIP ip for " + ec2_instance_id + ": " + nsip) instance_info['nsip'] = nsip + instance_info['nsip-public'] = nsip_public + if nsip_public is not "": + logger.info("Found Public NSIP ip for " + ec2_instance_id + ": " + nsip_public) result.append(instance_info) return result - -def put_aggr_stats(asg_name, stats_list): - aggregate_stats = {} - for stat in stats_list: - lbstats = stat.get('lbvserver') - if lbstats is None: - logger.info("No stats found") - continue - for lbstat in lbstats: - aggregate_stats['totalrequests'] = aggregate_stats.get('totalrequests', 0) + int(lbstat['totalrequests']) - aggregate_stats['totalrequestbytes'] = aggregate_stats.get('totalrequestbytes', 0) + int(lbstat['totalrequestbytes']) - aggregate_stats['curclntconnections'] = aggregate_stats.get('curclntconnections', 0) + int(lbstat['curclntconnections']) - aggregate_stats['surgecount'] = aggregate_stats.get('surgecount', 0) + int(lbstat['surgecount']) - - dims = {'vpxasg': asg_name} - dimensions = make_dimensions(dims) - metricData = [make_metric('totalrequests', dimensions, aggregate_stats.get('totalrequests', 0), 'Count'), - make_metric('totalrequestbytes', dimensions, aggregate_stats.get('totalrequestbytes', 0), 'Count'), - make_metric('curclntconnections', dimensions, aggregate_stats.get('curclntconnections', 0), 'Count'), - make_metric('surgecount', dimensions, aggregate_stats.get('surgecount', 0), 'Count'), - ] - cw_client.put_metric_data(Namespace='NetScaler', MetricData=metricData) - - -def put_stats(vpx_info, stats): - lbstats = stats.get('lbvserver') - if lbstats is None: - logger.info("No stats found") - return - - for lbstat in lbstats: - dims = {'lbname': lbstat['name'], 'vpxinstance': vpx_info['instance-id'], 'vpxasg': vpx_info['asg-name']} - dimensions = make_dimensions(dims) - # TODO sanitize str->int conv - metricData = [make_metric('totalrequests', dimensions, int(lbstat['totalrequests']), 'Count'), - make_metric('totalrequestbytes', dimensions, int(lbstat['totalrequestbytes']), 'Count'), - make_metric('curclntconnections', dimensions, int(lbstat['curclntconnections']), 'Count'), - make_metric('surgecount', dimensions, int(lbstat['surgecount']), 'Count'), - make_metric('health', dimensions, int(lbstat['vslbhealth']), 'Count'), - make_metric('state', dimensions, (lambda s: 1 if s == 'UP' else 0)(lbstat['state']), 'Count'), - make_metric('actsvcs', dimensions, int(lbstat['actsvcs']), 'Count'), - make_metric('inactsvcs', dimensions, int(lbstat['inactsvcs']), 'Count') - ] - cw_client.put_metric_data(Namespace='NetScaler', MetricData=metricData) +def split_metrics_list(metrics, size=20): + ''' + Method to split a list into chunks of specified size + ''' + for i in range(0, len(metrics), size): + yield metrics[i:i + size] + +def push_stats(metricData, namespace=CLOUDWATCH_NAMESPACE): + ''' + Method to push the metrics to Cloud Watch + ''' + if (len(metricData) > 20): + ''' + The max metric list length is 20 for AWS CloudWatch + So splitting list into lengths of 20 and + pushing multiple times if the length is greater than 20 + ''' + chuncked_metricData = split_metrics_list(metricData) + for data in chuncked_metricData: + push_out = cw_client.put_metric_data(Namespace=namespace, MetricData=data) + logger.info("Result of Pushing Metrics to Cloud Watch: " + str(push_out)) + else: + push_out = cw_client.put_metric_data(Namespace=namespace, MetricData=metricData) + logger.info("Result of Pushing Metrics to Cloud Watch: " + str(push_out)) def lambda_handler(event, context): @@ -137,11 +192,14 @@ def lambda_handler(event, context): logger.warn("Bailing since we can't get the required env var: " + ke.args[0]) return - + ''' + 1. Import the JSON Metrics Template + 2. Pull Nitro Stats from VPX(s) based on the JSON Metrics Template + 3. Fill the JSON Metrics template with the Nitro Stats value + 4. Push the Metrics to CloudWatch + ''' + metrics_json = get_metrics_template() vpx_instances = get_vpx_instances(asg_name) - stats_list = [] for vpx in vpx_instances: - stats = get_stats(vpx) - stats_list.append(stats) - put_stats(vpx, stats) - put_aggr_stats(asg_name, stats_list) + stats = fill_up_metrics(vpx, metrics_json) + push_stats(stats) From d74af5281fc0cdad05db5298fe0a10205b2c8188 Mon Sep 17 00:00:00 2001 From: Raghul Christus Date: Wed, 7 Oct 2020 12:30:09 +0530 Subject: [PATCH 2/4] Enhancements and bug fixes --- stats_lambda/stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats_lambda/stats.py b/stats_lambda/stats.py index 36ef250..15d4f03 100644 --- a/stats_lambda/stats.py +++ b/stats_lambda/stats.py @@ -3,7 +3,7 @@ import sys import urllib2 import os -from datetime import datetime, timedelta +from datetime import datetime import json import copy @@ -66,7 +66,7 @@ def parse_stats(vpx_instance_info, metrics_json, stats): filled_counter = counter if filled_counter['MetricName'] in stats[feature][feature]: filled_counter['Value'] = int(stats[feature][feature].get(filled_counter['MetricName'], 0)) - filled_counter['Timestamp'] = datetime.now() - timedelta(days=2) + filled_counter['Timestamp'] = datetime.now() filled_counter['Dimensions'][1]['Value'] = vpx_instance_info['asg-name'] # AutoScale Group filled_counter['Dimensions'][2]['Value'] = vpx_instance_info['instance-id'] # Instance ID filled_metrics.append(filled_counter) From 263ac6c998516b9d65cf6629c789163baaa3e2f2 Mon Sep 17 00:00:00 2001 From: Raghul Christus Date: Wed, 7 Oct 2020 14:27:41 +0530 Subject: [PATCH 3/4] Enhancements and bug fixes --- stats_lambda/stats.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/stats_lambda/stats.py b/stats_lambda/stats.py index 15d4f03..ec798fe 100644 --- a/stats_lambda/stats.py +++ b/stats_lambda/stats.py @@ -64,6 +64,10 @@ def parse_stats(vpx_instance_info, metrics_json, stats): continue for counter in metrics_json[feature]['counters']: filled_counter = counter + if type(stats[feature][feature]) == list: + filled_counter = get_each_stats(filled_counter, stats[feature][feature], feature, vpx_instance_info) + filled_metrics.extend(filled_counter) # Extend the list - Don't append + continue if filled_counter['MetricName'] in stats[feature][feature]: filled_counter['Value'] = int(stats[feature][feature].get(filled_counter['MetricName'], 0)) filled_counter['Timestamp'] = datetime.now() @@ -72,6 +76,24 @@ def parse_stats(vpx_instance_info, metrics_json, stats): filled_metrics.append(filled_counter) return filled_metrics +def get_each_stats(filled_counter, stats, feature, vpx_instance_info): + ''' + Method to iterate through the list of entities and create metrics + ''' + filled_metrics = [] + for each_stat in stats: + if filled_counter['MetricName'] in each_stat: + filled_counter['Value'] = int(each_stat.get(filled_counter['MetricName'], 0)) + filled_counter['Timestamp'] = datetime.now() + filled_counter['Dimensions'][1]['Value'] = vpx_instance_info['asg-name'] # AutoScale Group + filled_counter['Dimensions'][2]['Value'] = vpx_instance_info['instance-id'] # Instance ID + if len(filled_counter['Dimensions']) == 4: # assume the feature dimension is already set + filled_counter['Dimensions'][3]['Value'] = each_stat['name'] + elif len(filled_counter['Dimensions']) == 3: # newly add the feature dimension + filled_counter['Dimensions'].append({'Name': feature, 'Value': each_stat['name']}) # Add the name of the feature to a dimension + filled_metrics.append(filled_counter) + return filled_metrics + def fill_up_metrics(vpx_instance_info, metrics_json): ''' Method to fetch all the Nitro stats from the VPX @@ -202,4 +224,4 @@ def lambda_handler(event, context): vpx_instances = get_vpx_instances(asg_name) for vpx in vpx_instances: stats = fill_up_metrics(vpx, metrics_json) - push_stats(stats) + push_stats(stats) \ No newline at end of file From 19b596ef23b8613338d3d50220bb63f4d8d10b39 Mon Sep 17 00:00:00 2001 From: Raghul Christus Date: Fri, 9 Oct 2020 00:51:32 +0530 Subject: [PATCH 4/4] Datadog Support and Review comments incorporated --- stats_lambda/Makefile | 7 +- stats_lambda/citrixadcmetrics.py | 564 +++++++++++++++++++++++++++++++ stats_lambda/stats.py | 238 ++++++++++--- 3 files changed, 754 insertions(+), 55 deletions(-) create mode 100644 stats_lambda/citrixadcmetrics.py diff --git a/stats_lambda/Makefile b/stats_lambda/Makefile index 957c548..e50b1d4 100644 --- a/stats_lambda/Makefile +++ b/stats_lambda/Makefile @@ -12,7 +12,12 @@ help: stats.zip: stats.py # AWS Lambda NOW expects the script to be named as lambda_function.py cp stats.py lambda_function.py - zip -r9 stats.zip lambda_function.py + rm stats.zip + rm -rf package/ + pip install --target ./package datadog + cp -f citrixadcmetrics.py package/ + cd package && zip -r9 ../stats.zip . + cd .. && zip -g stats.zip lambda_function.py rm -f lambda_function.py package-lambda: stats.zip diff --git a/stats_lambda/citrixadcmetrics.py b/stats_lambda/citrixadcmetrics.py new file mode 100644 index 0000000..8e84d53 --- /dev/null +++ b/stats_lambda/citrixadcmetrics.py @@ -0,0 +1,564 @@ +metrics = {'csvserver': [{'Description': 'citrixadc_cs_packets_sent_total', + 'MetricName': 'totalpktssent', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_hits_total', + 'MetricName': 'tothits', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_request_bytes_total', + 'MetricName': 'totalrequestbytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_packets_received_total', + 'MetricName': 'totalpktsrecvd', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_responses_total', + 'MetricName': 'totalresponses', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_tolerating_ttlb_transactions_count', + 'MetricName': 'toleratingttlbtransactionsrate', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_ttlb_calculated_transactions_total', + 'MetricName': 'totcltttlbtransactions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_deferred_requests_total', + 'MetricName': 'deferredreq', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_spillover_count_total', + 'MetricName': 'totspillovers', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_invalid_response_request_total', + 'MetricName': 'invalidrequestresponse', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_requests_total', + 'MetricName': 'totalrequests', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_invalid_response_request_dropped_total', + 'MetricName': 'invalidrequestresponsedropped', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_frustrating_transactions_total', + 'MetricName': 'frustratingttlbtransactions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_tolerating_ttlb_transactions_total', + 'MetricName': 'toleratingttlbtransactions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_backup_server_divert_count_total', + 'MetricName': 'totvserverdownbackuphits', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_response_bytes_received_total', + 'MetricName': 'totalresponsebytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_request_rate_bytes', + 'MetricName': 'requestbytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_request_rate', + 'MetricName': 'requestsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_packets_received_rate', + 'MetricName': 'pktsrecvdrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_frustrating_transactions_rate', + 'MetricName': 'frustratingttlbtransactionsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_client_response_time_adex', + 'MetricName': 'cltresponsetimeapdex', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_spill_over_threshold', + 'MetricName': 'sothreshold', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_actual_server_current_connections', + 'MetricName': 'cursrvrconnections', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_labeled_connections_count', + 'MetricName': 'labelledconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_current_tcp_sessions_count', + 'MetricName': 'curmptcpsessions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_deferred_requets_rate', + 'MetricName': 'deferredreqrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_average_ttlb', + 'MetricName': 'avgcltttlb', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_ttlb_transactions_rate', + 'MetricName': 'cltttlbtransactionsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_total_responses_rate', + 'MetricName': 'responsesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_total_packets_sent_rate', + 'MetricName': 'pktssentrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_current_tcp_subflows_count', + 'MetricName': 'cursubflowconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_current_client_connection_count', + 'MetricName': 'curclntconnections', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_push_label_count', + 'MetricName': 'pushlabel', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_hits_rate', + 'MetricName': 'hitsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_cs_established_connections_count', + 'MetricName': 'establishedconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_cs_response_bytes_received_rate', + 'MetricName': 'responsebytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}], + 'lbvserver': [{'Description': 'citrixadc_lb_packets_sent_total', + 'MetricName': 'totalpktssent', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_hits_total', + 'MetricName': 'tothits', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_request_bytes_total', + 'MetricName': 'totalrequestbytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_packets_received_total', + 'MetricName': 'totalpktsrecvd', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_responses_total', + 'MetricName': 'totalresponses', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_surge_count', + 'MetricName': 'surgecount', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_tolerable_transactions_count', + 'MetricName': 'toleratingttlbtransactionsrate', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_ttlb_calculated_transactions_total', + 'MetricName': 'totcltttlbtransactions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_deffered_requests_total', + 'MetricName': 'deferredreq', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_spillover_count_total', + 'MetricName': 'totspillovers', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_invalid_response_request_total', + 'MetricName': 'invalidrequestresponse', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_requests_total', + 'MetricName': 'totalrequests', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_invalid_response_request_dropped_total', + 'MetricName': 'invalidrequestresponsedropped', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_frustrating_transactions_total', + 'MetricName': 'frustratingttlbtransactions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_tolerable_transactions_total', + 'MetricName': 'toleratingttlbtransactions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_backup_server_divert_count_total', + 'MetricName': 'totvserverdownbackuphits', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_response_bytes_received_total', + 'MetricName': 'totalresponsebytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_busy_error_total', + 'MetricName': 'totalsvrbusyerr', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_request_rate_bytes', + 'MetricName': 'requestbytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_request_rate', + 'MetricName': 'requestsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_packets_received_rate', + 'MetricName': 'pktsrecvdrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_frustrating_transactions_rate', + 'MetricName': 'frustratingttlbtransactionsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_client_response_time_adex', + 'MetricName': 'cltresponsetimeapdex', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_spill_over_threshold', + 'MetricName': 'sothreshold', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_actual_server_current_connections', + 'MetricName': 'cursrvrconnections', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_surge_queue_requests_count', + 'MetricName': 'svcsurgecount', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_labeled_connections_count', + 'MetricName': 'labelledconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_current_mtcp_sessions_count', + 'MetricName': 'curmptcpsessions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_deferred_requets_rate', + 'MetricName': 'deferredreqrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_average_ttlb', + 'MetricName': 'avgcltttlb', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_ttlb_transactions_rate', + 'MetricName': 'cltttlbtransactionsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_total_responses_rate', + 'MetricName': 'responsesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_active_sessions_count', + 'MetricName': 'actsvcs', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_total_packets_sent_rate', + 'MetricName': 'pktssentrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_current_mtcp_subflows_count', + 'MetricName': 'cursubflowconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_current_client_connection_count', + 'MetricName': 'curclntconnections', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_push_label_count', + 'MetricName': 'pushlabel', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_hits_rate', + 'MetricName': 'hitsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_established_connections_count', + 'MetricName': 'establishedconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_response_bytes_received_rate', + 'MetricName': 'responsebytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_lb_inactive_services_count', + 'MetricName': 'inactsvcs', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_lb_busy_error_rate', + 'MetricName': 'svrbusyerrrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}], + 'protocolhttp': [{'Description': 'citrixadc_http_tot_requests', + 'MetricName': 'httptotrequests', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_responses', + 'MetricName': 'httptotresponses', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_posts', + 'MetricName': 'httptotposts', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_gets', + 'MetricName': 'httptotgets', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_others', + 'MetricName': 'httptotothers', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_rx_request_bytes', + 'MetricName': 'httptotrxrequestbytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_rx_response_bytes', + 'MetricName': 'httptotrxresponsebytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_tot_tx_request_bytes', + 'MetricName': 'httptottxrequestbytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_noreuse_multipart_responses', + 'MetricName': 'httperrnoreusemultipart', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_incomplete_header_packets', + 'MetricName': 'httperrincompleteheaders', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_incomplete_requests', + 'MetricName': 'httperrincompleterequests', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_incomplete_responses', + 'MetricName': 'httperrincompleteresponses', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_server_responses', + 'MetricName': 'httperrserverbusy', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_large_body_packets', + 'MetricName': 'httperrlargecontent', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_large_chunk_requests', + 'MetricName': 'httperrlargechunk', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_err_tot_large_content_requests', + 'MetricName': 'httperrlargectlen', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_http_requests_rate', + 'MetricName': 'httprequestsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_rx_request_bytes_rate', + 'MetricName': 'httprxrequestbytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_rx_response_bytes_rate', + 'MetricName': 'httprxresponsebytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_tot_tx_request_bytes_rate', + 'MetricName': 'httptxrequestbytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_responses_rate', + 'MetricName': 'httpresponsesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_posts_rate', + 'MetricName': 'httppostsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_gets_rate', + 'MetricName': 'httpgetsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_others_rate', + 'MetricName': 'httpothersrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_err_noreuse_multipart_responses_rate', + 'MetricName': 'httperrnoreusemultipartrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_err_incomplete_requests_rate', + 'MetricName': 'httperrincompleterequestsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_err_incomplete_responses_rate', + 'MetricName': 'httperrincompleteresponsesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_http_err_server_responses_rate', + 'MetricName': 'httperrserverbusyrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}], + 'service': [{'Description': 'citrixadc_service_throughput', + 'MetricName': 'throughput', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_tot_requests', + 'MetricName': 'totalrequests', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_tot_responses', + 'MetricName': 'totalresponses', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_tot_request_bytes', + 'MetricName': 'totalrequestbytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_tot_response_bytes', + 'MetricName': 'totalresponsebytes', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_surge_count', + 'MetricName': 'surgecount', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_server_established_connections', + 'MetricName': 'svrestablishedconn', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_max_clients', + 'MetricName': 'maxclients', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_vsvr_hits', + 'MetricName': 'vsvrservicehits', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_throughput_rate', + 'MetricName': 'throughputrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_service_average_server_ttfb', + 'MetricName': 'avgsvrttfb', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_responses_rate', + 'MetricName': 'responsesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_service_request_bytes_rate', + 'MetricName': 'requestbytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_service_response_bytes_rate', + 'MetricName': 'responsebytesrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_service_current_client_connections', + 'MetricName': 'curclntconnections', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_current_server_connections', + 'MetricName': 'cursrvrconnections', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_current_pool_use', + 'MetricName': 'curreusepool', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_current_load', + 'MetricName': 'curload', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_current_flags', + 'MetricName': 'curtflags', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_service_vsvr_hits_rate', + 'MetricName': 'vsvrservicehitsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}, + {'Description': 'citrixadc_service_active_transactions', + 'MetricName': 'activetransactions', + 'Type': 'Count', + 'Unit': 'Count'}], + 'ssl': [{'Description': 'citrixadc_ssl_tot_sessions', + 'MetricName': 'ssltotsessions', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_ssl_session_rate', + 'MetricName': 'sslsessionsrate', + 'Type': 'Count', + 'Unit': 'Count/Second'}], + 'system': [{'Description': 'citrixadc_cpu_number', + 'MetricName': 'numcpus', + 'Type': 'Count', + 'Unit': 'Count'}, + {'Description': 'citrixadc_var_partition_free_mb', + 'MetricName': 'disk1avail', + 'Type': 'Count', + 'Unit': 'Megabytes'}, + {'Description': 'citrixadc_var_partition_used_mb', + 'MetricName': 'disk1used', + 'Type': 'Count', + 'Unit': 'Megabytes'}, + {'Description': 'citrixadc_flash_partition_free_mb', + 'MetricName': 'disk0avail', + 'Type': 'Count', + 'Unit': 'Megabytes'}, + {'Description': 'citrixadc_flash_partition_used_mb', + 'MetricName': 'disk0used', + 'Type': 'Count', + 'Unit': 'Megabytes'}, + {'Description': 'citrixadc_cpu_usage_percent', + 'MetricName': 'cpuusagepcnt', + 'Type': 'Count', + 'Unit': 'Percent'}, + {'Description': 'citrixadc_memory_usage_percent', + 'MetricName': 'memusagepcnt', + 'Type': 'Count', + 'Unit': 'Percent'}, + {'Description': 'citrixadc_management_cpu_usage_percent', + 'MetricName': 'mgmtcpuusagepcnt', + 'Type': 'Count', + 'Unit': 'Percent'}, + {'Description': 'citrixadc_packet_cpu_usage_percent', + 'MetricName': 'pktcpuusagepcnt', + 'Type': 'Count', + 'Unit': 'Percent'}, + {'Description': 'citrixadc_res_cpu_usage_percent', + 'MetricName': 'rescpuusagepcnt', + 'Type': 'Count', + 'Unit': 'Percent'}, + {'Description': 'citrixadc_var_partition_used_percent', + 'MetricName': 'disk1perusage', + 'Type': 'Count', + 'Unit': 'Percent'}, + {'Description': 'citrixadc_flash_partition_used_percent', + 'MetricName': 'disk0perusage', + 'Type': 'Count', + 'Unit': 'Percent'}]} \ No newline at end of file diff --git a/stats_lambda/stats.py b/stats_lambda/stats.py index ec798fe..697d763 100644 --- a/stats_lambda/stats.py +++ b/stats_lambda/stats.py @@ -6,6 +6,8 @@ from datetime import datetime import json import copy +from datadog import initialize, api +import citrixadcmetrics as metrics_template logging.basicConfig() @@ -17,8 +19,8 @@ ec2_client = boto3.client('ec2') cw_client = boto3.client('cloudwatch') asg_client = boto3.client('autoscaling') -METRICS_TEMPLATE = "https://raw.githubusercontent.com/christus02/citrix-adc-metrics-exporter-serverless/master/utils/metrics-template-creator/citrix-adc-cloudwatch-metrics-template.json" CLOUDWATCH_NAMESPACE = 'CITRIXADC' +DATADOG_PREFIX = 'citrixadc' ''' Use the INCLUDE_FEATURES list to specify what features stats to pull and push to CloudWatch @@ -28,47 +30,81 @@ INCLUDE_FEATURES = ['system', 'protocolhttp', 'lbvserver', 'csvserver', 'service'] #INCLUDE_FEATURES = [] -def get_metrics_template(): - ''' - Method to fetch the Metrics Template from a URL - ''' - headers = {'Content-Type': 'application/json'} - r = urllib2.Request(METRICS_TEMPLATE, headers=headers) - try: - resp = urllib2.urlopen(r) - return json.loads(resp.read()) - except urllib2.HTTPError as hte: - logger.info("Error Fetching the metrics template : Error code: " + - str(hte.code) + ", reason=" + hte.reason) - except: - logger.warn("Caught exception: " + str(sys.exc_info()[:2])) - return {} +CLOUDWATCH_TEMPLATE = { + "MetricName": "", + "Value": "", + "Timestamp": "", + "Unit": "Count", + "Dimensions": [ + { + "Name": "Description", + "Value": "" + }, + { + "Name": "CitrixADC-AutoScale-Group", + "Value": "" + }, + { + "Name": "CitrixADC-InstanceID", + "Value": "" + } + ] +} -def get_all_stats(vpx_instance_info, feature_list): - ''' - Method to fetch all Nitro Stats based on the provided feature list - ''' - stats = {} - for feature in feature_list: - stats[feature] = get_feature_stats(vpx_instance_info, feature) - return stats +DATADOG_TEMPLATE = { + "metric": "", + "description": "", + "type": "", + "points": "", + "host": "", + "tags": [ + "CitrixADC-AutoScale-Group:autoscalegroup", + "Source:AWS" + ] +} -def parse_stats(vpx_instance_info, metrics_json, stats): +def parse_stats_cloudwatch(vpx_instance_info, metrics, stats): ''' Method to update the metrics json template with the Nitro Stats got from the VPX + CLOUDWATCH_TEMPLATE = { + "MetricName": "", + "Value": "", + "Timestamp": "", + "Unit": "Count", + "Dimensions": [ + { + "Name": "Description", + "Value": "" + }, + { + "Name": "CitrixADC-AutoScale-Group", + "Value": "" + }, + { + "Name": "CitrixADC-InstanceID", + "Value": "" + } + ] + } ''' filled_metrics = [] for feature in stats.keys(): - if feature not in stats[feature]: - continue - for counter in metrics_json[feature]['counters']: - filled_counter = counter + for counter in metrics[feature]: + filled_counter = copy.deepcopy(CLOUDWATCH_TEMPLATE) if type(stats[feature][feature]) == list: - filled_counter = get_each_stats(filled_counter, stats[feature][feature], feature, vpx_instance_info) + filled_counter['MetricName'] = counter['MetricName'] + filled_counter['Dimensions'][0]['Value'] = counter['Description'] + filled_counter['Unit'] = counter['Unit'] + filled_counter = get_each_stats_cloudwatch(filled_counter, stats[feature][feature], feature, vpx_instance_info) filled_metrics.extend(filled_counter) # Extend the list - Don't append continue - if filled_counter['MetricName'] in stats[feature][feature]: + # Counter is from the input metrics template + # filled_counter is the CLOUDWATCH_TEMPLATE which is being filled + if counter['MetricName'] in stats[feature][feature]: + filled_counter['MetricName'] = counter['MetricName'] + filled_counter['Dimensions'][0]['Value'] = counter['Description'] + filled_counter['Unit'] = counter['Unit'] filled_counter['Value'] = int(stats[feature][feature].get(filled_counter['MetricName'], 0)) filled_counter['Timestamp'] = datetime.now() filled_counter['Dimensions'][1]['Value'] = vpx_instance_info['asg-name'] # AutoScale Group @@ -76,7 +112,7 @@ def parse_stats(vpx_instance_info, metrics_json, stats): filled_metrics.append(filled_counter) return filled_metrics -def get_each_stats(filled_counter, stats, feature, vpx_instance_info): +def get_each_stats_cloudwatch(filled_counter, stats, feature, vpx_instance_info): ''' Method to iterate through the list of entities and create metrics ''' @@ -94,26 +130,80 @@ def get_each_stats(filled_counter, stats, feature, vpx_instance_info): filled_metrics.append(filled_counter) return filled_metrics -def fill_up_metrics(vpx_instance_info, metrics_json): + +def parse_stats_datadog(vpx_instance_info, metrics, stats): ''' - Method to fetch all the Nitro stats from the VPX - and fill the JSON Metrics accordingly with the Nitro Stats - Value + Method to update the metrics json template with the + Nitro Stats got from the VPX + DATADOG_TEMPLATE = { + "metric": "", + "description": "", + "type": "", + "points": "", + "host": "", + "tags": [ + "CitrixADC-AutoScale-Group:autoscalegroup", + "Source:AWS" + ] + } ''' - features = metrics_json.keys() - selected_features = [] - if len(INCLUDE_FEATURES) != 0: - # Get Selective Stats only - for feature in features: - if feature in INCLUDE_FEATURES: - selected_features.append(feature) - else: - selected_features = features + filled_metrics = [] + for feature in stats.keys(): + for counter in metrics[feature]: + filled_counter = copy.deepcopy(DATADOG_TEMPLATE) + if type(stats[feature][feature]) == list: + filled_counter['metric'] = counter['MetricName'] + filled_counter['description'] = counter['Description'] + filled_counter['type'] = counter['Type'].lower() + filled_counter = get_each_stats_datadog(filled_counter, stats[feature][feature], feature, vpx_instance_info) + filled_metrics.extend(filled_counter) # Extend the list - Don't append + continue + # Counter is from the input metrics template + # filled_counter is the DATADOG_TEMPLATE which is being filled + if counter['MetricName'] in stats[feature][feature]: + filled_counter['metric'] = counter['MetricName'] + filled_counter['description'] = counter['Description'] + filled_counter['type'] = counter['Type'].lower() + filled_counter['points'] = int(stats[feature][feature].get(filled_counter['metric'], 0)) + filled_counter['host'] = vpx_instance_info['instance-id'] # Instance ID + filled_counter['tags'][0] = "CitrixADC-AutoScale-Group:" + vpx_instance_info['asg-name'] + filled_counter['metric'] = DATADOG_PREFIX + '.' + filled_counter['metric'] # Prefix with citrixadc for each metric + filled_metrics.append(filled_counter) + return filled_metrics - all_stats = get_all_stats(vpx_instance_info, selected_features) - filled_metrics = parse_stats(vpx_instance_info, metrics_json, all_stats) +def get_each_stats_datadog(filled_counter, stats, feature, vpx_instance_info): + ''' + Method to iterate through the list of entities and create metrics + ''' + filled_metrics = [] + for each_stat in stats: + if filled_counter['metric'] in each_stat: + filled_counter['points'] = int(each_stat.get(filled_counter['metric'], 0)) + filled_counter['host'] = vpx_instance_info['instance-id'] # Instance ID + filled_counter['tags'][0] = "CitrixADC-AutoScale-Group:" + vpx_instance_info['asg-name'] + # Assuming we have only 2 tags set + if len(filled_counter['tags']) == 3: + filled_counter['tags'][2] = feature + ":" + each_stat['name'] + elif len(filled_counter['tags']) == 2: + filled_counter['tags'].append(feature + ":" + each_stat['name']) # Add a tag with feature:name + filled_counter['metric'] = DATADOG_PREFIX + '.' + filled_counter['metric'] # Prefix with citrixadc for each metric + filled_metrics.append(filled_counter) return filled_metrics +def push_metrics_cloudwatch(vpx, metrics, stats): + filled_metrics = parse_stats_cloudwatch(vpx, metrics, stats) + post_cloudwatch_metrics_data(filled_metrics) + +def push_metrics_datadog(vpx, metrics, stats): + filled_metrics = parse_stats_datadog(vpx, metrics, stats) + post_datadog_metrics_data(filled_metrics) + +def pull_citrixadc_metrics(vpx, features): + stats = {} + for feature in features: + stats[feature] = get_feature_stats(vpx, feature) + return stats + def get_feature_stats(vpx_instance_info,feature): ''' Method to fetch feature specific Nitro stats from VPX @@ -187,7 +277,7 @@ def split_metrics_list(metrics, size=20): for i in range(0, len(metrics), size): yield metrics[i:i + size] -def push_stats(metricData, namespace=CLOUDWATCH_NAMESPACE): +def post_cloudwatch_metrics_data(metricData, namespace=CLOUDWATCH_NAMESPACE): ''' Method to push the metrics to Cloud Watch ''' @@ -205,23 +295,63 @@ def push_stats(metricData, namespace=CLOUDWATCH_NAMESPACE): push_out = cw_client.put_metric_data(Namespace=namespace, MetricData=metricData) logger.info("Result of Pushing Metrics to Cloud Watch: " + str(push_out)) +def post_datadog_metrics_data(metricData): + push_out = api.Metric.send(metricData) + logger.info("Result of Pushing Metrics to Datadog: " + str(push_out)) def lambda_handler(event, context): logger.info(str(event)) + + PUSH_TO_CLOUDWATCH = False + PUSH_TO_DATADOG = False + + # Check if Autoscale Group is provided in the ENV try: asg_name = os.environ['ASG_NAME'] except KeyError as ke: logger.warn("Bailing since we can't get the required env var: " + ke.args[0]) return + PUSH_TO_CLOUDWATCH = True + + # Check if Datadog's API Key is provided in the ENV + DATADOG_API_KEY = os.environ.get('DATADOG_API_KEY', '') + if DATADOG_API_KEY == '': + logger.warn("Could not push metrics to Datadog. Please provide the DataDog API Key in the ENV") + else: + logger.info("Pushing metrics to Datadog also") + PUSH_TO_DATADOG = True + # Datadog Configs + options = { + 'api_key': DATADOG_API_KEY + } + initialize(**options) + ''' - 1. Import the JSON Metrics Template - 2. Pull Nitro Stats from VPX(s) based on the JSON Metrics Template - 3. Fill the JSON Metrics template with the Nitro Stats value + 1. Use the Metrics Template bundled with the Lambda package + 2. Pull Nitro Stats from VPX(s) based on the Metrics Template + 3. Fill the Metrics template with the Nitro Stats value 4. Push the Metrics to CloudWatch + 5. Push to Metrics to Datadog (if enabled) ''' - metrics_json = get_metrics_template() + # Import the Metrics Template from the Lambda deployment package + metrics = metrics_template.metrics + + # Filter out the included features alone + features = metrics.keys() + selected_features = [] + if len(INCLUDE_FEATURES) != 0: + for feature in features: + if feature in INCLUDE_FEATURES: + selected_features.append(feature) + else: + selected_features = features + + # Get all Citrix ADC VPX from the provided AWS Autoscale Group vpx_instances = get_vpx_instances(asg_name) for vpx in vpx_instances: - stats = fill_up_metrics(vpx, metrics_json) - push_stats(stats) \ No newline at end of file + stats = pull_citrixadc_metrics(vpx, selected_features) + if PUSH_TO_CLOUDWATCH: + push_metrics_cloudwatch(vpx, metrics, stats) + if PUSH_TO_DATADOG: + push_metrics_datadog(vpx, metrics, stats)