From 29ed5c28612fc278c358f65a3a2deb1c4559d1eb Mon Sep 17 00:00:00 2001 From: Rudy Date: Tue, 29 Apr 2014 09:58:38 -0400 Subject: [PATCH 01/30] SNMP| First commit for SNMP Agent checks --- checks.d/snmp.py | 79 ++++++++++++++++++++++++++++++++++++++++ conf.d/snmp.yaml.example | 7 ++++ 2 files changed, 86 insertions(+) create mode 100644 checks.d/snmp.py create mode 100644 conf.d/snmp.yaml.example diff --git a/checks.d/snmp.py b/checks.d/snmp.py new file mode 100644 index 0000000000..812bae83ed --- /dev/null +++ b/checks.d/snmp.py @@ -0,0 +1,79 @@ +from checks import AgentCheck + +from pysnmp.entity.rfc3413.oneliner import cmdgen +import pysnmp.proto.rfc1902 as snmp_type + +snmp_counters = [snmp_type.Counter32, snmp_type.Counter64] +snmp_gauges = [snmp_type.Gauge32] + +target_oids = [ + (('SNMPv2-MIB','snmpInPkts'),'0'), + (('UDP-MIB','udpInDatagrams'),'0') + ] + +class SnmpCheck(AgentCheck): + + def __init__(self, name, init_config, agentConfig, instances=None): + AgentCheck.__init__(self, name, init_config, agentConfig, instances) + self.counter_state = {} + self.interface_list = {} + + @staticmethod + def get_auth_data(instance): + if "community_string" in instance: + return cmdgen.CommunityData(instance['community_string']) + else: + raise Exception("An authentication method needs to be provided") + + @staticmethod + def get_transport_target(instance): + if "ip_address" not in instance: + raise Exception("An IP address needs to be specified") + ip_address = instance["ip_address"] + port = instance.get("port", 161) + return cmdgen.UdpTransportTarget((ip_address,port)) + + def check(self, instance): + + transport_target = SnmpCheck.get_transport_target(instance) + auth_data = SnmpCheck.get_auth_data(instance) + + cmd_generator = cmdgen.CommandGenerator() + + errorIndication, errorStatus, errorIndex, varBinds = cmd_generator.getCmd( + auth_data, + transport_target, + *target_oids, + lookupNames=True, + lookupValues=True + ) + + if errorIndication: + self.log.warning(errorIndication) + else: + if errorStatus: + self.log.warning(errorStatus.prettyPrint()) + else: + for oid, value in varBinds: + self.report_as_statsd(oid.prettyPrint(), value) + + + def report_as_statsd(self, name, snmp_value): + snmp_class = getattr(snmp_value, '__class__') + value = int(snmp_value) + if snmp_class in snmp_counters: + self.counter(name, value, snmp_class) + elif snmp_class in snmp_gauges: + self.gauge(name, value) + + + def counter(self, name, value, snmp_class): + if name in self.counter_state: + diff = value - self.counter_state[name] + if diff < 0: + # Counters monotonically increase so it means the counter wrapped + diff += pow(2, 32 if snmp_class==snmp_type.Counter32 else 64) + self.increment(name, diff) + else: + self.log.info("Setting up initial value for Counter {0}".format(name)) + self.counter_state[name] = value diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example new file mode 100644 index 0000000000..97cd5d4d56 --- /dev/null +++ b/conf.d/snmp.yaml.example @@ -0,0 +1,7 @@ +init_config: + +#instances: +# - ip_address: localhost +# port: 1161 +# community_string: public + From 3a251aac1995c0e01f22ee026090ecf284ce47f6 Mon Sep 17 00:00:00 2001 From: Rudy Date: Tue, 29 Apr 2014 14:25:01 -0400 Subject: [PATCH 02/30] SNMP| Add interface monitoring --- checks.d/snmp.py | 102 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 20 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 812bae83ed..a8010ef37c 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -6,18 +6,63 @@ snmp_counters = [snmp_type.Counter32, snmp_type.Counter64] snmp_gauges = [snmp_type.Gauge32] -target_oids = [ - (('SNMPv2-MIB','snmpInPkts'),'0'), - (('UDP-MIB','udpInDatagrams'),'0') - ] +interface_oids = [ + ("IF-MIB", "ifInOctets"), + ("IF-MIB", "ifInErrors"), + ("IF-MIB", "ifInDiscards"), + ("IF-MIB", "ifOutOctets"), + ("IF-MIB", "ifOutErrors"), + ("IF-MIB", "ifOutDiscards") + ] + +device_oids = [ + (("UDP-MIB", "udpInDatagrams"),0), + (("TCP-MIB", "tcpCurrEstab"),0), + (("TCP-MIB", "tcpActiveOpens"),0), + (("TCP-MIB", "tcpPassiveOpens"),0) + ] + + + class SnmpCheck(AgentCheck): - + def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) self.counter_state = {} self.interface_list = {} - + for instance in instances: + if 'ip_address' in instance: + ip_address = instance["ip_address"] + self.counter_state[ip_address] = {} + self.interface_list[ip_address] = self.get_interfaces(instance) + + def get_interfaces(self, instance): + + interface_list = {} + + def get_interfaces_nb(): + result = SnmpCheck.snmp_get(instance, [(("IF-MIB","ifNumber"),0)])[0] + return int(result[1]) + + interface_nb = get_interfaces_nb() + interfaces_descr_oids = [] + for interface in range(interface_nb): + interface_index = interface + 1 #SNMP indexes start from 1 + interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) + interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) + + interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) + self.log.info(interfaces_description) + for i in range(interface_nb): + # order is guaranteed + descr = str(interfaces_description.pop(0)[1]) + type = int(interfaces_description.pop(0)[1]) + if type != 24: + interface_list[i+1] = descr + + return interface_list + @staticmethod def get_auth_data(instance): if "community_string" in instance: @@ -34,16 +79,33 @@ def get_transport_target(instance): return cmdgen.UdpTransportTarget((ip_address,port)) def check(self, instance): - + results = SnmpCheck.snmp_get(instance, device_oids) + for oid, value in results: + self.report_as_statsd(instance, oid, value, tags=None) + + for interface, descr in self.interface_list[instance['ip_address']].items(): + oids = [(oid, interface) for oid in interface_oids] + interface_results = SnmpCheck.snmp_get(instance, oids) + for oid, value in interface_results: + self.report_as_statsd(instance, oid, value, tags = ["interface:"+descr]) + + @staticmethod + def snmp_get(instance, oids): + """ + Perform a snmp get command to the device that instance + describe and return a list of tuble (name, values) + corresponding to the elements in oids. + """ transport_target = SnmpCheck.get_transport_target(instance) auth_data = SnmpCheck.get_auth_data(instance) cmd_generator = cmdgen.CommandGenerator() - errorIndication, errorStatus, errorIndex, varBinds = cmd_generator.getCmd( + snmp_command = cmd_generator.getCmd + errorIndication, errorStatus, errorIndex, varBinds = snmp_command( auth_data, transport_target, - *target_oids, + *oids, lookupNames=True, lookupValues=True ) @@ -54,26 +116,26 @@ def check(self, instance): if errorStatus: self.log.warning(errorStatus.prettyPrint()) else: - for oid, value in varBinds: - self.report_as_statsd(oid.prettyPrint(), value) - + return varBinds - def report_as_statsd(self, name, snmp_value): + def report_as_statsd(self, instance, oid, snmp_value, tags=[]): + name = "snmp." + oid.getMibSymbol()[1] snmp_class = getattr(snmp_value, '__class__') value = int(snmp_value) if snmp_class in snmp_counters: - self.counter(name, value, snmp_class) + self.counter(instance, name, value, snmp_class, tags) elif snmp_class in snmp_gauges: - self.gauge(name, value) + self.gauge(name, value, tags) - def counter(self, name, value, snmp_class): - if name in self.counter_state: - diff = value - self.counter_state[name] + def counter(self, instance, name, value, snmp_class, tags = []): + current_state = self.counter_state[instance['ip_address']] + if name in current_state: + diff = value - current_state[name] if diff < 0: # Counters monotonically increase so it means the counter wrapped diff += pow(2, 32 if snmp_class==snmp_type.Counter32 else 64) - self.increment(name, diff) + self.increment(name, diff,tags=tags) else: self.log.info("Setting up initial value for Counter {0}".format(name)) - self.counter_state[name] = value + current_state[name] = value From f9a004188de23ce9f15984bc9c842b74ab5580ac Mon Sep 17 00:00:00 2001 From: Rudy Date: Tue, 29 Apr 2014 15:53:23 -0400 Subject: [PATCH 03/30] SNMP| Fallback for interface identification in case of not indexed --- checks.d/snmp.py | 53 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index a8010ef37c..fafd924734 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -1,6 +1,7 @@ from checks import AgentCheck from pysnmp.entity.rfc3413.oneliner import cmdgen +from pysnmp.smi.exval import noSuchInstance import pysnmp.proto.rfc1902 as snmp_type snmp_counters = [snmp_type.Counter32, snmp_type.Counter64] @@ -43,23 +44,43 @@ def get_interfaces(self, instance): def get_interfaces_nb(): result = SnmpCheck.snmp_get(instance, [(("IF-MIB","ifNumber"),0)])[0] - return int(result[1]) + if noSuchInstance.isSameTypeWith(result[1]): + return None + else: + return int(result[1]) interface_nb = get_interfaces_nb() - interfaces_descr_oids = [] - for interface in range(interface_nb): - interface_index = interface + 1 #SNMP indexes start from 1 - interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) - interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) - - interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) - self.log.info(interfaces_description) - for i in range(interface_nb): - # order is guaranteed - descr = str(interfaces_description.pop(0)[1]) - type = int(interfaces_description.pop(0)[1]) - if type != 24: - interface_list[i+1] = descr + if interface_nb is not None: + interfaces_descr_oids = [] + for interface in range(interface_nb): + interface_index = interface + 1 #SNMP indexes start from 1 + interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) + interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) + + interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) + self.log.info(interfaces_description) + for i in range(interface_nb): + # order is guaranteed + descr = str(interfaces_description.pop(0)[1]) + type = int(interfaces_description.pop(0)[1]) + if type != 24: + interface_list[i+1] = descr + else: + empty_reply = False + interface_index = 1 + while not empty_reply: + interfaces_descr_oids = [] + interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) + interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) + interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) + descr = interfaces_description.pop(0)[1] + if noSuchInstance.isSameTypeWith(descr): + empty_reply= True + else: + type = int(interfaces_description.pop(0)[1]) + if type != 24: + interface_list[interface_index] = str(descr) + interface_index += 1 return interface_list @@ -119,6 +140,8 @@ def snmp_get(instance, oids): return varBinds def report_as_statsd(self, instance, oid, snmp_value, tags=[]): + if noSuchInstance.isSameTypeWith(snmp_value): + return name = "snmp." + oid.getMibSymbol()[1] snmp_class = getattr(snmp_value, '__class__') value = int(snmp_value) From fce8ab0e38fdbd0039442fe836801e47fcbff890 Mon Sep 17 00:00:00 2001 From: Rudy Date: Tue, 29 Apr 2014 16:36:16 -0400 Subject: [PATCH 04/30] SNMP| Add support for Custom tags Also fixes bug for Counter metrics on several interfaces --- checks.d/snmp.py | 14 +++++++++----- conf.d/snmp.yaml.example | 4 +++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index fafd924734..06dd3dd3ec 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -37,6 +37,8 @@ def __init__(self, name, init_config, agentConfig, instances=None): ip_address = instance["ip_address"] self.counter_state[ip_address] = {} self.interface_list[ip_address] = self.get_interfaces(instance) + tags = instance.get("tags",[]) + tags.append("device:" + ip_address) def get_interfaces(self, instance): @@ -100,15 +102,16 @@ def get_transport_target(instance): return cmdgen.UdpTransportTarget((ip_address,port)) def check(self, instance): + tags = instance.get("tags",[]) results = SnmpCheck.snmp_get(instance, device_oids) for oid, value in results: - self.report_as_statsd(instance, oid, value, tags=None) + self.report_as_statsd(instance, oid, value, tags=tags) for interface, descr in self.interface_list[instance['ip_address']].items(): oids = [(oid, interface) for oid in interface_oids] interface_results = SnmpCheck.snmp_get(instance, oids) for oid, value in interface_results: - self.report_as_statsd(instance, oid, value, tags = ["interface:"+descr]) + self.report_as_statsd(instance, oid, value, tags = tags + ["interface:"+descr]) @staticmethod def snmp_get(instance, oids): @@ -153,12 +156,13 @@ def report_as_statsd(self, instance, oid, snmp_value, tags=[]): def counter(self, instance, name, value, snmp_class, tags = []): current_state = self.counter_state[instance['ip_address']] - if name in current_state: - diff = value - current_state[name] + metric_id = name + str(tags) + if metric_id in current_state: + diff = value - current_state[metric_id] if diff < 0: # Counters monotonically increase so it means the counter wrapped diff += pow(2, 32 if snmp_class==snmp_type.Counter32 else 64) self.increment(name, diff,tags=tags) else: self.log.info("Setting up initial value for Counter {0}".format(name)) - current_state[name] = value + current_state[metric_id] = value diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index 97cd5d4d56..faa4e602ed 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -4,4 +4,6 @@ init_config: # - ip_address: localhost # port: 1161 # community_string: public - +# tags: +# - optional_tag_1 +# - optional_tag_2 From a18e457effdd626be9c5d2d1a899e2877ca25089 Mon Sep 17 00:00:00 2001 From: Rudy Date: Tue, 29 Apr 2014 18:03:23 -0400 Subject: [PATCH 05/30] SNMP| Allow use of snmp v3 authentication --- checks.d/snmp.py | 18 ++++++++++++++++++ conf.d/snmp.yaml.example | 14 ++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 06dd3dd3ec..d9e4341881 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -90,6 +90,24 @@ def get_interfaces_nb(): def get_auth_data(instance): if "community_string" in instance: return cmdgen.CommunityData(instance['community_string']) + elif "user" in instance: + user = instance["user"] + authKey = None + privKey = None + authProtocol = None + privProtocol = None + if "authKey" in instance: + authKey = instance["authKey"] + authProtocol = cmdgen.usmHMACMD5AuthProtocol + if "privKey" in instance: + privKey = instance["privKey"] + authProtocol = cmdgen.usmHMACMD5AuthProtocol + privProtocol = cmdgen.usmDESPrivProtocol + if "authProtocol" in instance: + authProtocol = getattr(cmdgen,instance["authProtocol"]) + if "privProtocol" in instance: + privProtocol = getattr(cmdgen,instance["privProtocol"]) + return cmdgen.UsmUserData(user, authKey, privKey, authProtocol, privProtocol) else: raise Exception("An authentication method needs to be provided") diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index faa4e602ed..4d5b7ccdf1 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -1,9 +1,23 @@ init_config: #instances: +# # SNMP v1-v2 configuration # - ip_address: localhost # port: 1161 # community_string: public # tags: # - optional_tag_1 # - optional_tag_2 +# +# # SNMP v3 configuration +# # check http://pysnmp.sourceforge.net/docs/current/security-configuration.html +# - ip_address: localhost +# port: 1161 +# user: user +# authKey: password +# privKey: private_key +# authProtocol: authProtocol +# privProtocol: privProtocol +# tags: +# - optional_tag_1 +# - optional_tag_2 From b6132760225b9aa3548ef2e504c896b8226556aa Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 30 Apr 2014 10:14:52 -0400 Subject: [PATCH 06/30] SNMP| Allows to add more metrics --- checks.d/snmp.py | 38 ++++++++++++++++++++------------------ conf.d/snmp.yaml.example | 20 ++++++++++++++++---- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index d9e4341881..f3ccd20778 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -7,26 +7,25 @@ snmp_counters = [snmp_type.Counter32, snmp_type.Counter64] snmp_gauges = [snmp_type.Gauge32] -interface_oids = [ - ("IF-MIB", "ifInOctets"), - ("IF-MIB", "ifInErrors"), - ("IF-MIB", "ifInDiscards"), - ("IF-MIB", "ifOutOctets"), - ("IF-MIB", "ifOutErrors"), - ("IF-MIB", "ifOutDiscards") - ] - -device_oids = [ - (("UDP-MIB", "udpInDatagrams"),0), - (("TCP-MIB", "tcpCurrEstab"),0), - (("TCP-MIB", "tcpActiveOpens"),0), - (("TCP-MIB", "tcpPassiveOpens"),0) - ] +class SnmpCheck(AgentCheck): + interface_oids = [ + ("IF-MIB", "ifInOctets"), + ("IF-MIB", "ifInErrors"), + ("IF-MIB", "ifInDiscards"), + ("IF-MIB", "ifOutOctets"), + ("IF-MIB", "ifOutErrors"), + ("IF-MIB", "ifOutDiscards") + ] + device_oids = [ + (("UDP-MIB", "udpInDatagrams"),0), + (("TCP-MIB", "tcpCurrEstab"),0), + (("TCP-MIB", "tcpActiveOpens"),0), + (("TCP-MIB", "tcpPassiveOpens"),0) + ] -class SnmpCheck(AgentCheck): def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) @@ -39,6 +38,9 @@ def __init__(self, name, init_config, agentConfig, instances=None): self.interface_list[ip_address] = self.get_interfaces(instance) tags = instance.get("tags",[]) tags.append("device:" + ip_address) + for metric in init_config["metrics"]: + SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) + def get_interfaces(self, instance): @@ -121,12 +123,12 @@ def get_transport_target(instance): def check(self, instance): tags = instance.get("tags",[]) - results = SnmpCheck.snmp_get(instance, device_oids) + results = SnmpCheck.snmp_get(instance, SnmpCheck.device_oids) for oid, value in results: self.report_as_statsd(instance, oid, value, tags=tags) for interface, descr in self.interface_list[instance['ip_address']].items(): - oids = [(oid, interface) for oid in interface_oids] + oids = [(oid, interface) for oid in SnmpCheck.interface_oids] interface_results = SnmpCheck.snmp_get(instance, oids) for oid, value in interface_results: self.report_as_statsd(instance, oid, value, tags = tags + ["interface:"+descr]) diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index 4d5b7ccdf1..ef7e0218e4 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -1,7 +1,17 @@ init_config: -#instances: -# # SNMP v1-v2 configuration + #Add additional metrics to monitor + +# metrics: +# - MIB: MIB-NAME +# symbol: metric-symbol +# index: 0 + + +instances: + + # SNMP v1-v2 configuration + # - ip_address: localhost # port: 1161 # community_string: public @@ -9,8 +19,10 @@ init_config: # - optional_tag_1 # - optional_tag_2 # -# # SNMP v3 configuration -# # check http://pysnmp.sourceforge.net/docs/current/security-configuration.html + + # SNMP v3 configuration + # check http://pysnmp.sourceforge.net/docs/current/security-configuration.html + # - ip_address: localhost # port: 1161 # user: user From 620853b82e221a23c8d227e3131b09f845b04962 Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 30 Apr 2014 10:29:29 -0400 Subject: [PATCH 07/30] SNMP| Fix:Don't use device as a tag --- checks.d/snmp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index f3ccd20778..4060513698 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -37,7 +37,7 @@ def __init__(self, name, init_config, agentConfig, instances=None): self.counter_state[ip_address] = {} self.interface_list[ip_address] = self.get_interfaces(instance) tags = instance.get("tags",[]) - tags.append("device:" + ip_address) + tags.append("snmp_device:" + ip_address) for metric in init_config["metrics"]: SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) From eeac4c6d9d89dfe878757f1c3a80c37a9208a159 Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 30 Apr 2014 11:00:03 -0400 Subject: [PATCH 08/30] SNMP| Takes all the metrics definition to conf.d --- checks.d/snmp.py | 22 ++++------------------ conf.d/snmp.yaml.example | 30 ++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 4060513698..cf7bd477d8 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -9,23 +9,8 @@ class SnmpCheck(AgentCheck): - interface_oids = [ - ("IF-MIB", "ifInOctets"), - ("IF-MIB", "ifInErrors"), - ("IF-MIB", "ifInDiscards"), - ("IF-MIB", "ifOutOctets"), - ("IF-MIB", "ifOutErrors"), - ("IF-MIB", "ifOutDiscards") - ] - - device_oids = [ - (("UDP-MIB", "udpInDatagrams"),0), - (("TCP-MIB", "tcpCurrEstab"),0), - (("TCP-MIB", "tcpActiveOpens"),0), - (("TCP-MIB", "tcpPassiveOpens"),0) - ] - - + interface_oids = [] + device_oids = [] def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) @@ -40,7 +25,8 @@ def __init__(self, name, init_config, agentConfig, instances=None): tags.append("snmp_device:" + ip_address) for metric in init_config["metrics"]: SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) - + for metric in init_config["interface_metrics"]: + SnmpCheck.interface_oids.append((metric["MIB"], metric["symbol"])) def get_interfaces(self, instance): diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index ef7e0218e4..a833afc5e5 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -1,12 +1,26 @@ init_config: - #Add additional metrics to monitor - -# metrics: -# - MIB: MIB-NAME -# symbol: metric-symbol -# index: 0 - + # Device metrics to monitor + metrics: + - MIB: UDP-MIB + symbol: udpInDatagrams + index: 0 + - MIB: TCP-MIB + symbol: tcpCurrEstab + index: 0 + - MIB: TCP-MIB + symbol: tcpActiveOpens + index: 0 + - MIB: TCP-MIB + symbol: tcpPassiveOpens + index: 0 + + # Metrics to monitor on each interface + interface_metrics: + - MIB: IF-MIB + symbol: ifInOctets + - MIB: IF-MIB + symbol: ifOutOctets instances: @@ -24,7 +38,7 @@ instances: # check http://pysnmp.sourceforge.net/docs/current/security-configuration.html # - ip_address: localhost -# port: 1161 +# port: 161 # default value # user: user # authKey: password # privKey: private_key From 022497019e816dd052797fbeaebca09309bbe20d Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 30 Apr 2014 15:20:11 -0400 Subject: [PATCH 09/30] SNMP| Raise Exception on failed calls remove debug logging --- checks.d/snmp.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index cf7bd477d8..00f8634d53 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -48,7 +48,6 @@ def get_interfaces_nb(): interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) - self.log.info(interfaces_description) for i in range(interface_nb): # order is guaranteed descr = str(interfaces_description.pop(0)[1]) @@ -141,10 +140,10 @@ def snmp_get(instance, oids): ) if errorIndication: - self.log.warning(errorIndication) + raise Exception(errorIndication) else: if errorStatus: - self.log.warning(errorStatus.prettyPrint()) + raise Exception(errorStatus.prettyPrint()) else: return varBinds From b5345288f9bd6329c68da7f42bc215cb4a07c395 Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 30 Apr 2014 18:44:28 -0400 Subject: [PATCH 10/30] SNMP| Allow for partial configuration --- checks.d/snmp.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 00f8634d53..6af55a25ea 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -16,17 +16,20 @@ def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) self.counter_state = {} self.interface_list = {} - for instance in instances: - if 'ip_address' in instance: - ip_address = instance["ip_address"] - self.counter_state[ip_address] = {} - self.interface_list[ip_address] = self.get_interfaces(instance) - tags = instance.get("tags",[]) - tags.append("snmp_device:" + ip_address) - for metric in init_config["metrics"]: - SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) - for metric in init_config["interface_metrics"]: - SnmpCheck.interface_oids.append((metric["MIB"], metric["symbol"])) + if instances is not None: + for instance in instances: + if 'ip_address' in instance: + ip_address = instance["ip_address"] + self.counter_state[ip_address] = {} + self.interface_list[ip_address] = self.get_interfaces(instance) + tags = instance.get("tags",[]) + tags.append("snmp_device:" + ip_address) + if "metric" in init_config: + for metric in init_config["metrics"]: + SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) + if "interface_metrics" in init_config: + for metric in init_config["interface_metrics"]: + SnmpCheck.interface_oids.append((metric["MIB"], metric["symbol"])) def get_interfaces(self, instance): From 8e6f03cd6b41f872dc32a8281015dba178bed4c0 Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 30 Apr 2014 17:59:48 -0400 Subject: [PATCH 11/30] SNMP| First pass at testing Setting up environment for Travis.ci and basic test --- .travis.yml | 4 ++++ requirements.txt | 2 ++ tests/snmp/snmpd.conf | 1 + tests/test_snmp.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+) create mode 100644 tests/snmp/snmpd.conf create mode 100644 tests/test_snmp.py diff --git a/.travis.yml b/.travis.yml index 6e0acd4826..52d7dd0d19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ before_install: - sudo apt-get install couchdb - sudo apt-get install lighttpd - sudo apt-get install gearman + - sudo apt-get install snmpd install: - pip install -r requirements.txt --use-mirrors - pip install . --use-mirrors @@ -47,11 +48,14 @@ before_script: - sudo mkdir -p /etc/dd-agent/ - sudo install -d -o "$(id -u)" /var/log/datadog - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/master/datadog.conf.example > /etc/dd-agent/datadog.conf" + - sudo service snmpd stop + - sudo bash -c "curl -L https://raw.github.com/DataDog/dd-agent/SNMP_checks/tests/snmp/snmpd.conf > /etc/snmp/snmpd.conf" # referring wrong branch during development - sudo service apache2 start - sudo service nginx start - sudo /etc/init.d/lighttpd start - sudo service tomcat6 restart - sudo service couchdb start + - sudo service snmpd start env: - DB=redis script: diff --git a/requirements.txt b/requirements.txt index dbc3d66896..1444902900 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ psutil gearman pylint boto +pysnmp +pysnmp-mibs diff --git a/tests/snmp/snmpd.conf b/tests/snmp/snmpd.conf new file mode 100644 index 0000000000..fab5fa3cfe --- /dev/null +++ b/tests/snmp/snmpd.conf @@ -0,0 +1 @@ +rocommunity public diff --git a/tests/test_snmp.py b/tests/test_snmp.py new file mode 100644 index 0000000000..886153e163 --- /dev/null +++ b/tests/test_snmp.py @@ -0,0 +1,34 @@ +import unittest +from tests.common import load_check + + +class TestSNMP(unittest.TestCase): + + def setUp(self): + self.agentConfig = { + 'version': '0.1', + 'api_key': 'toto' + } + + self.config = { + "init_config": { + "metrics": [{ + "MIB": "UDP-MIB", + "symbol": "udpInDatagrams", + "index": "0" + }] + }, + "instances": [{ + "ip_address": "localhost", + "community_string": "public" + }] + } + + def testInit(self): + # Initialize the check from checks.d + self.check = load_check('snmp', self.config, self.agentConfig) + self.assertGreater(self.check.counter_state.keys(), 0) + self.assertGreater(self.check.interface_list.keys(), 0) + +if __name__ == "__main__": + unittest.main() From 998269b1deee24e1f048bab9f12f30177929f186 Mon Sep 17 00:00:00 2001 From: Rudy Date: Thu, 1 May 2014 09:19:48 -0400 Subject: [PATCH 12/30] SNMP| Fix test assertGreater not available on 2.6 --- tests/test_snmp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_snmp.py b/tests/test_snmp.py index 886153e163..e6e41671a6 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -27,8 +27,8 @@ def setUp(self): def testInit(self): # Initialize the check from checks.d self.check = load_check('snmp', self.config, self.agentConfig) - self.assertGreater(self.check.counter_state.keys(), 0) - self.assertGreater(self.check.interface_list.keys(), 0) + self.assertTrue(self.check.counter_state.keys() > 0) + self.assertTrue(self.check.interface_list.keys() > 0) if __name__ == "__main__": unittest.main() From 35c00151b3299cf7b5471ad22212954fd776c957 Mon Sep 17 00:00:00 2001 From: Rudy Date: Thu, 1 May 2014 18:06:33 -0400 Subject: [PATCH 13/30] SNMP| Add a test performing the checks Also fix a bug detected by a test --- checks.d/snmp.py | 2 +- tests/test_snmp.py | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 6af55a25ea..2254ca70e0 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -24,7 +24,7 @@ def __init__(self, name, init_config, agentConfig, instances=None): self.interface_list[ip_address] = self.get_interfaces(instance) tags = instance.get("tags",[]) tags.append("snmp_device:" + ip_address) - if "metric" in init_config: + if "metrics" in init_config: for metric in init_config["metrics"]: SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) if "interface_metrics" in init_config: diff --git a/tests/test_snmp.py b/tests/test_snmp.py index e6e41671a6..0071037133 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -27,8 +27,27 @@ def setUp(self): def testInit(self): # Initialize the check from checks.d self.check = load_check('snmp', self.config, self.agentConfig) - self.assertTrue(self.check.counter_state.keys() > 0) - self.assertTrue(self.check.interface_list.keys() > 0) + # Assert the counter state dictionary was initialized + self.assertTrue(len(self.check.counter_state.keys()) > 0) + # Assert that some interface got detected on the host + self.assertTrue(len(self.check.interface_list["localhost"]) > 0) + # Assert that the device-level metrics is accessible + for item in dir(self.check): + print item, getattr(self.check,item) + self.assertEqual(len(self.check.device_oids), 1) + + + def testSNMPCheck(self): + + self.check = load_check('snmp', self.config, self.agentConfig) + + self.check.check(self.config['instances'][0]) + + # Metric assertions + metrics = self.check.get_metrics() + assert metrics + self.assertTrue(type(metrics) == type([])) + self.assertTrue(len(metrics) > 0) if __name__ == "__main__": unittest.main() From ccfa0f988645cf3d996847f51bd06d29444a6910 Mon Sep 17 00:00:00 2001 From: Rudy Date: Thu, 1 May 2014 18:38:27 -0400 Subject: [PATCH 14/30] Cln| Remove spurious print --- tests/test_snmp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_snmp.py b/tests/test_snmp.py index 0071037133..9384af752f 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -32,8 +32,6 @@ def testInit(self): # Assert that some interface got detected on the host self.assertTrue(len(self.check.interface_list["localhost"]) > 0) # Assert that the device-level metrics is accessible - for item in dir(self.check): - print item, getattr(self.check,item) self.assertEqual(len(self.check.device_oids), 1) From ab05bc466d48051e0f08e40c0ac5d09edfda7af2 Mon Sep 17 00:00:00 2001 From: Rudy Date: Thu, 1 May 2014 18:39:15 -0400 Subject: [PATCH 15/30] SNMP| Better tests for the check --- checks.d/snmp.py | 5 ++--- tests/test_snmp.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 2254ca70e0..836190c78a 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -9,9 +9,6 @@ class SnmpCheck(AgentCheck): - interface_oids = [] - device_oids = [] - def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) self.counter_state = {} @@ -24,6 +21,8 @@ def __init__(self, name, init_config, agentConfig, instances=None): self.interface_list[ip_address] = self.get_interfaces(instance) tags = instance.get("tags",[]) tags.append("snmp_device:" + ip_address) + SnmpCheck.interface_oids = [] + SnmpCheck.device_oids = [] if "metrics" in init_config: for metric in init_config["metrics"]: SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) diff --git a/tests/test_snmp.py b/tests/test_snmp.py index 9384af752f..da4f864459 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -1,4 +1,5 @@ import unittest +import time from tests.common import load_check @@ -16,6 +17,10 @@ def setUp(self): "MIB": "UDP-MIB", "symbol": "udpInDatagrams", "index": "0" + },{ + "MIB": "TCP-MIB", + "symbol": "tcpCurrEstab", + "index":"0" }] }, "instances": [{ @@ -32,7 +37,7 @@ def testInit(self): # Assert that some interface got detected on the host self.assertTrue(len(self.check.interface_list["localhost"]) > 0) # Assert that the device-level metrics is accessible - self.assertEqual(len(self.check.device_oids), 1) + self.assertEqual(len(self.check.device_oids), 2) def testSNMPCheck(self): @@ -40,12 +45,28 @@ def testSNMPCheck(self): self.check = load_check('snmp', self.config, self.agentConfig) self.check.check(self.config['instances'][0]) + metrics = self.check.get_metrics() + + # Assert that there is only the gauge metric because the counter is used + # as a rate so we don't report with 1 point + self.assertEqual(len(metrics), 1) + self.assertEqual(metrics[0][0], 'snmp.tcpCurrEstab') - # Metric assertions + # Sleep for 1 second so the rate interval >=1 + time.sleep(1) + # Run the check again so we get the rate + self.check.check(self.config['instances'][0]) metrics = self.check.get_metrics() - assert metrics - self.assertTrue(type(metrics) == type([])) - self.assertTrue(len(metrics) > 0) + + self.assertEqual(len(metrics) ,2) + expected_metrics = ['snmp.udpInDatagrams','snmp.tcpCurrEstab'] + for metric in expected_metrics: + metric_present=False + for result in metrics: + if result[0] == metric: + metric_present = True + break + self.assertTrue(metric_present) if __name__ == "__main__": unittest.main() From 869c63c567c8356bc628cff66975b09e2f8a2dd5 Mon Sep 17 00:00:00 2001 From: Rudy Date: Mon, 5 May 2014 13:51:43 -0400 Subject: [PATCH 16/30] SNMP| Metrics defined at the instance level --- checks.d/snmp.py | 23 ++++++++--------- conf.d/snmp.yaml.example | 55 ++++++++++++++++++++++------------------ tests/test_snmp.py | 13 ++++------ 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 836190c78a..f841c0b706 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -1,5 +1,7 @@ from checks import AgentCheck +from collections import defaultdict + from pysnmp.entity.rfc3413.oneliner import cmdgen from pysnmp.smi.exval import noSuchInstance import pysnmp.proto.rfc1902 as snmp_type @@ -11,24 +13,15 @@ class SnmpCheck(AgentCheck): def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) - self.counter_state = {} + self.counter_state = defaultdict(dict) self.interface_list = {} if instances is not None: for instance in instances: if 'ip_address' in instance: ip_address = instance["ip_address"] - self.counter_state[ip_address] = {} self.interface_list[ip_address] = self.get_interfaces(instance) tags = instance.get("tags",[]) tags.append("snmp_device:" + ip_address) - SnmpCheck.interface_oids = [] - SnmpCheck.device_oids = [] - if "metrics" in init_config: - for metric in init_config["metrics"]: - SnmpCheck.device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) - if "interface_metrics" in init_config: - for metric in init_config["interface_metrics"]: - SnmpCheck.interface_oids.append((metric["MIB"], metric["symbol"])) def get_interfaces(self, instance): @@ -110,12 +103,18 @@ def get_transport_target(instance): def check(self, instance): tags = instance.get("tags",[]) - results = SnmpCheck.snmp_get(instance, SnmpCheck.device_oids) + device_oids = [] + interface_oids = [] + for metric in instance.get('metrics',[]): + device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) + results = SnmpCheck.snmp_get(instance, device_oids) for oid, value in results: self.report_as_statsd(instance, oid, value, tags=tags) + for metric in instance.get('interface_metrics',[]): + interface_oids.append((metric["MIB"],metric["symbol"])) for interface, descr in self.interface_list[instance['ip_address']].items(): - oids = [(oid, interface) for oid in SnmpCheck.interface_oids] + oids = [(oid, interface) for oid in interface_oids] interface_results = SnmpCheck.snmp_get(instance, oids) for oid, value in interface_results: self.report_as_statsd(instance, oid, value, tags = tags + ["interface:"+descr]) diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index a833afc5e5..772bf677b2 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -1,38 +1,38 @@ init_config: - # Device metrics to monitor - metrics: - - MIB: UDP-MIB - symbol: udpInDatagrams - index: 0 - - MIB: TCP-MIB - symbol: tcpCurrEstab - index: 0 - - MIB: TCP-MIB - symbol: tcpActiveOpens - index: 0 - - MIB: TCP-MIB - symbol: tcpPassiveOpens - index: 0 - - # Metrics to monitor on each interface - interface_metrics: - - MIB: IF-MIB - symbol: ifInOctets - - MIB: IF-MIB - symbol: ifOutOctets instances: # SNMP v1-v2 configuration # - ip_address: localhost -# port: 1161 +# port: 161 # community_string: public # tags: -# - optional_tag_1 -# - optional_tag_2 +# - optional_tag_1 +# - optional_tag_2 # +# # Device metrics to monitor +# metrics: +# - MIB: UDP-MIB +# symbol: udpInDatagrams +# index: 0 +# - MIB: TCP-MIB +# symbol: tcpCurrEstab +# index: 0 +# - MIB: TCP-MIB +# symbol: tcpActiveOpens +# index: 0 +# - MIB: TCP-MIB +# symbol: tcpPassiveOpens +# index: 0 +# +# # Metrics to monitor on each interface +# interface_metrics: +# - MIB: IF-MIB +# symbol: ifInOctets +# - MIB: IF-MIB +# symbol: ifOutOctets # SNMP v3 configuration # check http://pysnmp.sourceforge.net/docs/current/security-configuration.html @@ -47,3 +47,10 @@ instances: # tags: # - optional_tag_1 # - optional_tag_2 +# +# # Metrics to monitor on each interface +# interface_metrics: +# - MIB: IF-MIB +# symbol: ifInOctets +# - MIB: IF-MIB +# symbol: ifOutOctets diff --git a/tests/test_snmp.py b/tests/test_snmp.py index da4f864459..836349ad08 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -13,6 +13,11 @@ def setUp(self): self.config = { "init_config": { + }, + "instances": [{ + "ip_address": "localhost", + "port":161, + "community_string": "public", "metrics": [{ "MIB": "UDP-MIB", "symbol": "udpInDatagrams", @@ -22,22 +27,14 @@ def setUp(self): "symbol": "tcpCurrEstab", "index":"0" }] - }, - "instances": [{ - "ip_address": "localhost", - "community_string": "public" }] } def testInit(self): # Initialize the check from checks.d self.check = load_check('snmp', self.config, self.agentConfig) - # Assert the counter state dictionary was initialized - self.assertTrue(len(self.check.counter_state.keys()) > 0) # Assert that some interface got detected on the host self.assertTrue(len(self.check.interface_list["localhost"]) > 0) - # Assert that the device-level metrics is accessible - self.assertEqual(len(self.check.device_oids), 2) def testSNMPCheck(self): From 80ddd73b314e6f3345237be53f413e046e7b0e25 Mon Sep 17 00:00:00 2001 From: Rudy Date: Mon, 5 May 2014 14:10:13 -0400 Subject: [PATCH 17/30] SNMP| Added docstring --- checks.d/snmp.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index f841c0b706..816debea82 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -24,6 +24,14 @@ def __init__(self, name, init_config, agentConfig, instances=None): tags.append("snmp_device:" + ip_address) def get_interfaces(self, instance): + ''' + Return all the network interfaces of an instance to be used to get metrics + on those interfaces. + If available, get the number of interfaces from the ifNumber MIB + and then query all of them in one request for their description. + If this info is not available, repeatedly query the interface description + in order to discover them all. + ''' interface_list = {} @@ -47,9 +55,11 @@ def get_interfaces_nb(): # order is guaranteed descr = str(interfaces_description.pop(0)[1]) type = int(interfaces_description.pop(0)[1]) - if type != 24: + if type != 24: # ignore localhost loopback interface_list[i+1] = descr else: + # ifNumber is not available, query the interfaces dor description + # until blank one empty_reply = False interface_index = 1 while not empty_reply: @@ -62,7 +72,7 @@ def get_interfaces_nb(): empty_reply= True else: type = int(interfaces_description.pop(0)[1]) - if type != 24: + if type != 24: # ignore localhost loopback interface_list[interface_index] = str(descr) interface_index += 1 @@ -71,8 +81,10 @@ def get_interfaces_nb(): @staticmethod def get_auth_data(instance): if "community_string" in instance: + # SNMP v1 - SNMP v2 return cmdgen.CommunityData(instance['community_string']) elif "user" in instance: + # SNMP v3 user = instance["user"] authKey = None privKey = None @@ -98,7 +110,7 @@ def get_transport_target(instance): if "ip_address" not in instance: raise Exception("An IP address needs to be specified") ip_address = instance["ip_address"] - port = instance.get("port", 161) + port = instance.get("port", 161) # Default SNMP port return cmdgen.UdpTransportTarget((ip_address,port)) def check(self, instance): From c77d416631858e45ea628eef82c094103de7cf90 Mon Sep 17 00:00:00 2001 From: Rudy Date: Mon, 19 May 2014 17:20:22 -0400 Subject: [PATCH 18/30] SNMP| Use the agent's aggregator's rate for SNMP counters Also make it more clear that we add the name of the SNMP_device to the tags --- checks.d/snmp.py | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 816debea82..6770175883 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -6,8 +6,8 @@ from pysnmp.smi.exval import noSuchInstance import pysnmp.proto.rfc1902 as snmp_type -snmp_counters = [snmp_type.Counter32, snmp_type.Counter64] -snmp_gauges = [snmp_type.Gauge32] +SNMP_COUNTERS = [snmp_type.Counter32, snmp_type.Counter64] +SNMP_GAUGES = [snmp_type.Gauge32] class SnmpCheck(AgentCheck): @@ -20,8 +20,6 @@ def __init__(self, name, init_config, agentConfig, instances=None): if 'ip_address' in instance: ip_address = instance["ip_address"] self.interface_list[ip_address] = self.get_interfaces(instance) - tags = instance.get("tags",[]) - tags.append("snmp_device:" + ip_address) def get_interfaces(self, instance): ''' @@ -115,13 +113,14 @@ def get_transport_target(instance): def check(self, instance): tags = instance.get("tags",[]) + ip_address = instance["ip_address"] device_oids = [] interface_oids = [] for metric in instance.get('metrics',[]): device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) results = SnmpCheck.snmp_get(instance, device_oids) for oid, value in results: - self.report_as_statsd(instance, oid, value, tags=tags) + self.submit_metric(instance, oid, value, tags=tags + ["snmp_device:" + ip_address]) for metric in instance.get('interface_metrics',[]): interface_oids.append((metric["MIB"],metric["symbol"])) @@ -129,7 +128,8 @@ def check(self, instance): oids = [(oid, interface) for oid in interface_oids] interface_results = SnmpCheck.snmp_get(instance, oids) for oid, value in interface_results: - self.report_as_statsd(instance, oid, value, tags = tags + ["interface:"+descr]) + self.submit_metric(instance, oid, value, tags = tags + ["snmp_device:" + ip_address, + "interface:"+descr]) @staticmethod def snmp_get(instance, oids): @@ -160,27 +160,14 @@ def snmp_get(instance, oids): else: return varBinds - def report_as_statsd(self, instance, oid, snmp_value, tags=[]): + def submit_metric(self, instance, oid, snmp_value, tags=[]): if noSuchInstance.isSameTypeWith(snmp_value): return name = "snmp." + oid.getMibSymbol()[1] snmp_class = getattr(snmp_value, '__class__') value = int(snmp_value) - if snmp_class in snmp_counters: - self.counter(instance, name, value, snmp_class, tags) - elif snmp_class in snmp_gauges: + if snmp_class in SNMP_COUNTERS: + self.rate(name, value, tags) + elif snmp_class in SNMP_GAUGES: self.gauge(name, value, tags) - - def counter(self, instance, name, value, snmp_class, tags = []): - current_state = self.counter_state[instance['ip_address']] - metric_id = name + str(tags) - if metric_id in current_state: - diff = value - current_state[metric_id] - if diff < 0: - # Counters monotonically increase so it means the counter wrapped - diff += pow(2, 32 if snmp_class==snmp_type.Counter32 else 64) - self.increment(name, diff,tags=tags) - else: - self.log.info("Setting up initial value for Counter {0}".format(name)) - current_state[metric_id] = value From 85774eb1991b12baf253318af9208a4118af6590 Mon Sep 17 00:00:00 2001 From: Rudy Date: Thu, 22 May 2014 15:14:48 -0400 Subject: [PATCH 19/30] SNMP Checks| Use single way to determine interface and improve logging --- checks.d/snmp.py | 63 +++++++++++++++--------------------------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 6770175883..03b597439b 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -25,55 +25,29 @@ def get_interfaces(self, instance): ''' Return all the network interfaces of an instance to be used to get metrics on those interfaces. - If available, get the number of interfaces from the ifNumber MIB - and then query all of them in one request for their description. - If this info is not available, repeatedly query the interface description + Repeatedly query the interface description in order to discover them all. ''' interface_list = {} - def get_interfaces_nb(): - result = SnmpCheck.snmp_get(instance, [(("IF-MIB","ifNumber"),0)])[0] - if noSuchInstance.isSameTypeWith(result[1]): - return None - else: - return int(result[1]) - - interface_nb = get_interfaces_nb() - if interface_nb is not None: + empty_reply = False + interface_index = 1 + while not empty_reply: interfaces_descr_oids = [] - for interface in range(interface_nb): - interface_index = interface + 1 #SNMP indexes start from 1 - interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) - interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) - + interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) + interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) - for i in range(interface_nb): - # order is guaranteed - descr = str(interfaces_description.pop(0)[1]) - type = int(interfaces_description.pop(0)[1]) - if type != 24: # ignore localhost loopback - interface_list[i+1] = descr - else: - # ifNumber is not available, query the interfaces dor description - # until blank one - empty_reply = False - interface_index = 1 - while not empty_reply: - interfaces_descr_oids = [] - interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) - interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) - interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) - descr = interfaces_description.pop(0)[1] - if noSuchInstance.isSameTypeWith(descr): - empty_reply= True - else: - type = int(interfaces_description.pop(0)[1]) - if type != 24: # ignore localhost loopback - interface_list[interface_index] = str(descr) - interface_index += 1 - + descr = interfaces_description.pop(0)[1] + if noSuchInstance.isSameTypeWith(descr): + empty_reply= True + else: + type = interfaces_description.pop(0)[1] + if not noSuchInstance.isSameTypeWith(type) and int(type) !=24: + # ignore localhost loopback + interface_list[interface_index] = str(descr) + self.log.info("Discovered interface %s" % str(descr)) + interface_index += 1 return interface_list @staticmethod @@ -153,15 +127,16 @@ def snmp_get(instance, oids): ) if errorIndication: - raise Exception(errorIndication) + raise Exception("{0} for instance {1}".format(errorIndication,instance["ip_address"])) else: if errorStatus: - raise Exception(errorStatus.prettyPrint()) + raise Exception("{0} for instance {1}".format(errorStatus.prettyPrint(),instance["ip_address"])) else: return varBinds def submit_metric(self, instance, oid, snmp_value, tags=[]): if noSuchInstance.isSameTypeWith(snmp_value): + self.log.warning("No such Mib available: %s" %oid.getMibSymbol()[1]) return name = "snmp." + oid.getMibSymbol()[1] snmp_class = getattr(snmp_value, '__class__') From 544883a60397c23758abc0249024c7a0205ed250 Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 28 May 2014 11:11:19 -0400 Subject: [PATCH 20/30] SNMP Checks| Allows to specify metric by oid + name --- checks.d/snmp.py | 23 +++++++++++++++++++---- conf.d/snmp.yaml.example | 8 +++++--- tests/test_snmp.py | 7 +++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 03b597439b..7908f18359 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -90,11 +90,19 @@ def check(self, instance): ip_address = instance["ip_address"] device_oids = [] interface_oids = [] + oid_names ={} for metric in instance.get('metrics',[]): - device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) + if 'MIB' in metric: + device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) + elif 'OID' in metric: + device_oids.append(metric['OID']) + oid_names[metric['OID']]=metric['name'] + else: + raise Exception('Unsupported metrics format in config file') results = SnmpCheck.snmp_get(instance, device_oids) for oid, value in results: - self.submit_metric(instance, oid, value, tags=tags + ["snmp_device:" + ip_address]) + self.submit_metric(instance, oid, value, tags=tags + ["snmp_device:" + ip_address], + oid_names = oid_names) for metric in instance.get('interface_metrics',[]): interface_oids.append((metric["MIB"],metric["symbol"])) @@ -134,11 +142,18 @@ def snmp_get(instance, oids): else: return varBinds - def submit_metric(self, instance, oid, snmp_value, tags=[]): + def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): if noSuchInstance.isSameTypeWith(snmp_value): self.log.warning("No such Mib available: %s" %oid.getMibSymbol()[1]) return - name = "snmp." + oid.getMibSymbol()[1] + if str(oid.getOid()) in oid_names: + name = "snmp."+ oid_names[str(oid.getOid())] + else: + try: + name = "snmp." + oid.getMibSymbol()[1] + except: + self.log.warning("Couldn't find a name for oid {0}".format(oid)) + snmp_class = getattr(snmp_value, '__class__') value = int(snmp_value) if snmp_class in SNMP_COUNTERS: diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index 772bf677b2..f977e347e1 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -14,18 +14,20 @@ instances: # # # Device metrics to monitor # metrics: +# # You can specify metrics using MIBS # - MIB: UDP-MIB # symbol: udpInDatagrams # index: 0 # - MIB: TCP-MIB -# symbol: tcpCurrEstab -# index: 0 -# - MIB: TCP-MIB # symbol: tcpActiveOpens # index: 0 # - MIB: TCP-MIB # symbol: tcpPassiveOpens # index: 0 +# # You can also specify an oid and the name under which it should appear +# # in Datadog +# - OID: 1.3.6.1.2.1.6.9.0 +# name: tcpConnections # # # Metrics to monitor on each interface # interface_metrics: diff --git a/tests/test_snmp.py b/tests/test_snmp.py index 836349ad08..96d52b714b 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -19,9 +19,8 @@ def setUp(self): "port":161, "community_string": "public", "metrics": [{ - "MIB": "UDP-MIB", - "symbol": "udpInDatagrams", - "index": "0" + "OID": "1.3.6.1.2.1.7.1.0", + "name": "udpDatagrams" },{ "MIB": "TCP-MIB", "symbol": "tcpCurrEstab", @@ -56,7 +55,7 @@ def testSNMPCheck(self): metrics = self.check.get_metrics() self.assertEqual(len(metrics) ,2) - expected_metrics = ['snmp.udpInDatagrams','snmp.tcpCurrEstab'] + expected_metrics = ['snmp.udpDatagrams','snmp.tcpCurrEstab'] for metric in expected_metrics: metric_present=False for result in metrics: From 5ad472b1e1afdc42ac5fdf0e6b25e250304cea7a Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 28 May 2014 16:14:49 -0400 Subject: [PATCH 21/30] SNMP Checks| Allow to use custom mibs if they are in the pysnmp format --- checks.d/snmp.py | 37 +++++++++++++++++++++++++++---------- conf.d/snmp.yaml.example | 3 ++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 7908f18359..6b3fa81ef5 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -4,6 +4,7 @@ from pysnmp.entity.rfc3413.oneliner import cmdgen from pysnmp.smi.exval import noSuchInstance +from pysnmp.smi import builder import pysnmp.proto.rfc1902 as snmp_type SNMP_COUNTERS = [snmp_type.Counter32, snmp_type.Counter64] @@ -11,16 +12,33 @@ class SnmpCheck(AgentCheck): + cmd_generator = None + def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) self.counter_state = defaultdict(dict) self.interface_list = {} + mibs_path = None + if init_config is not None: + mibs_path = init_config.get("mibs_folder") + SnmpCheck.create_command_generator(mibs_path) if instances is not None: for instance in instances: if 'ip_address' in instance: ip_address = instance["ip_address"] self.interface_list[ip_address] = self.get_interfaces(instance) + @classmethod + def create_command_generator(cls, mibs_path=None): + cls.cmd_generator = cmdgen.CommandGenerator() + if mibs_path is not None: + mibBuilder = cls.cmd_generator.snmpEngine.msgAndPduDsp.\ + mibInstrumController.mibBuilder + mibSources = mibBuilder.getMibSources() + ( + builder.DirMibSource(mibs_path), + ) + mibBuilder.setMibSources(*mibSources) + def get_interfaces(self, instance): ''' Return all the network interfaces of an instance to be used to get metrics @@ -50,8 +68,8 @@ def get_interfaces(self, instance): interface_index += 1 return interface_list - @staticmethod - def get_auth_data(instance): + @classmethod + def get_auth_data(cls, instance): if "community_string" in instance: # SNMP v1 - SNMP v2 return cmdgen.CommunityData(instance['community_string']) @@ -77,8 +95,8 @@ def get_auth_data(instance): else: raise Exception("An authentication method needs to be provided") - @staticmethod - def get_transport_target(instance): + @classmethod + def get_transport_target(cls, instance): if "ip_address" not in instance: raise Exception("An IP address needs to be specified") ip_address = instance["ip_address"] @@ -113,19 +131,18 @@ def check(self, instance): self.submit_metric(instance, oid, value, tags = tags + ["snmp_device:" + ip_address, "interface:"+descr]) - @staticmethod - def snmp_get(instance, oids): + @classmethod + def snmp_get(cls, instance, oids): """ Perform a snmp get command to the device that instance describe and return a list of tuble (name, values) corresponding to the elements in oids. """ - transport_target = SnmpCheck.get_transport_target(instance) - auth_data = SnmpCheck.get_auth_data(instance) + transport_target = cls.get_transport_target(instance) + auth_data = cls.get_auth_data(instance) - cmd_generator = cmdgen.CommandGenerator() - snmp_command = cmd_generator.getCmd + snmp_command = cls.cmd_generator.getCmd errorIndication, errorStatus, errorIndex, varBinds = snmp_command( auth_data, transport_target, diff --git a/conf.d/snmp.yaml.example b/conf.d/snmp.yaml.example index f977e347e1..eb6ccd9d7f 100644 --- a/conf.d/snmp.yaml.example +++ b/conf.d/snmp.yaml.example @@ -1,5 +1,6 @@ init_config: - +# #You can specify an additional folder for your custom mib files (python format) +# mibs_folder: /path/to/your/mibs/folder instances: From 52efcb6567b74f9183cb2e9f46adec2f9d49f3a6 Mon Sep 17 00:00:00 2001 From: Rudy Date: Wed, 28 May 2014 16:49:58 -0400 Subject: [PATCH 22/30] SNMP Checks | Test for custom mib folder --- tests/test_snmp.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_snmp.py b/tests/test_snmp.py index 96d52b714b..9911c6155f 100644 --- a/tests/test_snmp.py +++ b/tests/test_snmp.py @@ -13,6 +13,7 @@ def setUp(self): self.config = { "init_config": { + 'mibs_folder':'/etc/mibs' }, "instances": [{ "ip_address": "localhost", @@ -35,6 +36,14 @@ def testInit(self): # Assert that some interface got detected on the host self.assertTrue(len(self.check.interface_list["localhost"]) > 0) + mib_folders = self.check.cmd_generator.snmpEngine.msgAndPduDsp\ + .mibInstrumController.mibBuilder.getMibSources() + custom_folder_represented = False + for folder in mib_folders: + if '/etc/mibs' == folder.fullPath(): + custom_folder_represented = True + break + self.assertTrue(custom_folder_represented) def testSNMPCheck(self): From d8fda4948b2485717c1f572703cb65dfe0b63ca9 Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 30 May 2014 14:56:37 -0400 Subject: [PATCH 23/30] SNMP Checks| Handle no such object cases --- checks.d/snmp.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 6b3fa81ef5..cdf7c45c1c 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -3,13 +3,17 @@ from collections import defaultdict from pysnmp.entity.rfc3413.oneliner import cmdgen -from pysnmp.smi.exval import noSuchInstance +from pysnmp.smi.exval import noSuchInstance, noSuchObject from pysnmp.smi import builder import pysnmp.proto.rfc1902 as snmp_type SNMP_COUNTERS = [snmp_type.Counter32, snmp_type.Counter64] SNMP_GAUGES = [snmp_type.Gauge32] +def reply_invalid(oid): + return noSuchInstance.isSameTypeWith(oid) or \ + noSuchObject.isSameTypeWith(oid) + class SnmpCheck(AgentCheck): cmd_generator = None @@ -57,15 +61,13 @@ def get_interfaces(self, instance): interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) descr = interfaces_description.pop(0)[1] - if noSuchInstance.isSameTypeWith(descr): - empty_reply= True - else: - type = interfaces_description.pop(0)[1] - if not noSuchInstance.isSameTypeWith(type) and int(type) !=24: - # ignore localhost loopback - interface_list[interface_index] = str(descr) - self.log.info("Discovered interface %s" % str(descr)) - interface_index += 1 + empty_reply= reply_invalid(descr) + type = interfaces_description.pop(0)[1] + if not reply_invalid(type) and int(type) !=24: + # ignore localhost loopback + interface_list[interface_index] = str(descr) + self.log.info("Discovered interface %s" % str(descr)) + interface_index += 1 return interface_list @classmethod @@ -160,7 +162,7 @@ def snmp_get(cls, instance, oids): return varBinds def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): - if noSuchInstance.isSameTypeWith(snmp_value): + if reply_invalid(snmp_value): self.log.warning("No such Mib available: %s" %oid.getMibSymbol()[1]) return if str(oid.getOid()) in oid_names: From d51374489cd440b08082f549e485b21826df4eb7 Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 30 May 2014 15:18:55 -0400 Subject: [PATCH 24/30] SNMP Checks| Support type extension and add debug logging --- checks.d/snmp.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index cdf7c45c1c..96000ca601 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -7,8 +7,11 @@ from pysnmp.smi import builder import pysnmp.proto.rfc1902 as snmp_type -SNMP_COUNTERS = [snmp_type.Counter32, snmp_type.Counter64] -SNMP_GAUGES = [snmp_type.Gauge32] +convention_type_builder = builder.MibBuilder() +(CounterBasedGauge64, ZeroBasedCounter64) = convention_type_builder.importSymbols("HCNUM-TC","CounterBasedGauge64", "ZeroBasedCounter64") + +SNMP_COUNTERS = [snmp_type.Counter32, snmp_type.Counter64, ZeroBasedCounter64] +SNMP_GAUGES = [snmp_type.Gauge32, CounterBasedGauge64] def reply_invalid(oid): return noSuchInstance.isSameTypeWith(oid) or \ @@ -119,6 +122,7 @@ def check(self, instance): oid_names[metric['OID']]=metric['name'] else: raise Exception('Unsupported metrics format in config file') + self.log.debug("Querying device %s for %s oids",ip_address, len(device_oids)) results = SnmpCheck.snmp_get(instance, device_oids) for oid, value in results: self.submit_metric(instance, oid, value, tags=tags + ["snmp_device:" + ip_address], @@ -128,6 +132,7 @@ def check(self, instance): interface_oids.append((metric["MIB"],metric["symbol"])) for interface, descr in self.interface_list[instance['ip_address']].items(): oids = [(oid, interface) for oid in interface_oids] + self.log.debug("Querying device %s for %s oids",ip_address, len(oids)) interface_results = SnmpCheck.snmp_get(instance, oids) for oid, value in interface_results: self.submit_metric(instance, oid, value, tags = tags + ["snmp_device:" + ip_address, @@ -172,6 +177,7 @@ def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): name = "snmp." + oid.getMibSymbol()[1] except: self.log.warning("Couldn't find a name for oid {0}".format(oid)) + return snmp_class = getattr(snmp_value, '__class__') value = int(snmp_value) @@ -179,4 +185,6 @@ def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): self.rate(name, value, tags) elif snmp_class in SNMP_GAUGES: self.gauge(name, value, tags) + else: + self.log.warning("Unsupported metric type %s", snmp_class) From b1f5437333d77045a59d067b7727470e59c7e41d Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 30 May 2014 19:04:07 -0400 Subject: [PATCH 25/30] SNMP Checks| Extended type fix Dynamically loaded class comparison doesn't work Accessing python's hidden attributes is freaking ugly but the clean way (ASN1 isSameTypeWith) doesn't work because CounterBasedGauge64 (a gauge) match with both ZeroBasedCounter64 and Counter64 ( counters ), given that these classes are just passthrough see sources at http://nullege.com/codes/show/src@pysnmp-mibs-0.0.8a@pysnmp_mibs@HCNUM-TC.py --- checks.d/snmp.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 96000ca601..4207f059d0 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -10,8 +10,8 @@ convention_type_builder = builder.MibBuilder() (CounterBasedGauge64, ZeroBasedCounter64) = convention_type_builder.importSymbols("HCNUM-TC","CounterBasedGauge64", "ZeroBasedCounter64") -SNMP_COUNTERS = [snmp_type.Counter32, snmp_type.Counter64, ZeroBasedCounter64] -SNMP_GAUGES = [snmp_type.Gauge32, CounterBasedGauge64] +SNMP_COUNTERS = [snmp_type.Counter32.__name__, snmp_type.Counter64.__name__, ZeroBasedCounter64.__name__] +SNMP_GAUGES = [snmp_type.Gauge32.__name__, CounterBasedGauge64.__name__] def reply_invalid(oid): return noSuchInstance.isSameTypeWith(oid) or \ @@ -179,12 +179,15 @@ def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): self.log.warning("Couldn't find a name for oid {0}".format(oid)) return - snmp_class = getattr(snmp_value, '__class__') + snmp_class = snmp_value.__class__.__name__ value = int(snmp_value) - if snmp_class in SNMP_COUNTERS: - self.rate(name, value, tags) - elif snmp_class in SNMP_GAUGES: - self.gauge(name, value, tags) - else: - self.log.warning("Unsupported metric type %s", snmp_class) + for counter_class in SNMP_COUNTERS: + if snmp_class==counter_class: + self.rate(name, value, tags) + return + for gauge_class in SNMP_GAUGES: + if snmp_class==gauge_class: + self.gauge(name, value, tags) + return + self.log.warning("Unsupported metric type %s", snmp_class) From 4259c5e92865e4b5c78fb958763f3d1723a40c41 Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 13 Jun 2014 13:51:17 -0400 Subject: [PATCH 26/30] SNMP checks| Remove bare except Also don't convert to an int until it's sure --- checks.d/snmp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 4207f059d0..ef6aa90bfb 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -175,18 +175,19 @@ def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): else: try: name = "snmp." + oid.getMibSymbol()[1] - except: + except Exception: self.log.warning("Couldn't find a name for oid {0}".format(oid)) return snmp_class = snmp_value.__class__.__name__ - value = int(snmp_value) for counter_class in SNMP_COUNTERS: if snmp_class==counter_class: + value = int(snmp_value) self.rate(name, value, tags) return for gauge_class in SNMP_GAUGES: if snmp_class==gauge_class: + value = int(snmp_value) self.gauge(name, value, tags) return self.log.warning("Unsupported metric type %s", snmp_class) From e9675d68f77d99a223bfffeb47815ee520329bf9 Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 13 Jun 2014 15:21:07 -0400 Subject: [PATCH 27/30] SNMP Checks | Better interface discovery --- checks.d/snmp.py | 60 ++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index ef6aa90bfb..bd679a58ae 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -13,6 +13,14 @@ SNMP_COUNTERS = [snmp_type.Counter32.__name__, snmp_type.Counter64.__name__, ZeroBasedCounter64.__name__] SNMP_GAUGES = [snmp_type.Gauge32.__name__, CounterBasedGauge64.__name__] +# IF-MIB magic values +IF_TABLE_TYPE_POS = 9 +IF_TABLE_INDEX_POS = 10 +IF_INDEX = 1 +IF_DESCR = 2 +IF_TYPE = 3 +LOCALHOST_INTERFACE = 24 + def reply_invalid(oid): return noSuchInstance.isSameTypeWith(oid) or \ noSuchObject.isSameTypeWith(oid) @@ -47,30 +55,38 @@ def create_command_generator(cls, mibs_path=None): mibBuilder.setMibSources(*mibSources) def get_interfaces(self, instance): - ''' - Return all the network interfaces of an instance to be used to get metrics - on those interfaces. - Repeatedly query the interface description - in order to discover them all. - ''' - interface_list = {} + transport_target = self.get_transport_target(instance) + auth_data = self.get_auth_data(instance) + + snmp_command = self.cmd_generator.nextCmd + errorIndication, errorStatus, errorIndex, varBinds = snmp_command( + auth_data, + transport_target, + '.1.3.6.1.2.1.2.2.1.', + lookupValues = True + ) + + ifTable = defaultdict(dict) + if errorIndication: + raise Exception("{0} for instance {1}".format(errorIndication,instance["ip_address"])) + else: + if errorStatus: + raise Exception("{0} for instance {1}".format(errorStatus.prettyPrint(),instance["ip_address"])) + else: + for tableRow in varBinds: + for name, val in tableRow: + ifTable[name.asTuple()[IF_TABLE_INDEX_POS]][name.asTuple()[IF_TABLE_TYPE_POS]] = val + + self.log.debug("Interface Table discovered %s" % ifTable) + for index in ifTable: + type = ifTable[index].get(IF_TYPE) + descr = ifTable[index].get(IF_DESCR) + if not reply_invalid(type): + if int(type) != LOCALHOST_INTERFACE and not reply_invalid(descr): + interface_list[index] = str(descr) + self.log.info("Discovered interface %s" % str(descr)) - empty_reply = False - interface_index = 1 - while not empty_reply: - interfaces_descr_oids = [] - interfaces_descr_oids.append((("IF-MIB","ifDescr"),interface_index)) - interfaces_descr_oids.append((("IF-MIB","ifType"),interface_index)) - interfaces_description = SnmpCheck.snmp_get(instance, interfaces_descr_oids) - descr = interfaces_description.pop(0)[1] - empty_reply= reply_invalid(descr) - type = interfaces_description.pop(0)[1] - if not reply_invalid(type) and int(type) !=24: - # ignore localhost loopback - interface_list[interface_index] = str(descr) - self.log.info("Discovered interface %s" % str(descr)) - interface_index += 1 return interface_list @classmethod From b240ee8520429e5a302ac7357733a62c92bf53e1 Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 13 Jun 2014 15:42:26 -0400 Subject: [PATCH 28/30] SNMP Checks| Reorganized imports --- checks.d/snmp.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index bd679a58ae..2e94db9957 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -1,14 +1,16 @@ -from checks import AgentCheck - +# std from collections import defaultdict +# project +from checks import AgentCheck + +# 3rd party from pysnmp.entity.rfc3413.oneliner import cmdgen from pysnmp.smi.exval import noSuchInstance, noSuchObject from pysnmp.smi import builder import pysnmp.proto.rfc1902 as snmp_type -convention_type_builder = builder.MibBuilder() -(CounterBasedGauge64, ZeroBasedCounter64) = convention_type_builder.importSymbols("HCNUM-TC","CounterBasedGauge64", "ZeroBasedCounter64") +(CounterBasedGauge64, ZeroBasedCounter64) = builder.MibBuilder().importSymbols("HCNUM-TC","CounterBasedGauge64", "ZeroBasedCounter64") SNMP_COUNTERS = [snmp_type.Counter32.__name__, snmp_type.Counter64.__name__, ZeroBasedCounter64.__name__] SNMP_GAUGES = [snmp_type.Gauge32.__name__, CounterBasedGauge64.__name__] From 1c09b09ae2a5a2405890dea52d30d4963c855cde Mon Sep 17 00:00:00 2001 From: Rudy Date: Fri, 13 Jun 2014 18:47:40 -0400 Subject: [PATCH 29/30] SNMP Checks| Add documentation --- checks.d/snmp.py | 53 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index 2e94db9957..d346d0dc68 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -10,15 +10,17 @@ from pysnmp.smi import builder import pysnmp.proto.rfc1902 as snmp_type +# Additional types that are not part of the SNMP protocol. cf RFC 2856 (CounterBasedGauge64, ZeroBasedCounter64) = builder.MibBuilder().importSymbols("HCNUM-TC","CounterBasedGauge64", "ZeroBasedCounter64") +# Metric type that we support SNMP_COUNTERS = [snmp_type.Counter32.__name__, snmp_type.Counter64.__name__, ZeroBasedCounter64.__name__] SNMP_GAUGES = [snmp_type.Gauge32.__name__, CounterBasedGauge64.__name__] # IF-MIB magic values +IF_TABLE_OID = '.1.3.6.1.2.1.2.2.1.' IF_TABLE_TYPE_POS = 9 IF_TABLE_INDEX_POS = 10 -IF_INDEX = 1 IF_DESCR = 2 IF_TYPE = 3 LOCALHOST_INTERFACE = 24 @@ -33,12 +35,16 @@ class SnmpCheck(AgentCheck): def __init__(self, name, init_config, agentConfig, instances=None): AgentCheck.__init__(self, name, init_config, agentConfig, instances) - self.counter_state = defaultdict(dict) + self.interface_list = {} + + # Load Custom MIB directory mibs_path = None if init_config is not None: mibs_path = init_config.get("mibs_folder") SnmpCheck.create_command_generator(mibs_path) + + # Detect the interfaces of the instance and retain their indexes if instances is not None: for instance in instances: if 'ip_address' in instance: @@ -47,6 +53,11 @@ def __init__(self, name, init_config, agentConfig, instances=None): @classmethod def create_command_generator(cls, mibs_path=None): + ''' + Create a command generator to perform all the snmp query + If mibs_path is not None, load the mibs present in the custom mibs + folder (Need to be in pysnmp format) + ''' cls.cmd_generator = cmdgen.CommandGenerator() if mibs_path is not None: mibBuilder = cls.cmd_generator.snmpEngine.msgAndPduDsp.\ @@ -57,6 +68,17 @@ def create_command_generator(cls, mibs_path=None): mibBuilder.setMibSources(*mibSources) def get_interfaces(self, instance): + ''' + Query the IF-MIB table to get the list of interfaces and their description + Ignore the interfaces that are loopback (localhost) + The nextCmd will return all the value in a format like this: + + [(1.3.6.1.2.1.2.2.1.{what type of information}.{index of the interface}, information)] + + For now we only care about the index, the description(eg. interface name eth0) and the type + to ignore the loopback interface + See http://www.alvestrand.no/objectid/1.3.6.1.2.1.2.2.1.html for more info about the table + ''' interface_list = {} transport_target = self.get_transport_target(instance) auth_data = self.get_auth_data(instance) @@ -65,7 +87,7 @@ def get_interfaces(self, instance): errorIndication, errorStatus, errorIndex, varBinds = snmp_command( auth_data, transport_target, - '.1.3.6.1.2.1.2.2.1.', + IF_TABLE_OID, lookupValues = True ) @@ -93,6 +115,10 @@ def get_interfaces(self, instance): @classmethod def get_auth_data(cls, instance): + ''' + Generate a Security Parameters object based on the configuration of the instance + See http://pysnmp.sourceforge.net/docs/current/security-configuration.html + ''' if "community_string" in instance: # SNMP v1 - SNMP v2 return cmdgen.CommunityData(instance['community_string']) @@ -120,6 +146,9 @@ def get_auth_data(cls, instance): @classmethod def get_transport_target(cls, instance): + ''' + Generate a Transport target object based on the configuration of the instance + ''' if "ip_address" not in instance: raise Exception("An IP address needs to be specified") ip_address = instance["ip_address"] @@ -130,13 +159,15 @@ def check(self, instance): tags = instance.get("tags",[]) ip_address = instance["ip_address"] device_oids = [] - interface_oids = [] oid_names ={} + + # Check the metrics completely defined for metric in instance.get('metrics',[]): if 'MIB' in metric: device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) elif 'OID' in metric: device_oids.append(metric['OID']) + # Associate the name to the OID so that we can perform the matching oid_names[metric['OID']]=metric['name'] else: raise Exception('Unsupported metrics format in config file') @@ -146,6 +177,9 @@ def check(self, instance): self.submit_metric(instance, oid, value, tags=tags + ["snmp_device:" + ip_address], oid_names = oid_names) + # Check the metrics defined per interface by appending the index + # of the table to create a fully defined metric + interface_oids = [] for metric in instance.get('interface_metrics',[]): interface_oids.append((metric["MIB"],metric["symbol"])) for interface, descr in self.interface_list[instance['ip_address']].items(): @@ -185,9 +219,17 @@ def snmp_get(cls, instance, oids): return varBinds def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): + ''' + Convert the values reported as pysnmp-Managed Objects to values and + report them to the aggregator + ''' if reply_invalid(snmp_value): + # Metrics not present in the queried object self.log.warning("No such Mib available: %s" %oid.getMibSymbol()[1]) return + + # Get the name for the OID either from the name that we can decode + # or from the name that was specified for it if str(oid.getOid()) in oid_names: name = "snmp."+ oid_names[str(oid.getOid())] else: @@ -197,6 +239,9 @@ def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): self.log.warning("Couldn't find a name for oid {0}".format(oid)) return + # Ugly hack but couldn't find a cleaner way + # Proper way would be to use the ASN1 method isSameTypeWith but this + # returns True in the case of CounterBasedGauge64 and Counter64 for example snmp_class = snmp_value.__class__.__name__ for counter_class in SNMP_COUNTERS: if snmp_class==counter_class: From 19be52b597b8c979cadb36716ae9c3b0bfa0f0e5 Mon Sep 17 00:00:00 2001 From: Rudy Date: Tue, 17 Jun 2014 15:58:08 -0400 Subject: [PATCH 30/30] SNMP Checks| Camel case and spacing --- checks.d/snmp.py | 88 ++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/checks.d/snmp.py b/checks.d/snmp.py index d346d0dc68..6c0a0a96a0 100644 --- a/checks.d/snmp.py +++ b/checks.d/snmp.py @@ -60,12 +60,12 @@ def create_command_generator(cls, mibs_path=None): ''' cls.cmd_generator = cmdgen.CommandGenerator() if mibs_path is not None: - mibBuilder = cls.cmd_generator.snmpEngine.msgAndPduDsp.\ - mibInstrumController.mibBuilder - mibSources = mibBuilder.getMibSources() + ( + mib_builder = cls.cmd_generator.snmpEngine.msgAndPduDsp.\ + mibInstrumController.mibBuilder + mib_sources = mib_builder.getMibSources() + ( builder.DirMibSource(mibs_path), ) - mibBuilder.setMibSources(*mibSources) + mib_builder.setMibSources(*mib_sources) def get_interfaces(self, instance): ''' @@ -84,28 +84,28 @@ def get_interfaces(self, instance): auth_data = self.get_auth_data(instance) snmp_command = self.cmd_generator.nextCmd - errorIndication, errorStatus, errorIndex, varBinds = snmp_command( + error_indication, error_status, error_index, var_binds = snmp_command( auth_data, transport_target, IF_TABLE_OID, lookupValues = True ) - ifTable = defaultdict(dict) - if errorIndication: - raise Exception("{0} for instance {1}".format(errorIndication,instance["ip_address"])) + if_table = defaultdict(dict) + if error_indication: + raise Exception("{0} for instance {1}".format(error_indication, instance["ip_address"])) else: - if errorStatus: - raise Exception("{0} for instance {1}".format(errorStatus.prettyPrint(),instance["ip_address"])) + if error_status: + raise Exception("{0} for instance {1}".format(error_status.prettyPrint(), instance["ip_address"])) else: - for tableRow in varBinds: - for name, val in tableRow: - ifTable[name.asTuple()[IF_TABLE_INDEX_POS]][name.asTuple()[IF_TABLE_TYPE_POS]] = val - - self.log.debug("Interface Table discovered %s" % ifTable) - for index in ifTable: - type = ifTable[index].get(IF_TYPE) - descr = ifTable[index].get(IF_DESCR) + for table_row in var_binds: + for name, val in table_row: + if_table[name.asTuple()[IF_TABLE_INDEX_POS]][name.asTuple()[IF_TABLE_TYPE_POS]] = val + + self.log.debug("Interface Table discovered %s" % if_table) + for index in if_table: + type = if_table[index].get(IF_TYPE) + descr = if_table[index].get(IF_DESCR) if not reply_invalid(type): if int(type) != LOCALHOST_INTERFACE and not reply_invalid(descr): interface_list[index] = str(descr) @@ -125,22 +125,22 @@ def get_auth_data(cls, instance): elif "user" in instance: # SNMP v3 user = instance["user"] - authKey = None - privKey = None - authProtocol = None - privProtocol = None + auth_key = None + priv_key = None + auth_protocol = None + priv_protocol = None if "authKey" in instance: - authKey = instance["authKey"] - authProtocol = cmdgen.usmHMACMD5AuthProtocol + auth_key = instance["authKey"] + auth_protocol = cmdgen.usmHMACMD5AuthProtocol if "privKey" in instance: - privKey = instance["privKey"] - authProtocol = cmdgen.usmHMACMD5AuthProtocol - privProtocol = cmdgen.usmDESPrivProtocol + priv_key = instance["privKey"] + auth_protocol = cmdgen.usmHMACMD5AuthProtocol + priv_protocol = cmdgen.usmDESPrivProtocol if "authProtocol" in instance: - authProtocol = getattr(cmdgen,instance["authProtocol"]) + auth_protocol = getattr(cmdgen, instance["authProtocol"]) if "privProtocol" in instance: - privProtocol = getattr(cmdgen,instance["privProtocol"]) - return cmdgen.UsmUserData(user, authKey, privKey, authProtocol, privProtocol) + priv_protocol = getattr(cmdgen, instance["privProtocol"]) + return cmdgen.UsmUserData(user, auth_key, priv_key, auth_protocol, priv_protocol) else: raise Exception("An authentication method needs to be provided") @@ -153,7 +153,7 @@ def get_transport_target(cls, instance): raise Exception("An IP address needs to be specified") ip_address = instance["ip_address"] port = instance.get("port", 161) # Default SNMP port - return cmdgen.UdpTransportTarget((ip_address,port)) + return cmdgen.UdpTransportTarget((ip_address, port)) def check(self, instance): tags = instance.get("tags",[]) @@ -162,16 +162,16 @@ def check(self, instance): oid_names ={} # Check the metrics completely defined - for metric in instance.get('metrics',[]): + for metric in instance.get('metrics', []): if 'MIB' in metric: - device_oids.append(((metric["MIB"],metric["symbol"]),metric["index"])) + device_oids.append(((metric["MIB"], metric["symbol"]), metric["index"])) elif 'OID' in metric: device_oids.append(metric['OID']) # Associate the name to the OID so that we can perform the matching - oid_names[metric['OID']]=metric['name'] + oid_names[metric['OID']] = metric['name'] else: raise Exception('Unsupported metrics format in config file') - self.log.debug("Querying device %s for %s oids",ip_address, len(device_oids)) + self.log.debug("Querying device %s for %s oids", ip_address, len(device_oids)) results = SnmpCheck.snmp_get(instance, device_oids) for oid, value in results: self.submit_metric(instance, oid, value, tags=tags + ["snmp_device:" + ip_address], @@ -180,11 +180,11 @@ def check(self, instance): # Check the metrics defined per interface by appending the index # of the table to create a fully defined metric interface_oids = [] - for metric in instance.get('interface_metrics',[]): - interface_oids.append((metric["MIB"],metric["symbol"])) + for metric in instance.get('interface_metrics', []): + interface_oids.append((metric["MIB"], metric["symbol"])) for interface, descr in self.interface_list[instance['ip_address']].items(): oids = [(oid, interface) for oid in interface_oids] - self.log.debug("Querying device %s for %s oids",ip_address, len(oids)) + self.log.debug("Querying device %s for %s oids", ip_address, len(oids)) interface_results = SnmpCheck.snmp_get(instance, oids) for oid, value in interface_results: self.submit_metric(instance, oid, value, tags = tags + ["snmp_device:" + ip_address, @@ -202,7 +202,7 @@ def snmp_get(cls, instance, oids): snmp_command = cls.cmd_generator.getCmd - errorIndication, errorStatus, errorIndex, varBinds = snmp_command( + error_indication, error_status, error_index, var_binds = snmp_command( auth_data, transport_target, *oids, @@ -210,13 +210,13 @@ def snmp_get(cls, instance, oids): lookupValues=True ) - if errorIndication: - raise Exception("{0} for instance {1}".format(errorIndication,instance["ip_address"])) + if error_indication: + raise Exception("{0} for instance {1}".format(error_indication, instance["ip_address"])) else: - if errorStatus: - raise Exception("{0} for instance {1}".format(errorStatus.prettyPrint(),instance["ip_address"])) + if error_status: + raise Exception("{0} for instance {1}".format(error_status.prettyPrint(), instance["ip_address"])) else: - return varBinds + return var_binds def submit_metric(self, instance, oid, snmp_value, tags=[], oid_names={}): '''