Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch logging to go-kit #447

Merged
merged 1 commit into from
Sep 11, 2019
Merged

Switch logging to go-kit #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
43 changes: 22 additions & 21 deletions collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ import (
"strings"
"time"

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
"github.com/soniah/gosnmp"
"gopkg.in/alecthomas/kingpin.v2"

Expand Down Expand Up @@ -81,7 +82,7 @@ func listToOid(l []int) string {
return strings.Join(result, ".")
}

func ScrapeTarget(target string, config *config.Module) ([]gosnmp.SnmpPDU, error) {
func ScrapeTarget(target string, config *config.Module, logger log.Logger) ([]gosnmp.SnmpPDU, error) {
// Set the options.
snmp := gosnmp.GoSNMP{}
snmp.MaxRepetitions = config.WalkParams.MaxRepetitions
Expand Down Expand Up @@ -123,16 +124,16 @@ func ScrapeTarget(target string, config *config.Module) ([]gosnmp.SnmpPDU, error
oids = maxOids
}

log.Debugf("Getting %d OIDs from target %q", oids, snmp.Target)
level.Debug(logger).Log("msg", "Getting OIDs", "oids", oids)
getStart := time.Now()
packet, err := snmp.Get(getOids[:oids])
if err != nil {
return nil, fmt.Errorf("error getting target %s: %s", snmp.Target, err)
}
log.Debugf("Get of %d OIDs completed in %s", oids, time.Since(getStart))
level.Debug(logger).Log("msg", "Get of OIDs completed", "oids", oids, "duration_seconds", time.Since(getStart))
// SNMPv1 will return packet error for unsupported OIDs.
if packet.Error == gosnmp.NoSuchName && snmp.Version == gosnmp.Version1 {
log.Debugf("OID %s not supported by target %s", getOids[0], snmp.Target)
level.Debug(logger).Log("msg", "OID not supported by target", "oids", getOids[0])
getOids = getOids[oids:]
continue
}
Expand All @@ -143,7 +144,7 @@ func ScrapeTarget(target string, config *config.Module) ([]gosnmp.SnmpPDU, error
}
for _, v := range packet.Variables {
if v.Type == gosnmp.NoSuchObject || v.Type == gosnmp.NoSuchInstance {
log.Debugf("OID %s not supported by target %s", v.Name, snmp.Target)
level.Debug(logger).Log("msg", "OID not supported by target", "oids", v.Name)
continue
}
result = append(result, v)
Expand All @@ -153,7 +154,7 @@ func ScrapeTarget(target string, config *config.Module) ([]gosnmp.SnmpPDU, error

for _, subtree := range config.Walk {
var pdus []gosnmp.SnmpPDU
log.Debugf("Walking target %q subtree %q", snmp.Target, subtree)
level.Debug(logger).Log("msg", "Walking subtree", "oid", subtree)
walkStart := time.Now()
if snmp.Version == gosnmp.Version1 {
pdus, err = snmp.WalkAll(subtree)
Expand All @@ -163,7 +164,7 @@ func ScrapeTarget(target string, config *config.Module) ([]gosnmp.SnmpPDU, error
if err != nil {
return nil, fmt.Errorf("error walking target %s: %s", snmp.Target, err)
}
log.Debugf("Walk of target %q subtree %q completed in %s", snmp.Target, subtree, time.Since(walkStart))
level.Debug(logger).Log("msg", "Walk of subtree completed", "oid", subtree, "duration_seconds", time.Since(walkStart))

result = append(result, pdus...)
}
Expand Down Expand Up @@ -196,6 +197,7 @@ func buildMetricTree(metrics []*config.Metric) *MetricNode {
type collector struct {
target string
module *config.Module
logger log.Logger
}

// Describe implements Prometheus.Collector.
Expand All @@ -206,9 +208,9 @@ func (c collector) Describe(ch chan<- *prometheus.Desc) {
// Collect implements Prometheus.Collector.
func (c collector) Collect(ch chan<- prometheus.Metric) {
start := time.Now()
pdus, err := ScrapeTarget(c.target, c.module)
pdus, err := ScrapeTarget(c.target, c.module, c.logger)
if err != nil {
log.Infof("Error scraping target %s: %s", c.target, err)
level.Info(c.logger).Log("msg", "Error scraping target", "err", err)
ch <- prometheus.NewInvalidMetric(prometheus.NewDesc("snmp_error", "Error scraping target", nil, nil), err)
return
}
Expand Down Expand Up @@ -239,7 +241,7 @@ PduLoop:
}
if head.metric != nil {
// Found a match.
samples := pduToSamples(oidList[i+1:], &pdu, head.metric, oidToPdu)
samples := pduToSamples(oidList[i+1:], &pdu, head.metric, oidToPdu, c.logger)
for _, sample := range samples {
ch <- sample
}
Expand Down Expand Up @@ -318,7 +320,7 @@ func parseDateAndTime(pdu *gosnmp.SnmpPDU) (float64, error) {
return float64(t.Unix()), nil
}

func pduToSamples(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, oidToPdu map[string]gosnmp.SnmpPDU) []prometheus.Metric {
func pduToSamples(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, oidToPdu map[string]gosnmp.SnmpPDU, logger log.Logger) []prometheus.Metric {
var err error
// The part of the OID that is the indexes.
labels := indexesToLabels(indexOids, metric, oidToPdu)
Expand All @@ -344,7 +346,7 @@ func pduToSamples(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, o
t = prometheus.GaugeValue
value, err = parseDateAndTime(pdu)
if err != nil {
log.Debugf("error parsing DateAndTime: %s", err)
level.Debug(logger).Log("msg", "Error parsing DateAndTime", "err", err)
return []prometheus.Metric{}
}
case "EnumAsInfo":
Expand All @@ -369,16 +371,16 @@ func pduToSamples(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, o
metricType = t
} else {
metricType = "OctetString"
log.Debugf("Unable to handle type value %d at %s for %s", val, prevOid, metric.Name)
level.Debug(logger).Log("msg", "Unable to handle type value", "value", val, "oid", prevOid, "metric", metric.Name)
}
} else {
metricType = "OctetString"
log.Debugf("Unable to find type at %s for %s", prevOid, metric.Name)
level.Debug(logger).Log("msg", "Unable to find type at oid for metric", "oid", prevOid, "metric", metric.Name)
}
}

if len(metric.RegexpExtracts) > 0 {
return applyRegexExtracts(metric, pduValueAsString(pdu, metricType), labelnames, labelvalues)
return applyRegexExtracts(metric, pduValueAsString(pdu, metricType), labelnames, labelvalues, logger)
}
// For strings we put the value as a label with the same name as the metric.
// If the name is already an index, we do not need to set it again.
Expand All @@ -398,19 +400,19 @@ func pduToSamples(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, o
return []prometheus.Metric{sample}
}

func applyRegexExtracts(metric *config.Metric, pduValue string, labelnames, labelvalues []string) []prometheus.Metric {
func applyRegexExtracts(metric *config.Metric, pduValue string, labelnames, labelvalues []string, logger log.Logger) []prometheus.Metric {
results := []prometheus.Metric{}
for name, strMetricSlice := range metric.RegexpExtracts {
for _, strMetric := range strMetricSlice {
indexes := strMetric.Regex.FindStringSubmatchIndex(pduValue)
if indexes == nil {
log.Debugf("No match found for regexp: %v against value: %v for metric %v", strMetric.Regex.String(), pduValue, metric.Name)
level.Debug(logger).Log("msg", "No match found for regexp", "metric", metric.Name, "value", pduValue, "regex", strMetric.Regex.String())
continue
}
res := strMetric.Regex.ExpandString([]byte{}, strMetric.Value, pduValue, indexes)
v, err := strconv.ParseFloat(string(res), 64)
if err != nil {
log.Debugf("Error parsing float64 from value: %v for metric: %v", res, metric.Name)
level.Debug(logger).Log("msg", "Error parsing float64 from value", "metric", metric.Name, "value", pduValue, "regex", strMetric.Regex.String(), "extracted_value", res)
continue
}
newMetric, err := prometheus.NewConstMetric(prometheus.NewDesc(metric.Name+name, metric.Help+" (regex extracted)", labelnames, nil),
Expand Down Expand Up @@ -530,7 +532,6 @@ func pduValueAsString(pdu *gosnmp.SnmpPDU, typ string) string {
return ""
default:
// This shouldn't happen.
log.Infof("Got PDU with unexpected type: Name: %s Value: '%s', Go Type: %T SNMP Type: %v", pdu.Name, pdu.Value, pdu.Value, pdu.Type)
snmpUnexpectedPduType.Inc()
return fmt.Sprintf("%s", pdu.Value)
}
Expand Down Expand Up @@ -628,7 +629,7 @@ func indexOidsAsString(indexOids []int, typ string, fixedSize int, implied bool)
}
return fmt.Sprintf("%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X", parts...), subOid, indexOids
default:
log.Fatalf("Unknown index type %s", typ)
panic(fmt.Sprintf("Unknown index type %s", typ))
return "", nil, nil
}
}
Expand Down
10 changes: 4 additions & 6 deletions collector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ import (
"regexp"
"testing"

"github.com/go-kit/kit/log"
"github.com/prometheus/client_model/go"
"github.com/prometheus/common/log"
"github.com/soniah/gosnmp"
kingpin "gopkg.in/alecthomas/kingpin.v2"

Expand Down Expand Up @@ -493,7 +493,7 @@ func TestPduToSample(t *testing.T) {
}

for i, c := range cases {
metrics := pduToSamples(c.indexOids, c.pdu, c.metric, c.oidToPdu)
metrics := pduToSamples(c.indexOids, c.pdu, c.metric, c.oidToPdu, log.NewNopLogger())
if len(metrics) != len(c.expectedMetrics) && !c.shouldErr {
t.Fatalf("Unexpected number of metrics returned for case %v: want %v, got %v", i, len(c.expectedMetrics), len(metrics))
}
Expand Down Expand Up @@ -534,9 +534,7 @@ func TestGetPduValue(t *testing.T) {
}

func TestGetPduLargeValue(t *testing.T) {
// Setup default flags and suppress logging.
log.AddFlags(kingpin.CommandLine)
_, err := kingpin.CommandLine.Parse([]string{"--log.level", "fatal"})
_, err := kingpin.CommandLine.Parse([]string{})
if err != nil {
t.Fatal(err)
}
Expand All @@ -550,7 +548,7 @@ func TestGetPduLargeValue(t *testing.T) {
t.Fatalf("Got incorrect counter wrapping for Counter64: %v", value)
}

_, err = kingpin.CommandLine.Parse([]string{"--log.level", "fatal", "--no-snmp.wrap-large-counters"})
_, err = kingpin.CommandLine.Parse([]string{"--no-snmp.wrap-large-counters"})
if err != nil {
t.Fatal(err)
}
Expand Down
56 changes: 38 additions & 18 deletions generator/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,36 @@ import (
"path/filepath"
"strings"

"github.com/prometheus/common/log"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/common/promlog"
"github.com/prometheus/common/promlog/flag"
"gopkg.in/alecthomas/kingpin.v2"
"gopkg.in/yaml.v2"

"github.com/prometheus/snmp_exporter/config"
)

// Generate a snmp_exporter config and write it out.
func generateConfig(nodes *Node, nameToNode map[string]*Node) {
func generateConfig(nodes *Node, nameToNode map[string]*Node, logger log.Logger) error {
outputPath, err := filepath.Abs(*outputPath)
if err != nil {
log.Fatal("Unable to determine absolute path for output")
return fmt.Errorf("unable to determine absolute path for output")
}

content, err := ioutil.ReadFile("generator.yml")
if err != nil {
log.Fatalf("Error reading yml config: %s", err)
return fmt.Errorf("error reading yml config: %s", err)
}
cfg := &Config{}
err = yaml.UnmarshalStrict(content, cfg)
if err != nil {
log.Fatalf("Error parsing yml config: %s", err)
return fmt.Errorf("error parsing yml config: %s", err)
}

outputConfig := config.Config{}
for name, m := range cfg.Modules {
log.Infof("Generating config for module %s", name)
level.Info(logger).Log("msg", "Generating config for module", "module", name)
// Give each module a copy of the tree so that it can be modified.
mNodes := nodes.Copy()
// Build the map with new pointers.
Expand All @@ -55,34 +58,39 @@ func generateConfig(nodes *Node, nameToNode map[string]*Node) {
mNameToNode[n.Oid] = n
mNameToNode[n.Label] = n
})
outputConfig[name] = generateConfigModule(m, mNodes, mNameToNode)
out, err := generateConfigModule(m, mNodes, mNameToNode, logger)
if err != nil {
return err
}
outputConfig[name] = out
outputConfig[name].WalkParams = m.WalkParams
log.Infof("Generated %d metrics for module %s", len(outputConfig[name].Metrics), name)
level.Info(logger).Log("msg", "Generated metrics", "module", name, "metrics", len(outputConfig[name].Metrics))
}

config.DoNotHideSecrets = true
out, err := yaml.Marshal(outputConfig)
config.DoNotHideSecrets = false
if err != nil {
log.Fatalf("Error marshaling yml: %s", err)
return fmt.Errorf("error marshaling yml: %s", err)
}

// Check the generated config to catch auth/version issues.
err = yaml.UnmarshalStrict(out, &config.Config{})
if err != nil {
log.Fatalf("Error parsing generated config: %s", err)
return fmt.Errorf("error parsing generated config: %s", err)
}

f, err := os.Create(outputPath)
if err != nil {
log.Fatalf("Error opening output file: %s", err)
return fmt.Errorf("error opening output file: %s", err)
}
out = append([]byte("# WARNING: This file was auto-generated using snmp_exporter generator, manual changes will be lost.\n"), out...)
_, err = f.Write(out)
if err != nil {
log.Fatalf("Error writing to output file: %s", err)
return fmt.Errorf("error writing to output file: %s", err)
}
log.Infof("Config written to %s", outputPath)
level.Info(logger).Log("msg", "Config written", "file", outputPath)
return nil
}

var (
Expand All @@ -94,22 +102,34 @@ var (
)

func main() {
log.AddFlags(kingpin.CommandLine)
promlogConfig := &promlog.Config{}
flag.AddFlags(kingpin.CommandLine, promlogConfig)
kingpin.HelpFlag.Short('h')
command := kingpin.Parse()
logger := promlog.New(promlogConfig)

parseOutput := strings.TrimSpace(initSNMP())
parseOutput, err := initSNMP(logger)
if err != nil {
level.Error(logger).Log("msg", "Error initializing netsnmp", "err", err)
os.Exit(1)
}

parseOutput = strings.TrimSpace(parseOutput)
parseErrors := len(parseOutput) != 0
if parseErrors {
log.Warnf("NetSNMP reported %d parse error(s)", len(strings.Split(parseOutput, "\n")))
level.Warn(logger).Log("msg", "NetSNMP reported parse error(s)", "errors", len(strings.Split(parseOutput, "\n")))
}

nodes := getMIBTree()
nameToNode := prepareTree(nodes)
nameToNode := prepareTree(nodes, logger)

switch command {
case generateCommand.FullCommand():
generateConfig(nodes, nameToNode)
err := generateConfig(nodes, nameToNode, logger)
if err != nil {
level.Error(logger).Log("msg", "Error generating config netsnmp", "err", err)
os.Exit(1)
}
case parseErrorsCommand.FullCommand():
fmt.Println(parseOutput)
case dumpCommand.FullCommand():
Expand Down
Loading