Add geo fields to add_host_metadata processor. (elastic#9392)

This lets users add geo data if they have it to the host


(cherry picked from commit c6c4a30)

libbeat/docs/processors-using.asciidoc

@@ -895,6 +895,14 @@ processors:
 - add_host_metadata:
     netinfo.enabled: false
     cache.ttl: 5m
+    geo:
+      name: nyc-dc1-rack1
+      location: 40.7128, -74.0060
+      continent_name: North America
+      country_iso_code: US
+      region_name: New York
+      region_iso_code: NY
+      city_name: New York
 -------------------------------------------------------------------------------
 
 It has the following settings:
@@ -903,6 +911,23 @@ It has the following settings:
 
 `cache.ttl`:: (Optional) The processor uses an internal cache for the host metadata. This sets the cache expiration time. The default is 5m, negative values disable caching altogether.
 
+`geo.name`:: User definable token to be used for identifying a discrete location. Frequently a datacenter, rack, or similar.
+
+`geo.location`:: Longitude and latitude in comma separated format.
+
+`geo.continent_name`:: Name of the continent.
+
+`geo.country_name`:: Name of the country.
+
+`geo.region_name`:: Name of the region.
+
+`geo.city_name`:: Name of the city.
+
+`geo.country_iso_code`:: ISO country code.
+
+`geo.region_iso_code`:: ISO region code.
+
+
 The `add_host_metadata` processor annotates each event with relevant metadata from the host machine.
 The fields added to the event are looking as following:
 
@@ -921,7 +946,16 @@ The fields added to the event are looking as following:
          "name":"Mac OS X"
       },
       "ip": ["192.168.0.1", "10.0.0.1"],
-      "mac": ["00:25:96:12:34:56", "72:00:06:ff:79:f1"]
+      "mac": ["00:25:96:12:34:56", "72:00:06:ff:79:f1"],
+      "geo": {
+          "continent_name": "North America",
+          "country_iso_code": "US",
+          "region_name": "New York",
+          "region_iso_code": "NY",
+          "city_name": "New York",
+          "name": "nyc-dc1-rack1",
+          "location": "40.7128, -74.0060"
+        }
    }
 }
 -------------------------------------------------------------------------------

libbeat/processors/add_host_metadata/add_host_metadata.go

@@ -20,6 +20,7 @@ package add_host_metadata
 import (
 	"fmt"
 	"net"
+	"regexp"
 	"sync"
 	"time"
 
@@ -43,8 +44,9 @@ type addHostMetadata struct {
 		time.Time
 		sync.Mutex
 	}
-	data   common.MapStrPointer
-	config Config
+	data    common.MapStrPointer
+	geoData common.MapStr
+	config  Config
 }
 
 const (
@@ -62,6 +64,46 @@ func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error)
 		data:   common.NewMapStrPointer(nil),
 	}
 	p.loadData()
+
+	if config.Geo != nil {
+		if len(config.Geo.Location) > 0 {
+			// Regexp matching a number with an optional decimal component
+			// Valid numbers: '123', '123.23', etc.
+			latOrLon := `\-?\d+(\.\d+)?`
+
+			// Regexp matching a pair of lat lon coordinates.
+			// e.g. 40.123, -92.929
+			locRegexp := `^\s*` + // anchor to start of string with optional whitespace
+				latOrLon + // match the latitude
+				`\s*\,\s*` + // match the separator. optional surrounding whitespace
+				latOrLon + // match the longitude
+				`\s*$` //optional whitespace then end anchor
+
+			if m, _ := regexp.MatchString(locRegexp, config.Geo.Location); !m {
+				return nil, errors.New(fmt.Sprintf("Invalid lat,lon  string for add_host_metadata: %s", config.Geo.Location))
+			}
+		}
+
+		geoFields := common.MapStr{
+			"name":             config.Geo.Name,
+			"location":         config.Geo.Location,
+			"continent_name":   config.Geo.ContinentName,
+			"country_iso_code": config.Geo.CountryISOCode,
+			"region_name":      config.Geo.RegionName,
+			"region_iso_code":  config.Geo.RegionISOCode,
+			"city_name":        config.Geo.CityName,
+		}
+		// Delete any empty values
+		blankStringMatch := regexp.MustCompile(`^\s*$`)
+		for k, v := range geoFields {
+			vStr := v.(string)
+			if blankStringMatch.MatchString(vStr) {
+				delete(geoFields, k)
+			}
+		}
+		p.geoData = common.MapStr{"host": common.MapStr{"geo": geoFields}}
+	}
+
 	return p, nil
 }
 
@@ -73,6 +115,10 @@ func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) {
 	}
 
 	event.Fields.DeepUpdate(p.data.Get().Clone())
+
+	if len(p.geoData) > 0 {
+		event.Fields.DeepUpdate(p.geoData)
+	}
 	return event, nil
 }
 

libbeat/processors/add_host_metadata/add_host_metadata_test.go

@@ -18,10 +18,13 @@
 package add_host_metadata
 
 import (
+	"fmt"
 	"runtime"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/libbeat/beat"
@@ -100,3 +103,103 @@ func TestConfigNetInfoEnabled(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, v)
 }
+
+func TestConfigGeoEnabled(t *testing.T) {
+	event := &beat.Event{
+		Fields:    common.MapStr{},
+		Timestamp: time.Now(),
+	}
+
+	config := map[string]interface{}{
+		"geo.name":             "yerevan-am",
+		"geo.location":         "40.177200, 44.503490",
+		"geo.continent_name":   "Asia",
+		"geo.country_iso_code": "AM",
+		"geo.region_name":      "Erevan",
+		"geo.region_iso_code":  "AM-ER",
+		"geo.city_name":        "Yerevan",
+	}
+
+	testConfig, err := common.NewConfigFrom(config)
+	assert.NoError(t, err)
+
+	p, err := newHostMetadataProcessor(testConfig)
+	require.NoError(t, err)
+
+	newEvent, err := p.Run(event)
+	assert.NoError(t, err)
+
+	for configKey, configValue := range config {
+		t.Run(fmt.Sprintf("Check of %s", configKey), func(t *testing.T) {
+			v, err := newEvent.GetValue(fmt.Sprintf("host.%s", configKey))
+			assert.NoError(t, err)
+			assert.Equal(t, configValue, v, "Could not find in %s", newEvent)
+		})
+	}
+}
+
+func TestPartialGeo(t *testing.T) {
+	event := &beat.Event{
+		Fields:    common.MapStr{},
+		Timestamp: time.Now(),
+	}
+
+	config := map[string]interface{}{
+		"geo.name":      "yerevan-am",
+		"geo.city_name": "  ",
+	}
+
+	testConfig, err := common.NewConfigFrom(config)
+	assert.NoError(t, err)
+
+	p, err := newHostMetadataProcessor(testConfig)
+	require.NoError(t, err)
+
+	newEvent, err := p.Run(event)
+	assert.NoError(t, err)
+
+	v, err := newEvent.Fields.GetValue("host.geo.name")
+	assert.NoError(t, err)
+	assert.Equal(t, "yerevan-am", v)
+
+	missing := []string{"continent_name", "country_name", "country_iso_code", "region_name", "region_iso_code", "city_name"}
+
+	for _, k := range missing {
+		path := "host.geo." + k
+		v, err = newEvent.Fields.GetValue(path)
+
+		assert.Equal(t, common.ErrKeyNotFound, err, "din expect to find %v", path)
+	}
+}
+
+func TestGeoLocationValidation(t *testing.T) {
+	locations := []struct {
+		str   string
+		valid bool
+	}{
+		{"40.177200, 44.503490", true},
+		{"-40.177200, -44.503490", true},
+		{"garbage", false},
+		{"9999999999", false},
+	}
+
+	for _, location := range locations {
+		t.Run(fmt.Sprintf("Location %s validation should be %t", location.str, location.valid), func(t *testing.T) {
+
+			conf, err := common.NewConfigFrom(map[string]interface{}{
+				"geo": map[string]interface{}{
+					"location": location.str,
+				},
+			})
+			require.NoError(t, err)
+
+			_, err = newHostMetadataProcessor(conf)
+
+			if location.valid {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+			}
+		})
+	}
+}

libbeat/processors/add_host_metadata/config.go

@@ -25,6 +25,18 @@ import (
 type Config struct {
 	NetInfoEnabled bool          `config:"netinfo.enabled"` // Add IP and MAC to event
 	CacheTTL       time.Duration `config:"cache.ttl"`
+	Geo            *GeoConfig    `config:"geo"`
+}
+
+// GeoConfig contains geo configuration data.
+type GeoConfig struct {
+	Name           string `config:"name"`
+	Location       string `config:"location"`
+	ContinentName  string `config:"continent_name"`
+	CountryISOCode string `config:"country_iso_code"`
+	RegionName     string `config:"region_name"`
+	RegionISOCode  string `config:"region_iso_code"`
+	CityName       string `config:"city_name"`
 }
 
 func defaultConfig() Config {