Remove deprecated geoip feature (#3766)

Geoip was deprecated in 5.0 and is now removed for 6.0.

.travis.yml

@@ -85,7 +85,6 @@ addons:
     packages:
       - python-virtualenv
       - libpcap-dev
-      - geoip-database
 
 before_install:
   - umask 022

CHANGELOG.asciidoc

@@ -31,6 +31,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
   `cgroups` to `process.cgroups.enabled`. {pull}3519[3519]
 - Change fieldnames couchbase.node.couch.*.actual_disk_size.* to couchbase.node.couch.*.disk_size.* {pull}3545[3545]
 *Packetbeat*
+- Remove deprecated geoip. {pull}3766[3766]
 
 *Winlogbeat*
 

NOTICE

@@ -507,34 +507,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
---------------------------------------------------------------------
-github.com/nranchev/go-libGeoIP
---------------------------------------------------------------------
-Copyright (c) 2010, Nikola Ranchev
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of the <organization> nor the
-      names of its contributors may be used to endorse or promote products
-      derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 --------------------------------------------------------------------
 github.com/pierrec/lz4
 --------------------------------------------------------------------

glide.yaml

@@ -4,8 +4,6 @@ import:
   version: v1.0.0
 - package: gopkg.in/yaml.v2
   version: a83829b6f1293c91addabc89d0571c246397bbf4
-- package: github.com/nranchev/go-libGeoIP
-  version: c78e8bd2dd3599feb21fd30886043979e82fe948
 - package: golang.org/x/sys
   version: 62bee037599929a6e9146f29d10dd5208c43507d
   subpackages:

libbeat/docs/generalconfig.asciidoc

@@ -102,32 +102,3 @@ value is 1000.
 
 Sets the maximum number of CPUs that can be executing simultaneously. The
 default is the number of logical CPUs available in the system.
-
-===== geoip.paths
-
-deprecated[5.0.0, Please use the https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-geoip.html[Geoip processor in Ingest Node] or the https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html[Logstash GeoIP filter] instead]
-
-This configuration option is currently used by Packetbeat only and it will be removed in version 6.0.
-
-The paths to search for GeoIP databases. The Beat loads the first installed GeoIP database
-that if finds. Then, for each transaction, the Beat exports the GeoIP location of the client.
-
-The recommended values for geoip.paths are `/usr/share/GeoIP/GeoLiteCity.dat`
-and `/usr/local/var/GeoIP/GeoLiteCity.dat`.
-
-If no paths are configured, GeoIP is disabled.
-
-Example:
-
-[source,yaml]
-------------------------------------------------------------------------------
-geoip:
-  paths:
-    - "/usr/share/GeoIP/GeoLiteCity.dat"
-    - "/usr/local/var/GeoIP/GeoLiteCity.dat"
-------------------------------------------------------------------------------
-
-*Important*: For GeoIP support to function correctly, the
-https://dev.maxmind.com/geoip/legacy/geolite/[GeoLite City database] is required.
-
-

libbeat/publisher/publish.go

@@ -10,7 +10,6 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/outputs"
 	"github.com/elastic/beats/libbeat/processors"
-	"github.com/nranchev/go-libGeoIP"
 
 	// load supported output plugins
 	_ "github.com/elastic/beats/libbeat/outputs/console"
@@ -61,7 +60,6 @@ type BeatPublisher struct {
 	disabled    bool
 	Index       string
 	Output      []*outputWorker
-	geoLite     *libgeo.GeoIP
 	Processors  *processors.Processors
 
 	globalEventMetadata common.EventMetadata // Fields and tags to add to each event.
@@ -85,7 +83,6 @@ type BeatPublisher struct {
 type ShipperConfig struct {
 	common.EventMetadata `config:",inline"` // Fields and tags to add to each event.
 	Name                 string             `config:"name"`
-	Geoip                common.Geoip       `config:"geoip"`
 
 	// internal publisher queue sizes
 	QueueSize     *int `config:"queue_size"`
@@ -127,10 +124,6 @@ func (publisher *BeatPublisher) GetServerName(ip string) string {
 	return ""
 }
 
-func (publisher *BeatPublisher) GeoLite() *libgeo.GeoIP {
-	return publisher.geoLite
-}
-
 func (publisher *BeatPublisher) Connect() Client {
 	atomic.AddUint32(&publisher.numClients, 1)
 	return newClient(publisher)
@@ -168,8 +161,6 @@ func (publisher *BeatPublisher) init(
 
 	shipper.InitShipperConfig()
 
-	publisher.geoLite = common.LoadGeoIPData(shipper.Geoip)
-
 	publisher.wsPublisher.Init()
 	publisher.wsOutput.Init()
 

metricbeat/tests/system/config/metricbeat.yml.j2

@@ -113,16 +113,6 @@ tags: [
   {%- endif -%}
 ]
 
-{% if geoip_paths is not none %}
-geoip:
-  paths: [
-      {%- for path in geoip_paths -%}
-          "{{ beat.working_dir + '/' + path }}"
-          {%- if not loop.last %}, {% endif -%}
-      {%- endfor -%}
-  ]
-{%- endif %}
-
 {%- if processors %}
 
 #================================ Filters =====================================

packetbeat/_meta/fields.yml

@@ -41,14 +41,6 @@
         the `client_ip` for non proxy clients.
       format: Dotted notation.
 
-    - name: client_location
-      type: geo_point
-      example: 40.715, -74.011
-      description: >
-        DEPRECATED. Please use `client_geoip` instead.
-        The GeoIP location of the `real_ip` IP address or of the `client_ip` address if the `real_ip` is
-        disabled. The field is a string containing the latitude and longitude separated by a comma.
-
     - name: client_geoip
       description: The GeoIP information of the client.
       type: group

packetbeat/_meta/fields_base.yml

@@ -41,14 +41,6 @@
         the `client_ip` for non proxy clients.
       format: Dotted notation.
 
-    - name: client_location
-      type: geo_point
-      example: 40.715, -74.011
-      description: >
-        DEPRECATED. Please use `client_geoip` instead.
-        The GeoIP location of the `real_ip` IP address or of the `client_ip` address if the `real_ip` is
-        disabled. The field is a string containing the latitude and longitude separated by a comma.
-
     - name: client_geoip
       description: The GeoIP information of the client.
       type: group

packetbeat/docs/fields.asciidoc

@@ -1114,16 +1114,6 @@ If the server initiating the transaction is a proxy, this field contains the ori
 Unless this field is disabled, it always has a value, and it matches the `client_ip` for non proxy clients.
 
 
-[float]
-=== client_location
-
-type: geo_point
-
-example: 40.715, -74.011
-
-DEPRECATED. Please use `client_geoip` instead. The GeoIP location of the `real_ip` IP address or of the `client_ip` address if the `real_ip` is disabled. The field is a string containing the latitude and longitude separated by a comma.
-
-
 [float]
 == client_geoip Fields
 

packetbeat/publish/publish.go

@@ -2,13 +2,11 @@ package publish
 
 import (
 	"errors"
-	"fmt"
 	"sync"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/publisher"
-	"github.com/nranchev/go-libGeoIP"
 )
 
 type Transactions interface {
@@ -24,7 +22,6 @@ type PacketbeatPublisher struct {
 	client publisher.Client
 
 	topo           topologyProvider
-	geoLite        *libgeo.GeoIP
 	ignoreOutgoing bool
 
 	wg   sync.WaitGroup
@@ -44,7 +41,6 @@ type ChanTransactions struct {
 type topologyProvider interface {
 	IsPublisherIP(ip string) bool
 	GetServerName(ip string) string
-	GeoLite() *libgeo.GeoIP
 }
 
 func (t *ChanTransactions) PublishTransaction(event common.MapStr) bool {
@@ -67,7 +63,6 @@ func NewPublisher(
 	return &PacketbeatPublisher{
 		pub:            pub,
 		topo:           topo,
-		geoLite:        topo.GeoLite(),
 		ignoreOutgoing: ignoreOutgoing,
 		client:         pub.Connect(),
 		done:           make(chan struct{}),
@@ -151,10 +146,6 @@ func (p *PacketbeatPublisher) onFlow(events []common.MapStr) {
 			continue
 		}
 
-		if !p.addGeoIPToFlow(event) {
-			continue
-		}
-
 		pub = append(pub, event)
 	}
 
@@ -233,78 +224,5 @@ func (p *PacketbeatPublisher) normalizeTransAddr(event common.MapStr) bool {
 
 	}
 
-	if p.geoLite != nil {
-		realIP, exists := event["real_ip"]
-		if exists && len(realIP.(common.NetString)) > 0 {
-			loc := p.geoLite.GetLocationByIP(string(realIP.(common.NetString)))
-			if loc != nil && loc.Latitude != 0 && loc.Longitude != 0 {
-				loc := fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude)
-				event["client_location"] = loc
-			}
-		} else {
-			if len(srcServer) == 0 && src != nil { // only for external IP addresses
-				loc := p.geoLite.GetLocationByIP(src.IP)
-				if loc != nil && loc.Latitude != 0 && loc.Longitude != 0 {
-					loc := fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude)
-					event["client_location"] = loc
-				}
-			}
-		}
-	}
-
-	return true
-}
-
-func (p *PacketbeatPublisher) addGeoIPToFlow(event common.MapStr) bool {
-
-	getLocation := func(host common.MapStr, ip_type string) string {
-
-		ip, exists := host[ip_type]
-		if !exists {
-			return ""
-		}
-
-		str, ok := ip.(string)
-		if !ok {
-			logp.Warn("IP address must be string")
-			return ""
-		}
-		loc := p.geoLite.GetLocationByIP(str)
-		if loc == nil || loc.Latitude == 0 || loc.Longitude == 0 {
-			return ""
-		}
-
-		return fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude)
-	}
-
-	if p.geoLite == nil {
-		return true
-	}
-
-	ipFieldNames := [][]string{
-		{"ip", "ip_location"},
-		{"outter_ip", "outter_ip_location"},
-		{"ipv6", "ipv6_location"},
-		{"outter_ipv6", "outter_ipv6_location"},
-	}
-
-	source := event["source"].(common.MapStr)
-	dest := event["dest"].(common.MapStr)
-
-	for _, name := range ipFieldNames {
-
-		loc := getLocation(source, name[0])
-		if loc != "" {
-			source[name[1]] = loc
-		}
-
-		loc = getLocation(dest, name[0])
-		if loc != "" {
-			dest[name[1]] = loc
-		}
-	}
-	event["source"] = source
-	event["dest"] = dest
-
 	return true
 }