From 0ae9a3df2b78aa88f6babde7742b5a7a37dfd2d1 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 27 Mar 2017 12:53:44 +0200 Subject: [PATCH] Remove deprecated geoip feature (#3766) Geoip was deprecated in 5.0 and is now removed for 6.0. --- .travis.yml | 1 - CHANGELOG.asciidoc | 1 + NOTICE | 28 -- glide.yaml | 2 - libbeat/common/geolite.go | 64 ---- libbeat/docs/generalconfig.asciidoc | 29 -- libbeat/publisher/publish.go | 9 - .../tests/system/config/metricbeat.yml.j2 | 10 - packetbeat/_meta/fields.yml | 8 - packetbeat/_meta/fields_base.yml | 8 - packetbeat/docs/fields.asciidoc | 10 - packetbeat/publish/publish.go | 82 ---- .../tests/system/config/packetbeat.yml.j2 | 10 - packetbeat/tests/system/files/geoip_city.dat | Bin 188 -> 0 bytes .../tests/system/files/geoip_city_blocks.csv | 3 - .../tests/system/files/geoip_city_loc.csv | 3 - packetbeat/tests/system/test_0008_realip.py | 6 - packetbeat/tests/system/test_0011_geoip.py | 67 ---- .../nranchev/go-libGeoIP/LICENSE.txt | 24 -- .../nranchev/go-libGeoIP/README.textile | 46 --- .../nranchev/go-libGeoIP/example/example.go | 39 -- .../github.com/nranchev/go-libGeoIP/libgeo.go | 354 ------------------ winlogbeat/config/config.go | 3 +- winlogbeat/config/config_test.go | 2 +- 24 files changed, 3 insertions(+), 806 deletions(-) delete mode 100644 libbeat/common/geolite.go delete mode 100644 packetbeat/tests/system/files/geoip_city.dat delete mode 100644 packetbeat/tests/system/files/geoip_city_blocks.csv delete mode 100644 packetbeat/tests/system/files/geoip_city_loc.csv delete mode 100644 packetbeat/tests/system/test_0011_geoip.py delete mode 100644 vendor/github.com/nranchev/go-libGeoIP/LICENSE.txt delete mode 100644 vendor/github.com/nranchev/go-libGeoIP/README.textile delete mode 100644 vendor/github.com/nranchev/go-libGeoIP/example/example.go delete mode 100644 vendor/github.com/nranchev/go-libGeoIP/libgeo.go diff --git a/.travis.yml b/.travis.yml index ab0e550c6d01..90e2894cc9ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,6 @@ addons: packages: - python-virtualenv - libpcap-dev - - geoip-database before_install: - umask 022 diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2d59868aae36..5f74312404b8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -31,6 +31,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] `cgroups` to `process.cgroups.enabled`. {pull}3519[3519] - Change fieldnames couchbase.node.couch.*.actual_disk_size.* to couchbase.node.couch.*.disk_size.* {pull}3545[3545] *Packetbeat* +- Remove deprecated geoip. {pull}3766[3766] *Winlogbeat* diff --git a/NOTICE b/NOTICE index 24a2f33cc65c..43fd915fcfed 100644 --- a/NOTICE +++ b/NOTICE @@ -507,34 +507,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------- -github.com/nranchev/go-libGeoIP --------------------------------------------------------------------- -Copyright (c) 2010, Nikola Ranchev -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -------------------------------------------------------------------- github.com/pierrec/lz4 -------------------------------------------------------------------- diff --git a/glide.yaml b/glide.yaml index 7d3248c48452..8e51970cb3b3 100644 --- a/glide.yaml +++ b/glide.yaml @@ -4,8 +4,6 @@ import: version: v1.0.0 - package: gopkg.in/yaml.v2 version: a83829b6f1293c91addabc89d0571c246397bbf4 -- package: github.com/nranchev/go-libGeoIP - version: c78e8bd2dd3599feb21fd30886043979e82fe948 - package: golang.org/x/sys version: 62bee037599929a6e9146f29d10dd5208c43507d subpackages: diff --git a/libbeat/common/geolite.go b/libbeat/common/geolite.go deleted file mode 100644 index 38abad3c258e..000000000000 --- a/libbeat/common/geolite.go +++ /dev/null @@ -1,64 +0,0 @@ -package common - -import ( - "os" - "path/filepath" - - "github.com/elastic/beats/libbeat/logp" - - "github.com/nranchev/go-libGeoIP" -) - -// Geoip represents a string slice of GeoIP paths -type Geoip struct { - Paths *[]string -} - -func LoadGeoIPData(config Geoip) *libgeo.GeoIP { - geoipPaths := []string{} - - if config.Paths != nil { - geoipPaths = *config.Paths - } - if len(geoipPaths) == 0 { - // disabled - return nil - } - - logp.Warn("GeoIP lookup support is deprecated and will be removed in version 6.0.") - - // look for the first existing path - var geoipPath string - for _, path := range geoipPaths { - fi, err := os.Lstat(path) - if err != nil { - logp.Err("GeoIP path could not be loaded: %s", path) - continue - } - - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - // follow symlink - geoipPath, err = filepath.EvalSymlinks(path) - if err != nil { - logp.Warn("Could not load GeoIP data: %s", err.Error()) - return nil - } - } else { - geoipPath = path - } - break - } - - if len(geoipPath) == 0 { - logp.Warn("Couldn't load GeoIP database") - return nil - } - - geoLite, err := libgeo.Load(geoipPath) - if err != nil { - logp.Warn("Could not load GeoIP data: %s", err.Error()) - } - - logp.Info("Loaded GeoIP data from: %s", geoipPath) - return geoLite -} diff --git a/libbeat/docs/generalconfig.asciidoc b/libbeat/docs/generalconfig.asciidoc index 53aed7c3bca2..de46cc542a67 100644 --- a/libbeat/docs/generalconfig.asciidoc +++ b/libbeat/docs/generalconfig.asciidoc @@ -102,32 +102,3 @@ value is 1000. Sets the maximum number of CPUs that can be executing simultaneously. The default is the number of logical CPUs available in the system. - -===== geoip.paths - -deprecated[5.0.0, Please use the https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-geoip.html[Geoip processor in Ingest Node] or the https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html[Logstash GeoIP filter] instead] - -This configuration option is currently used by Packetbeat only and it will be removed in version 6.0. - -The paths to search for GeoIP databases. The Beat loads the first installed GeoIP database -that if finds. Then, for each transaction, the Beat exports the GeoIP location of the client. - -The recommended values for geoip.paths are `/usr/share/GeoIP/GeoLiteCity.dat` -and `/usr/local/var/GeoIP/GeoLiteCity.dat`. - -If no paths are configured, GeoIP is disabled. - -Example: - -[source,yaml] ------------------------------------------------------------------------------- -geoip: - paths: - - "/usr/share/GeoIP/GeoLiteCity.dat" - - "/usr/local/var/GeoIP/GeoLiteCity.dat" ------------------------------------------------------------------------------- - -*Important*: For GeoIP support to function correctly, the -https://dev.maxmind.com/geoip/legacy/geolite/[GeoLite City database] is required. - - diff --git a/libbeat/publisher/publish.go b/libbeat/publisher/publish.go index 05443e35d1b2..d066c6ffa662 100644 --- a/libbeat/publisher/publish.go +++ b/libbeat/publisher/publish.go @@ -10,7 +10,6 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs" "github.com/elastic/beats/libbeat/processors" - "github.com/nranchev/go-libGeoIP" // load supported output plugins _ "github.com/elastic/beats/libbeat/outputs/console" @@ -61,7 +60,6 @@ type BeatPublisher struct { disabled bool Index string Output []*outputWorker - geoLite *libgeo.GeoIP Processors *processors.Processors globalEventMetadata common.EventMetadata // Fields and tags to add to each event. @@ -85,7 +83,6 @@ type BeatPublisher struct { type ShipperConfig struct { common.EventMetadata `config:",inline"` // Fields and tags to add to each event. Name string `config:"name"` - Geoip common.Geoip `config:"geoip"` // internal publisher queue sizes QueueSize *int `config:"queue_size"` @@ -127,10 +124,6 @@ func (publisher *BeatPublisher) GetServerName(ip string) string { return "" } -func (publisher *BeatPublisher) GeoLite() *libgeo.GeoIP { - return publisher.geoLite -} - func (publisher *BeatPublisher) Connect() Client { atomic.AddUint32(&publisher.numClients, 1) return newClient(publisher) @@ -168,8 +161,6 @@ func (publisher *BeatPublisher) init( shipper.InitShipperConfig() - publisher.geoLite = common.LoadGeoIPData(shipper.Geoip) - publisher.wsPublisher.Init() publisher.wsOutput.Init() diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index 4a0fab297e26..33eb3e6a4289 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -113,16 +113,6 @@ tags: [ {%- endif -%} ] -{% if geoip_paths is not none %} -geoip: - paths: [ - {%- for path in geoip_paths -%} - "{{ beat.working_dir + '/' + path }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] -{%- endif %} - {%- if processors %} #================================ Filters ===================================== diff --git a/packetbeat/_meta/fields.yml b/packetbeat/_meta/fields.yml index 027035594561..325551720eea 100644 --- a/packetbeat/_meta/fields.yml +++ b/packetbeat/_meta/fields.yml @@ -41,14 +41,6 @@ the `client_ip` for non proxy clients. format: Dotted notation. - - name: client_location - type: geo_point - example: 40.715, -74.011 - description: > - DEPRECATED. Please use `client_geoip` instead. - The GeoIP location of the `real_ip` IP address or of the `client_ip` address if the `real_ip` is - disabled. The field is a string containing the latitude and longitude separated by a comma. - - name: client_geoip description: The GeoIP information of the client. type: group diff --git a/packetbeat/_meta/fields_base.yml b/packetbeat/_meta/fields_base.yml index f7e21a650fa3..5f74ecdb3a02 100644 --- a/packetbeat/_meta/fields_base.yml +++ b/packetbeat/_meta/fields_base.yml @@ -41,14 +41,6 @@ the `client_ip` for non proxy clients. format: Dotted notation. - - name: client_location - type: geo_point - example: 40.715, -74.011 - description: > - DEPRECATED. Please use `client_geoip` instead. - The GeoIP location of the `real_ip` IP address or of the `client_ip` address if the `real_ip` is - disabled. The field is a string containing the latitude and longitude separated by a comma. - - name: client_geoip description: The GeoIP information of the client. type: group diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 1aec20510c3c..5009ceb677bc 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -1114,16 +1114,6 @@ If the server initiating the transaction is a proxy, this field contains the ori Unless this field is disabled, it always has a value, and it matches the `client_ip` for non proxy clients. -[float] -=== client_location - -type: geo_point - -example: 40.715, -74.011 - -DEPRECATED. Please use `client_geoip` instead. The GeoIP location of the `real_ip` IP address or of the `client_ip` address if the `real_ip` is disabled. The field is a string containing the latitude and longitude separated by a comma. - - [float] == client_geoip Fields diff --git a/packetbeat/publish/publish.go b/packetbeat/publish/publish.go index c68eb5b1b4be..8cfbcbf0108f 100644 --- a/packetbeat/publish/publish.go +++ b/packetbeat/publish/publish.go @@ -2,13 +2,11 @@ package publish import ( "errors" - "fmt" "sync" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/publisher" - "github.com/nranchev/go-libGeoIP" ) type Transactions interface { @@ -24,7 +22,6 @@ type PacketbeatPublisher struct { client publisher.Client topo topologyProvider - geoLite *libgeo.GeoIP ignoreOutgoing bool wg sync.WaitGroup @@ -44,7 +41,6 @@ type ChanTransactions struct { type topologyProvider interface { IsPublisherIP(ip string) bool GetServerName(ip string) string - GeoLite() *libgeo.GeoIP } func (t *ChanTransactions) PublishTransaction(event common.MapStr) bool { @@ -67,7 +63,6 @@ func NewPublisher( return &PacketbeatPublisher{ pub: pub, topo: topo, - geoLite: topo.GeoLite(), ignoreOutgoing: ignoreOutgoing, client: pub.Connect(), done: make(chan struct{}), @@ -151,10 +146,6 @@ func (p *PacketbeatPublisher) onFlow(events []common.MapStr) { continue } - if !p.addGeoIPToFlow(event) { - continue - } - pub = append(pub, event) } @@ -233,78 +224,5 @@ func (p *PacketbeatPublisher) normalizeTransAddr(event common.MapStr) bool { } - if p.geoLite != nil { - realIP, exists := event["real_ip"] - if exists && len(realIP.(common.NetString)) > 0 { - loc := p.geoLite.GetLocationByIP(string(realIP.(common.NetString))) - if loc != nil && loc.Latitude != 0 && loc.Longitude != 0 { - loc := fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude) - event["client_location"] = loc - } - } else { - if len(srcServer) == 0 && src != nil { // only for external IP addresses - loc := p.geoLite.GetLocationByIP(src.IP) - if loc != nil && loc.Latitude != 0 && loc.Longitude != 0 { - loc := fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude) - event["client_location"] = loc - } - } - } - } - - return true -} - -func (p *PacketbeatPublisher) addGeoIPToFlow(event common.MapStr) bool { - - getLocation := func(host common.MapStr, ip_type string) string { - - ip, exists := host[ip_type] - if !exists { - return "" - } - - str, ok := ip.(string) - if !ok { - logp.Warn("IP address must be string") - return "" - } - loc := p.geoLite.GetLocationByIP(str) - if loc == nil || loc.Latitude == 0 || loc.Longitude == 0 { - return "" - } - - return fmt.Sprintf("%f, %f", loc.Latitude, loc.Longitude) - } - - if p.geoLite == nil { - return true - } - - ipFieldNames := [][]string{ - {"ip", "ip_location"}, - {"outter_ip", "outter_ip_location"}, - {"ipv6", "ipv6_location"}, - {"outter_ipv6", "outter_ipv6_location"}, - } - - source := event["source"].(common.MapStr) - dest := event["dest"].(common.MapStr) - - for _, name := range ipFieldNames { - - loc := getLocation(source, name[0]) - if loc != "" { - source[name[1]] = loc - } - - loc = getLocation(dest, name[0]) - if loc != "" { - dest[name[1]] = loc - } - } - event["source"] = source - event["dest"] = dest - return true } diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index 86be018a1ea1..ef56907d3f3e 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -170,16 +170,6 @@ tags: [ {%- endif -%} ] -{% if geoip_paths is not none %} -geoip: - paths: [ - {%- for path in geoip_paths -%} - "{{ beat.working_dir + '/' + path }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} - ] -{%- endif %} - {%- if processors %} #================================ Filters ===================================== diff --git a/packetbeat/tests/system/files/geoip_city.dat b/packetbeat/tests/system/files/geoip_city.dat deleted file mode 100644 index 4b6756927833c3122a2da5707bc404be5e219ee7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 188 zcmZQ%V2}VKCI$v(7@GymU}a!n12foR;v8TGCz#{{liXmEhk=0?%m5n22WIetNdX21 zK`=uICM(RqAOdEK!bHTt3~?|i$-tmxVQ9wSlvr*A SW|WdxqE}G)|Nnm`pj7~J0T1*5 diff --git a/packetbeat/tests/system/files/geoip_city_blocks.csv b/packetbeat/tests/system/files/geoip_city_blocks.csv deleted file mode 100644 index 2ca5f9a358d3..000000000000 --- a/packetbeat/tests/system/files/geoip_city_blocks.csv +++ /dev/null @@ -1,3 +0,0 @@ -Copyright (c) 2012 MaxMind LLC. All Rights Reserved. -locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode -"1509369600","1509369855","350669" diff --git a/packetbeat/tests/system/files/geoip_city_loc.csv b/packetbeat/tests/system/files/geoip_city_loc.csv deleted file mode 100644 index c96241081fe6..000000000000 --- a/packetbeat/tests/system/files/geoip_city_loc.csv +++ /dev/null @@ -1,3 +0,0 @@ -Copyright (c) 2012 MaxMind LLC. All Rights Reserved. -locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode -350669,"DE","16","Berlin","10119",52.5285,13.4109,, diff --git a/packetbeat/tests/system/test_0008_realip.py b/packetbeat/tests/system/test_0008_realip.py index 5b55f73d3a6f..bd028b1faabb 100644 --- a/packetbeat/tests/system/test_0008_realip.py +++ b/packetbeat/tests/system/test_0008_realip.py @@ -12,9 +12,7 @@ def test_x_forward_for(self): http_ports=[8002], http_real_ip_header="X-Forward-For", http_send_all_headers=True, - geoip_paths=["geoip_city.dat"] ) - self.copy_files(["geoip_city.dat"]) self.run_packetbeat(pcap="http_realip.pcap", debug_selectors=["http"]) objs = self.read_output() @@ -22,16 +20,13 @@ def test_x_forward_for(self): o = objs[0] assert o["real_ip"] == "89.247.39.104" - assert o["client_location"] == "52.528503, 13.410904" def test_x_forwarded_for_multiple_ip(self): self.render_config_template( http_ports=[80], http_real_ip_header="X-Forwarded-For", http_send_all_headers=True, - geoip_paths=["geoip_city.dat"] ) - self.copy_files(["geoip_city.dat"]) self.run_packetbeat(pcap="http_x_forwarded_for.pcap", debug_selectors=["http"]) objs = self.read_output() @@ -39,4 +34,3 @@ def test_x_forwarded_for_multiple_ip(self): o = objs[0] assert o["real_ip"] == "89.247.39.104" - assert o["client_location"] == "52.528503, 13.410904" diff --git a/packetbeat/tests/system/test_0011_geoip.py b/packetbeat/tests/system/test_0011_geoip.py deleted file mode 100644 index d1343102c95b..000000000000 --- a/packetbeat/tests/system/test_0011_geoip.py +++ /dev/null @@ -1,67 +0,0 @@ -from packetbeat import BaseTest -import os - -""" -Tests for reading the geoip files. -""" - - -class Test(BaseTest): - - def test_geoip_config_disabled(self): - self.render_config_template( - http_ports=[8002], - http_real_ip_header="X-Forward-For", - http_send_all_headers=True, - geoip_paths=[] - ) - self.run_packetbeat(pcap="http_realip.pcap", debug_selectors=["http"]) - - objs = self.read_output() - assert len(objs) == 1 - o = objs[0] - - assert o["real_ip"] == "89.247.39.104" - assert "client_location" not in o - - def test_geoip_config_from_file(self): - self.render_config_template( - http_ports=[8002], - http_real_ip_header="X-Forward-For", - http_send_all_headers=True, - geoip_paths=["geoip_city.dat"] - ) - # geoip_onrange.dat is generated from geoip_onerange.csv - # by using https://github.com/mteodoro/mmutils - self.copy_files(["geoip_city.dat"]) - self.run_packetbeat(pcap="http_realip.pcap", debug_selectors=["http"]) - - objs = self.read_output() - assert len(objs) == 1 - o = objs[0] - - assert o["real_ip"] == "89.247.39.104" - assert o["client_location"] == "52.528503, 13.410904" - - def test_geoip_symlink(self): - """ - Should be able to follow symlinks to GeoIP libs. - """ - self.render_config_template( - http_ports=[8002], - http_real_ip_header="X-Forward-For", - http_send_all_headers=True, - geoip_paths=["geoip.dat"] - ) - self.copy_files(["geoip_city.dat"]) - os.symlink("geoip_city.dat", - os.path.join(self.working_dir, "geoip.dat")) - - self.run_packetbeat(pcap="http_realip.pcap", debug_selectors=["http"]) - - objs = self.read_output() - assert len(objs) == 1 - o = objs[0] - - assert o["real_ip"] == "89.247.39.104" - assert o["client_location"] == "52.528503, 13.410904" diff --git a/vendor/github.com/nranchev/go-libGeoIP/LICENSE.txt b/vendor/github.com/nranchev/go-libGeoIP/LICENSE.txt deleted file mode 100644 index 5a0cb2dfabbe..000000000000 --- a/vendor/github.com/nranchev/go-libGeoIP/LICENSE.txt +++ /dev/null @@ -1,24 +0,0 @@ -Copyright (c) 2010, Nikola Ranchev -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/nranchev/go-libGeoIP/README.textile b/vendor/github.com/nranchev/go-libGeoIP/README.textile deleted file mode 100644 index 2e449b8a6f56..000000000000 --- a/vendor/github.com/nranchev/go-libGeoIP/README.textile +++ /dev/null @@ -1,46 +0,0 @@ -h2. General Description - -p{width:500px}. This is the Go implementation of the "Maxmind":http://www.maxmind.com/app/ip-location GeoIP API. It is incomplete and work in progress the initial goal is support only two of the database types - the City Lite and Country Lite. The only supported method is loading the full db on startup into memory (memory cache). - -h3. Supported Access Methods - -* In Memory (Load(string)) - -h3. Supported Database Formats - -* Country Edition (dbType=1) -* City Edition REV 0 (dbType=6) -* City Edition REV 1 (dbType=2) - -h3. Supported Lookups - -* By IP Address (GetLocationByIP(string)) -* By IP Number (GetLocationByIPNum(uint32)) - -h3. Supported Responses - -* CountryCode string (available in all databases) -* CountryName string (available in all databases) -* City string -* Region string -* PostalCode string -* Latitude float32 -* Longitude float32 - -h3. To Do -* Implement better error handling (report the error on load and lookups) -* Better returns, country edition has only code and name (perhaps use interfaces) -* Add test cases and benchmarking -* Add support for more database formats - -h3. Build - -make (See Makefile for more details) - -h3. Example - -./example DBFILE IPADDRESS (i.e. ./example GeoIP.dat 1.1.1.1) - -h3. Usage - -Please see example.go for a complete example of how to use this library. diff --git a/vendor/github.com/nranchev/go-libGeoIP/example/example.go b/vendor/github.com/nranchev/go-libGeoIP/example/example.go deleted file mode 100644 index 8252b6272e09..000000000000 --- a/vendor/github.com/nranchev/go-libGeoIP/example/example.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "github.com/nranchev/go-libGeoIP" -) - -func main() { - flag.Parse() - - // Check the number of arguments - if flag.NArg() < 2 { - fmt.Printf("usage: main DBFILE IPADDRESS\n") - return - } - - // Set the arguments - dbFile := flag.Arg(0) - ipAddr := flag.Arg(1) - - // Load the database file, exit on failure - gi, err := libgeo.Load(dbFile) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - // Lookup the IP and display the details if country is found - loc := gi.GetLocationByIP(ipAddr) - if loc != nil { - fmt.Printf("Country: %s (%s)\n", loc.CountryName, loc.CountryCode) - fmt.Printf("City: %s\n", loc.City) - fmt.Printf("Region: %s\n", loc.Region) - fmt.Printf("Postal Code: %s\n", loc.PostalCode) - fmt.Printf("Latitude: %f\n", loc.Latitude) - fmt.Printf("Longitude: %f\n", loc.Longitude) - } -} diff --git a/vendor/github.com/nranchev/go-libGeoIP/libgeo.go b/vendor/github.com/nranchev/go-libGeoIP/libgeo.go deleted file mode 100644 index 4794a55b087c..000000000000 --- a/vendor/github.com/nranchev/go-libGeoIP/libgeo.go +++ /dev/null @@ -1,354 +0,0 @@ -/** - * libgeo.go - * - * Copyright (c) 2010, Nikola Ranchev - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - Neither the name of the nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package libgeo - -// Dependencies -import ( - "errors" - "os" -) - -// Globals (const arrays that will be initialized inside init()) -var ( - countryCode = []string{ - "--", "AP", "EU", "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AN", "AO", "AQ", "AR", - "AS", "AT", "AU", "AW", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", - "BM", "BN", "BO", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", - "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CX", "CY", "CZ", - "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", - "FJ", "FK", "FM", "FO", "FR", "FX", "GA", "GB", "GD", "GE", "GF", "GH", "GI", "GL", - "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", - "HT", "HU", "ID", "IE", "IL", "IN", "IO", "IQ", "IR", "IS", "IT", "JM", "JO", "JP", - "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", - "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "MG", "MH", "MK", - "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", - "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", - "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", - "QA", "RE", "RO", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", - "SK", "SL", "SM", "SN", "SO", "SR", "ST", "SV", "SY", "SZ", "TC", "TD", "TF", "TG", - "TH", "TJ", "TK", "TM", "TN", "TO", "TL", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", - "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", - "YT", "RS", "ZA", "ZM", "ME", "ZW", "A1", "A2", "O1", "AX", "GG", "IM", "JE", "BL", - "MF", "BQ", "SS", "O1"} - countryName = []string{ - "N/A", "Asia/Pacific Region", "Europe", "Andorra", "United Arab Emirates", - "Afghanistan", "Antigua and Barbuda", "Anguilla", "Albania", "Armenia", - "Netherlands Antilles", "Angola", "Antarctica", "Argentina", "American Samoa", - "Austria", "Australia", "Aruba", "Azerbaijan", "Bosnia and Herzegovina", - "Barbados", "Bangladesh", "Belgium", "Burkina Faso", "Bulgaria", "Bahrain", - "Burundi", "Benin", "Bermuda", "Brunei Darussalam", "Bolivia", "Brazil", "Bahamas", - "Bhutan", "Bouvet Island", "Botswana", "Belarus", "Belize", "Canada", - "Cocos (Keeling) Islands", "Congo, The Democratic Republic of the", - "Central African Republic", "Congo", "Switzerland", "Cote D'Ivoire", - "Cook Islands", "Chile", "Cameroon", "China", "Colombia", "Costa Rica", "Cuba", - "Cape Verde", "Christmas Island", "Cyprus", "Czech Republic", "Germany", - "Djibouti", "Denmark", "Dominica", "Dominican Republic", "Algeria", "Ecuador", - "Estonia", "Egypt", "Western Sahara", "Eritrea", "Spain", "Ethiopia", "Finland", - "Fiji", "Falkland Islands (Malvinas)", "Micronesia, Federated States of", - "Faroe Islands", "France", "France, Metropolitan", "Gabon", "United Kingdom", - "Grenada", "Georgia", "French Guiana", "Ghana", "Gibraltar", "Greenland", "Gambia", - "Guinea", "Guadeloupe", "Equatorial Guinea", "Greece", - "South Georgia and the South Sandwich Islands", "Guatemala", "Guam", - "Guinea-Bissau", "Guyana", "Hong Kong", "Heard Island and McDonald Islands", - "Honduras", "Croatia", "Haiti", "Hungary", "Indonesia", "Ireland", "Israel", "India", - "British Indian Ocean Territory", "Iraq", "Iran, Islamic Republic of", - "Iceland", "Italy", "Jamaica", "Jordan", "Japan", "Kenya", "Kyrgyzstan", "Cambodia", - "Kiribati", "Comoros", "Saint Kitts and Nevis", - "Korea, Democratic People's Republic of", "Korea, Republic of", "Kuwait", - "Cayman Islands", "Kazakhstan", "Lao People's Democratic Republic", "Lebanon", - "Saint Lucia", "Liechtenstein", "Sri Lanka", "Liberia", "Lesotho", "Lithuania", - "Luxembourg", "Latvia", "Libyan Arab Jamahiriya", "Morocco", "Monaco", - "Moldova, Republic of", "Madagascar", "Marshall Islands", - "Macedonia", "Mali", "Myanmar", "Mongolia", - "Macau", "Northern Mariana Islands", "Martinique", "Mauritania", "Montserrat", - "Malta", "Mauritius", "Maldives", "Malawi", "Mexico", "Malaysia", "Mozambique", - "Namibia", "New Caledonia", "Niger", "Norfolk Island", "Nigeria", "Nicaragua", - "Netherlands", "Norway", "Nepal", "Nauru", "Niue", "New Zealand", "Oman", "Panama", - "Peru", "French Polynesia", "Papua New Guinea", "Philippines", "Pakistan", - "Poland", "Saint Pierre and Miquelon", "Pitcairn Islands", "Puerto Rico", - "Palestinian Territory", "Portugal", "Palau", "Paraguay", "Qatar", - "Reunion", "Romania", "Russian Federation", "Rwanda", "Saudi Arabia", - "Solomon Islands", "Seychelles", "Sudan", "Sweden", "Singapore", "Saint Helena", - "Slovenia", "Svalbard and Jan Mayen", "Slovakia", "Sierra Leone", "San Marino", - "Senegal", "Somalia", "Suriname", "Sao Tome and Principe", "El Salvador", - "Syrian Arab Republic", "Swaziland", "Turks and Caicos Islands", "Chad", - "French Southern Territories", "Togo", "Thailand", "Tajikistan", "Tokelau", - "Turkmenistan", "Tunisia", "Tonga", "Timor-Leste", "Turkey", "Trinidad and Tobago", - "Tuvalu", "Taiwan", "Tanzania, United Republic of", "Ukraine", "Uganda", - "United States Minor Outlying Islands", "United States", "Uruguay", "Uzbekistan", - "Holy See (Vatican City State)", "Saint Vincent and the Grenadines", - "Venezuela", "Virgin Islands, British", "Virgin Islands, U.S.", "Vietnam", - "Vanuatu", "Wallis and Futuna", "Samoa", "Yemen", "Mayotte", "Serbia", - "South Africa", "Zambia", "Montenegro", "Zimbabwe", "Anonymous Proxy", - "Satellite Provider", "Other", "Aland Islands", "Guernsey", "Isle of Man", "Jersey", - "Saint Barthelemy", "Saint Martin", "Bonaire, Saint Eustatius and Saba", - "South Sudan", "Other"} -) - -// Constants -const ( - maxRecordLength = 4 - standardRecordLength = 3 - countryBegin = 16776960 - structureInfoMaxSize = 20 - fullRecordLength = 60 - segmentRecordLength = 3 - - // DB Types - dbCountryEdition = byte(1) - dbCityEditionRev0 = byte(6) - dbCityEditionRev1 = byte(2) -) - -// These are some structs -type GeoIP struct { - databaseSegment int // No need to make an array of size 1 - recordLength int // Set to one of the constants above - dbType byte // Store the database type - data []byte // All of the data from the DB file -} -type Location struct { - CountryCode string // If country ed. only country info is filled - CountryName string // If country ed. only country info is filled - Region string - City string - PostalCode string - Latitude float32 - Longitude float32 -} - -// Load the database file in memory, detect the db format and setup the GeoIP struct -func Load(filename string) (gi *GeoIP, err error) { - // Try to open the requested file - dbInfo, err := os.Lstat(filename) - if err != nil { - return - } - dbFile, err := os.Open(filename) - if err != nil { - return - } - - // Copy the db into memory - gi = new(GeoIP) - gi.data = make([]byte, dbInfo.Size()) - dbFile.Read(gi.data) - dbFile.Close() - - // Check the database type - gi.dbType = dbCountryEdition // Default the database to country edition - gi.databaseSegment = countryBegin // Default to country DB - gi.recordLength = standardRecordLength // Default to country DB - - // Search for the DB type headers - delim := make([]byte, 3) - for i := 0; i < structureInfoMaxSize; i++ { - delim = gi.data[len(gi.data)-i-3-1 : len(gi.data)-i-1] - if int8(delim[0]) == -1 && int8(delim[1]) == -1 && int8(delim[2]) == -1 { - gi.dbType = gi.data[len(gi.data)-i-1] - // If we detect city edition set the correct segment offset - if gi.dbType == dbCityEditionRev0 || gi.dbType == dbCityEditionRev1 { - buf := make([]byte, segmentRecordLength) - buf = gi.data[len(gi.data)-i-1+1 : len(gi.data)-i-1+4] - gi.databaseSegment = 0 - for j := 0; j < segmentRecordLength; j++ { - gi.databaseSegment += (int(buf[j]) << uint8(j*8)) - } - } - break - } - } - - // Support older formats - if gi.dbType >= 106 { - gi.dbType -= 105 - } - - // Reject unsupported formats - if gi.dbType != dbCountryEdition && gi.dbType != dbCityEditionRev0 && gi.dbType != dbCityEditionRev1 { - err = errors.New("Unsupported database format") - return - } - - return -} - -// Lookup by IP address and return location -func (gi *GeoIP) GetLocationByIP(ip string) *Location { - return gi.GetLocationByIPNum(AddrToNum(ip)) -} - -// Lookup by IP number and return location -func (gi *GeoIP) GetLocationByIPNum(ipNum uint32) *Location { - // Perform the lookup on the database to see if the record is found - offset := gi.lookupByIPNum(ipNum) - - // Check if the country was found - if gi.dbType == dbCountryEdition && offset-countryBegin == 0 || - gi.dbType != dbCountryEdition && gi.databaseSegment == offset { - return nil - } - - // Create a generic location structure - location := new(Location) - - // If the database is country - if gi.dbType == dbCountryEdition { - location.CountryCode = countryCode[offset-countryBegin] - location.CountryName = countryName[offset-countryBegin] - - return location - } - - // Find the max record length - recPointer := offset + (2*gi.recordLength-1)*gi.databaseSegment - recordEnd := recPointer + fullRecordLength - if len(gi.data)-recPointer < fullRecordLength { - recordEnd = len(gi.data) - } - - // Read the country code/name first - location.CountryCode = countryCode[gi.data[recPointer]] - location.CountryName = countryName[gi.data[recPointer]] - readLen := 1 - recPointer += 1 - - // Get the region - for readLen = 0; gi.data[recPointer+readLen] != '\000' && - recPointer+readLen < recordEnd; readLen++ { - } - if readLen != 0 { - location.Region = string(gi.data[recPointer : recPointer+readLen]) - } - recPointer += readLen + 1 - - // Get the city - for readLen = 0; gi.data[recPointer+readLen] != '\000' && - recPointer+readLen < recordEnd; readLen++ { - } - if readLen != 0 { - location.City = string(gi.data[recPointer : recPointer+readLen]) - } - recPointer += readLen + 1 - - // Get the postal code - for readLen = 0; gi.data[recPointer+readLen] != '\000' && - recPointer+readLen < recordEnd; readLen++ { - } - if readLen != 0 { - location.PostalCode = string(gi.data[recPointer : recPointer+readLen]) - } - recPointer += readLen + 1 - - // Get the latitude - coordinate := float32(0) - for j := 0; j < 3; j++ { - coordinate += float32(int32(gi.data[recPointer+j]) << uint8(j*8)) - } - location.Latitude = float32(coordinate/10000 - 180) - recPointer += 3 - - // Get the longitude - coordinate = 0 - for j := 0; j < 3; j++ { - coordinate += float32(int32(gi.data[recPointer+j]) << uint8(j*8)) - } - location.Longitude = float32(coordinate/10000 - 180) - - return location -} - -// Read the database and return record position -func (gi *GeoIP) lookupByIPNum(ip uint32) int { - buf := make([]byte, 2*maxRecordLength) - x := make([]int, 2) - offset := 0 - for depth := 31; depth >= 0; depth-- { - for i := 0; i < 2*maxRecordLength; i++ { - buf[i] = gi.data[(2*gi.recordLength*offset)+i] - } - for i := 0; i < 2; i++ { - x[i] = 0 - for j := 0; j < gi.recordLength; j++ { - var y int = int(buf[i*gi.recordLength+j]) - if y < 0 { - y += 256 - } - x[i] += (y << uint(j*8)) - } - } - if (ip & (1 << uint(depth))) > 0 { - if x[1] >= gi.databaseSegment { - return x[1] - } - offset = x[1] - } else { - if x[0] >= gi.databaseSegment { - return x[0] - } - offset = x[0] - } - } - return 0 -} - -// Convert ip address to an int representation -func AddrToNum(ip string) uint32 { - octet := uint32(0) - ipnum := uint32(0) - i := 3 - for j := 0; j < len(ip); j++ { - c := byte(ip[j]) - if c == '.' { - if octet > 255 { - return 0 - } - ipnum <<= 8 - ipnum += octet - i-- - octet = 0 - } else { - t := octet - octet <<= 3 - octet += t - octet += t - c -= '0' - if c > 9 { - return 0 - } - octet += uint32(c) - } - } - if (octet > 255) || (i != 0) { - return 0 - } - ipnum <<= 8 - return uint32(ipnum + octet) -} diff --git a/winlogbeat/config/config.go b/winlogbeat/config/config.go index 895f1869d0d5..b46452fc7ccf 100644 --- a/winlogbeat/config/config.go +++ b/winlogbeat/config/config.go @@ -45,8 +45,7 @@ func (s Settings) Validate() error { // TODO: winlogbeat should not try to validate top-level beats config validKeys := []string{ - "fields", "fields_under_root", "tags", - "name", "geoip", + "fields", "fields_under_root", "tags", "name", "queue_size", "bulk_queue_size", "max_procs", "processors", "logging", "output", "path", "winlogbeat", "dashboards", diff --git a/winlogbeat/config/config_test.go b/winlogbeat/config/config_test.go index 2d8f44a61270..bf1ceb09294c 100644 --- a/winlogbeat/config/config_test.go +++ b/winlogbeat/config/config_test.go @@ -47,7 +47,7 @@ func TestConfigValidate(t *testing.T) { map[string]interface{}{"other": "value"}, }, "1 error: Invalid top-level key 'other' found. Valid keys are bulk_queue_size, dashboards, " + - "fields, fields_under_root, geoip, logging, max_procs, " + + "fields, fields_under_root, logging, max_procs, " + "name, output, path, processors, queue_size, tags, winlogbeat", }, {