Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add network from MaxMind Geo ASN database #61676

Merged
merged 6 commits into from
Sep 24, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/reference/ingest/processors/geoip.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,10 @@ and `location`. The fields actually added depend on what has been found and whic
`country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties
were configured in `properties`.
* If the GeoLite2 ASN database is used, then the following fields may be added under the `target_field`: `ip`,
`asn`, and `organization_name`. The fields actually added depend on what has been found and which properties were configured
`asn`, `organization_name` and `network`. The fields actually added depend on what has been found and which properties were configured
in `properties`.


Here is an example that uses the default city database and adds the geographical information to the `geoip` field based on the `ip` field:

[source,console]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.ingest.geoip;

import com.maxmind.db.Network;
import com.maxmind.geoip2.exception.AddressNotFoundException;
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
Expand Down Expand Up @@ -345,6 +346,7 @@ private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) {

Integer asn = response.getAutonomousSystemNumber();
String organization_name = response.getAutonomousSystemOrganization();
Network network = response.getNetwork();

Map<String, Object> geoData = new HashMap<>();
for (Property property : this.properties) {
Expand All @@ -362,6 +364,11 @@ private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) {
geoData.put("organization_name", organization_name);
}
break;
case NETWORK:
if (network != null) {
geoData.put("network", network.toString());
}
break;
}
}
return geoData;
Expand All @@ -376,7 +383,7 @@ public static final class Factory implements Processor.Factory {
Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE
));
static final Set<Property> DEFAULT_ASN_PROPERTIES = Collections.unmodifiableSet(EnumSet.of(
Property.IP, Property.ASN, Property.ORGANIZATION_NAME
Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
));

private final Map<String, DatabaseReaderLazyLoader> databaseReaders;
Expand Down Expand Up @@ -464,7 +471,8 @@ enum Property {
TIMEZONE,
LOCATION,
ASN,
ORGANIZATION_NAME;
ORGANIZATION_NAME,
NETWORK;

static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.of(
Property.IP, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_NAME,
Expand All @@ -475,7 +483,7 @@ enum Property {
Property.IP, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE
);
static final EnumSet<Property> ALL_ASN_PROPERTIES = EnumSet.of(
Property.IP, Property.ASN, Property.ORGANIZATION_NAME
Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK
);

public static Property parseProperty(String databaseType, String value) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ public void testBuildWithAsnDbAndCityFields() throws Exception {
config.put("properties", Collections.singletonList(cityProperty));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
"]. valid values are [IP, ASN, ORGANIZATION_NAME]"));
"]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]"));
}

public void testBuildNonExistingDbFile() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,10 +209,11 @@ public void testAsn() throws Exception {
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
@SuppressWarnings("unchecked")
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
assertThat(geoData.size(), equalTo(3));
assertThat(geoData.size(), equalTo(4));
assertThat(geoData.get("ip"), equalTo(ip));
assertThat(geoData.get("asn"), equalTo(1136));
assertThat(geoData.get("organization_name"), equalTo("KPN B.V."));
assertThat(geoData.get("network"), equalTo("82.168.0.0/14"));
}

public void testAddressIsNotInTheDatabase() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,8 @@
index: test
id: 1
- match: { _source.field1: "82.171.64.0" }
- length: { _source.geoip: 3 }
- length: { _source.geoip: 4 }
- match: { _source.geoip.ip: "82.171.64.0" }
- match: { _source.geoip.asn: 1136 }
- match: { _source.geoip.organization_name: "KPN B.V." }
- match: { _source.geoip.network: "82.168.0.0/14" }