Update GeoIP database service URL (elastic#69862)

This change updates GeoIP database service URL to the new https://geoip.elastic.co/v1/database and removes (now optional) key/UUID parameter.
It also fixes geoip-fixture to provide 3 different test databases (City, Country and ASN).
It also unmutes GeoIpDownloaderIT. testGeoIpDatabasesDownload with additional logging and increased timeouts which tries to address elastic#69594

modules/ingest-geoip/build.gradle

@@ -42,7 +42,7 @@ def fixtureAddress = {
   assert useFixture: 'closure should not be used without a fixture'
   int ephemeralPort = tasks.getByPath(":test:fixtures:geoip-fixture:postProcessFixture").ext."test.fixtures.geoip-fixture.tcp.80"
   assert ephemeralPort > 0
-  return "http://127.0.0.1:${ephemeralPort}"
+  return "http://127.0.0.1:${ephemeralPort}/"
 }
 
 if (useFixture) {

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

@@ -66,7 +66,6 @@ public void disableDownloader(){
         assertTrue(settingsResponse.isAcknowledged());
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/69594")
     public void testGeoIpDatabasesDownload() throws Exception {
         // use short wait for local fixture, longer when we hit real service
         int waitTime = ENDPOINT == null ? 120 : 10;

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -60,7 +60,7 @@ class GeoIpDownloader extends AllocatedPersistentTask {
     public static final Setting<TimeValue> POLL_INTERVAL_SETTING = Setting.timeSetting("geoip.downloader.poll.interval",
         TimeValue.timeValueDays(3), TimeValue.timeValueDays(1), Property.Dynamic, Property.NodeScope);
     public static final Setting<String> ENDPOINT_SETTING = Setting.simpleString("geoip.downloader.endpoint",
-        "https://paisano.elastic.dev/v1/geoip/database", Property.NodeScope);
+        "https://geoip.elastic.co/v1/database", Property.NodeScope);
 
     public static final String GEOIP_DOWNLOADER = "geoip-downloader";
     static final String DATABASES_INDEX = ".geoip_databases";
@@ -106,7 +106,9 @@ void updateDatabases() throws IOException {
 
     @SuppressWarnings("unchecked")
     private <T> List<T> fetchDatabasesOverview() throws IOException {
-        byte[] data = httpClient.getBytes(endpoint + "?key=11111111-1111-1111-1111-111111111111&elastic_geoip_service_tos=agree");
+        String url = endpoint + "?elastic_geoip_service_tos=agree";
+        logger.info("fetching geoip databases overview from [" + url + "]");
+        byte[] data = httpClient.getBytes(url);
         try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
             DeprecationHandler.THROW_UNSUPPORTED_OPERATION, data)) {
             return (List<T>) parser.list();

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java

@@ -84,8 +84,8 @@ private InputStream getInputStream(HttpURLConnection conn) throws IOException {
 
     private HttpURLConnection createConnection(String url) throws IOException {
         HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
-        conn.setConnectTimeout(5000);
-        conn.setReadTimeout(5000);
+        conn.setConnectTimeout(10000);
+        conn.setReadTimeout(10000);
         conn.setDoOutput(false);
         conn.setInstanceFollowRedirects(false);
         return conn;

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java

@@ -325,7 +325,7 @@ public void testUpdateDatabases() throws IOException {
         builder.map(singletonMap("a", 2));
         builder.endArray();
         builder.close();
-        when(httpClient.getBytes("a.b?key=11111111-1111-1111-1111-111111111111&elastic_geoip_service_tos=agree"))
+        when(httpClient.getBytes("a.b?elastic_geoip_service_tos=agree"))
             .thenReturn(baos.toByteArray());
         Iterator<Map<String, Object>> it = maps.iterator();
         geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool,

test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java

@@ -36,8 +36,9 @@ public class GeoIpHttpFixture {
         this.server = HttpServer.create(new InetSocketAddress(InetAddress.getByName(args[0]), Integer.parseInt(args[1])), 0);
         this.server.createContext("/", exchange -> {
             String query = exchange.getRequestURI().getQuery();
-            if (query.contains("elastic_geoip_service_tos=agree") == false) {
+            if (query == null || query.contains("elastic_geoip_service_tos=agree") == false) {
                 exchange.sendResponseHeaders(400, 0);
+                exchange.getResponseBody().close();
                 return;
             }
             String data = rawData.replace("endpoint", "http://" + exchange.getRequestHeaders().getFirst("Host"));
@@ -46,9 +47,10 @@ public class GeoIpHttpFixture {
                 writer.write(data);
             }
         });
-        this.server.createContext("/db.mmdb.gz", exchange -> {
+        this.server.createContext("/db", exchange -> {
             exchange.sendResponseHeaders(200, 0);
-            try (InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream("/GeoIP2-City-Test.mmdb.gz");
+            String dbName = exchange.getRequestURI().getPath().replaceAll(".*/db", "");
+            try (InputStream inputStream = GeoIpHttpFixture.class.getResourceAsStream(dbName);
                  OutputStream outputStream = exchange.getResponseBody()) {
                 int read2;
                 while ((read2 = inputStream.read(bytes)) != -1) {

test/fixtures/geoip-fixture/src/main/resources/data.json

@@ -1,20 +1,20 @@
 [
   {
-    "md5_hash": "dd3ac893b3b858d8f45674345dfe6fe9",
+    "md5_hash": "a7813f5053276eb22639b61c61e3e56c",
     "name": "GeoLite2-City.mmdb.gz",
-    "url": "endpoint/db.mmdb.gz",
+    "url": "endpoint/db/GeoLite2-City.mmdb.gz",
     "provider": "maxmind"
   },
   {
-    "md5_hash": "dd3ac893b3b858d8f45674345dfe6fe9",
+    "md5_hash": "f452b1aece8dc6137485ddbb0401d6fe",
     "name": "GeoLite2-ASN.mmdb.gz",
-    "url": "endpoint/db.mmdb.gz",
+    "url": "endpoint/db/GeoLite2-ASN.mmdb.gz",
     "provider": "maxmind"
   },
   {
-    "md5_hash": "dd3ac893b3b858d8f45674345dfe6fe9",
+    "md5_hash": "dc366cf57a4f101d722b4cd10d4a9825",
     "name": "GeoLite2-Country.mmdb.gz",
-    "url": "endpoint/db.mmdb.gz",
+    "url": "endpoint/db/GeoLite2-Country.mmdb.gz",
     "provider": "maxmind"
   }
 ]