DNS discovery fixes (#855)

* Imported testkit directory * Migrating tests to testkit (#832) * Migrating tests to testkit Short summary of this update: - removed migrated tests - verifyConnectivity support - resolver support - consume support Test mapping (dest: stub/routing.py): - shouldHandleAcquireReadSession -> test_should_read_successfully_from_reader_using_session_run - shouldHandleAcquireReadTransaction -> test_should_read_successfully_from_reader_using_tx_function - shouldHandleAcquireReadSessionAndTransaction -> test_should_read_successfully_from_reader_using_tx_run - shouldRoundRobinReadServers -> test_should_round_robin_readers_when_reading_using_session_run - shouldRoundRobinReadServersWhenUsingTransaction -> test_should_round_robin_readers_when_reading_using_tx_run - shouldThrowSessionExpiredIfReadServerDisappears -> test_should_fail_when_reading_from_unexpectedly_interrupting_reader_using_session_run - shouldThrowSessionExpiredIfReadServerDisappearsWhenUsingTransaction -> test_should_fail_when_reading_from_unexpectedly_interrupting_reader_using_tx_run - shouldThrowSessionExpiredIfWriteServerDisappears -> test_should_fail_when_writing_on_unexpectedly_interrupting_writer_using_session_run - shouldThrowSessionExpiredIfWriteServerDisappearsWhenUsingTransaction -> test_should_fail_when_writing_on_unexpectedly_interrupting_writer_using_tx_run - shouldHandleAcquireWriteSession -> test_should_write_successfully_on_writer_using_session_run - shouldHandleAcquireWriteTransaction -> test_should_write_successfully_on_writer_using_tx_function - shouldHandleAcquireWriteSessionAndTransaction -> test_should_write_successfully_on_writer_using_tx_run - shouldRoundRobinWriteSessions -> test_should_round_robin_writers_when_writing_using_session_run - shouldRoundRobinWriteSessionsInTransaction -> test_should_round_robin_writers_when_writing_using_tx_run - shouldFailOnNonDiscoverableServer -> test_should_fail_discovery_when_router_fails_with_procedure_not_found_code - shouldFailRandomFailureInGetServers -> test_should_fail_discovery_when_router_fails_with_unknown_code - shouldHandleLeaderSwitchWhenWriting -> test_should_fail_when_writing_on_writer_that_returns_not_a_leader_code - shouldHandleLeaderSwitchWhenWritingWithoutConsuming -> test_should_fail_when_writing_without_explicit_consumption_on_writer_that_returns_not_a_leader_code - shouldHandleLeaderSwitchWhenWritingInTransaction -> test_should_fail_when_writing_on_writer_that_returns_not_a_leader_code_using_tx_run - shouldUseWriteSessionModeAndInitialBookmark -> test_should_use_write_session_mode_and_initial_bookmark_when_writing_using_tx_run - shouldUseReadSessionModeAndInitialBookmark -> test_should_use_read_session_mode_and_initial_bookmark_when_reading_using_tx_run - shouldPassBookmarkFromTransactionToTransaction -> test_should_pass_bookmark_from_tx_to_tx_using_tx_run - shouldRetryReadTransactionUntilSuccess -> test_should_retry_read_tx_until_success - shouldRetryWriteTransactionUntilSuccess -> test_should_retry_write_tx_until_success - shouldRetryReadTransactionAndPerformRediscoveryUntilSuccess -> test_should_retry_read_tx_and_rediscovery_until_success - shouldRetryWriteTransactionAndPerformRediscoveryUntilSuccess -> test_should_retry_write_tx_and_rediscovery_until_success - shouldUseInitialRouterForRediscoveryWhenAllOtherRoutersAreDead -> test_should_use_initial_router_for_discovery_when_others_unavailable - shouldInvokeProcedureGetRoutingTableWhenServerVersionPermits -> test_should_successfully_read_from_readable_router_using_tx_function - shouldSendEmptyRoutingContextInHelloMessage -> test_should_send_empty_hello - shouldServeReadsButFailWritesWhenNoWritersAvailable -> test_should_serve_reads_and_fail_writes_when_no_writers_available - shouldAcceptRoutingTableWithoutWritersAndThenRediscover -> test_should_accept_routing_table_without_writers_and_then_rediscover - shouldTreatRoutingTableWithSingleRouterAsValid -> test_should_accept_routing_table_with_single_router - shouldSendMultipleBookmarks -> test_should_successfully_send_multiple_bookmarks - shouldForgetAddressOnDatabaseUnavailableError -> test_should_forget_address_on_database_unavailable_error - shouldUseResolverDuringRediscoveryWhenExistingRoutersFail -> test_should_use_resolver_during_rediscovery_when_existing_routers_fail - shouldRevertToInitialRouterIfKnownRouterThrowsProtocolErrors -> test_should_revert_to_initial_router_if_known_router_throws_protocol_errors * Removing redundant stub server scripts * Migrating tests to testkit part 2 (#839) - shouldSendRoutingContextToServer -> test_should_successfully_get_routing_table_with_context - shouldSendRoutingContextInHelloMessage -> test_should_successfully_get_routing_table_with_context - shouldHandleLeaderSwitchAndRetryWhenWritingInTxFunction -> test_should_write_successfully_on_leader_switch_using_tx_function - shouldSendInitialBookmark -> test_should_use_write_session_mode_and_initial_bookmark_when_writing_using_tx_run - shouldRetryWriteTransactionUntilSuccessWithWhenLeaderIsRemoved -> test_should_retry_write_until_success_with_leader_change_using_tx_function - shouldRetryWriteTransactionUntilSuccessWithWhenLeaderIsRemovedV3 -> test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function * Migrating tests to testkit part 3 (#840) Adding support for supportsMultiDB call. And exporting the following tests to testkit: - shouldServerWithBoltV4SupportMultiDb -> test_should_successfully_check_if_support_for_multi_db_is_available - shouldServerWithBoltV3NotSupportMultiDb -> test_should_successfully_check_if_support_for_multi_db_is_available Removing redundant scripts * Stub tests migration part 4 (#847) Removed RoutingDriverMultidatabaseBoltKitIT Migrated tests: - shouldDiscoverForDatabase -> test_should_read_successfully_from_reader_using_session_run (this tests seems to cover the same use-case and uses a non-default DB for 4+ versions) - shouldRetryOnEmptyDiscoveryResult -> test_should_read_successfully_on_empty_discovery_result_using_session_run - shouldThrowRoutingErrorIfDatabaseNotFound -> test_should_fail_with_routing_failure_on_db_not_found_discovery_failure - shouldBeAbleToServeReachableDatabase -> test_should_read_successfully_from_reachable_db_after_trying_unreachable_db (message check has been removed) - shouldPassSystemBookmarkWhenGettingRoutingTableForMultiDB -> test_should_pass_system_bookmark_when_getting_rt_for_multi_db (seems to be applicable to V4 only, also the stub server doesn't seem to check bookmarks) - shouldIgnoreSystemBookmarkWhenGettingRoutingTable -> test_should_ignore_system_bookmark_when_getting_rt_for_multi_db - shouldDriverVerifyConnectivity -> test_should_successfully_get_routing_table_with_context (pre-existing test that already tests the connectivity) Also removed redundant scripts and added code support to DriverError * Fix for initial routers DNS resolution (#849) * Fix for initial routers DNS resolution This update fixes the following issue: #833 The desired behaviour for getting a routing table from the initial router (either on bootstrap or when all known routers have failed) is: - resolve the domain name to all IPs - attempt getting a routing table from all of them until first one succeeds by: - getting a connection - trying to get a successful routing table response Prior to this change, the connection pools were created for host and port pairs. When domain name of the host resolves to multiple IP addresses, such pools provide connections to those IPs as a group. While this works for readers and writers, it negatively impacts the routing table fetching process as there is no guarantee which IP address the provided connection is setup for. This update delivers the following changes: - connection pools for routers are IP address based, which allows for deterministic connection retrieval - the resolved IP address set is kept up-to-date (in case known router IPs change) to make sure that the unused connection pools are flushed - the domain name resolution logic has been made configurable (it is private at the moment and is used to facilitate testing) - the testkit backend has been updated to support the domain name resolution configuration (a new test has been added to testkit to cover the issue described above) - the testkit backend has been updated to support connection timeout driver configuration - several tests have been updated to adopt the new changes * Updating test name * Fixed SSL handling (#851) This update fixes a number of SSL-related tests in testkit and CausalClusteringIT.shouldDropBrokenOldConnections test. The connection pooling strategy has been updated to use the same connection pool when the connection host is unambiguous. Removed hardcoded domain name resolution from the BoltServerAddress and moved the logic to ChannelConnectorImpl that uses the DomainNameResolver. * Updated .gitignore
neo4j · Mar 23, 2021 · e5e8ab3 · e5e8ab3
1 parent 10da2e2
commit e5e8ab3
Show file tree

Hide file tree

Showing 93 changed files with 1,569 additions and 2,261 deletions.
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,5 @@ enterprise/server-enterprise/neo4j-home
 integrationtests/data
 dependency-reduced-pom.xml
 venv
+testkit-backend/bin/
+testkit/CAs/
diff --git a/driver/src/main/java/org/neo4j/driver/internal/BoltServerAddress.java b/driver/src/main/java/org/neo4j/driver/internal/BoltServerAddress.java
@@ -18,19 +18,13 @@
  */
 package org.neo4j.driver.internal;
 
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
 import java.net.URI;
-import java.net.UnknownHostException;
-import java.util.List;
 import java.util.Objects;
 import java.util.stream.Stream;
 
 import org.neo4j.driver.net.ServerAddress;
 
 import static java.util.Objects.requireNonNull;
-import static java.util.stream.Collectors.toList;
 
 /**
  * Holds a host and port pair that denotes a Bolt server address.
@@ -40,12 +34,11 @@ public class BoltServerAddress implements ServerAddress
     public static final int DEFAULT_PORT = 7687;
     public static final BoltServerAddress LOCAL_DEFAULT = new BoltServerAddress( "localhost", DEFAULT_PORT );
 
-    private final String host; // This could either be the same as originalHost or it is an IP address resolved from the original host.
-    private final int port;
+    protected final String host; // Host or IP address.
+    private final String connectionHost; // Either is equal to the host or is explicitly provided on creation and is expected to be a resolved IP address.
+    protected final int port;
     private final String stringValue;
 
-    private InetAddress resolved;
-
     public BoltServerAddress( String address )
     {
         this( uriFrom( address ) );
@@ -58,15 +51,17 @@ public BoltServerAddress( URI uri )
 
     public BoltServerAddress( String host, int port )
     {
-        this( host, null, port );
+        this( host, host, port );
     }
 
-    private BoltServerAddress( String host, InetAddress resolved, int port )
+    public BoltServerAddress( String host, String connectionHost, int port )
     {
         this.host = requireNonNull( host, "host" );
-        this.resolved = resolved;
+        this.connectionHost = requireNonNull( connectionHost, "connectionHost" );
         this.port = requireValidPort( port );
-        this.stringValue = resolved != null ? String.format( "%s(%s):%d", host, resolved.getHostAddress(), port ) : String.format( "%s:%d", host, port );
+        this.stringValue = host.equals( connectionHost )
+                           ? String.format( "%s:%d", host, port )
+                           : String.format( "%s(%s):%d", host, connectionHost, port );
     }
 
     public static BoltServerAddress from( ServerAddress address )
@@ -87,14 +82,14 @@ public boolean equals( Object o )
         {
             return false;
         }
-        BoltServerAddress that = (BoltServerAddress) o;
-        return port == that.port && host.equals( that.host );
+        BoltServerAddress address = (BoltServerAddress) o;
+        return port == address.port && host.equals( address.host ) && connectionHost.equals( address.connectionHost );
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hash( host, port );
+        return Objects.hash( host, connectionHost, port );
     }
 
     @Override
@@ -103,44 +98,6 @@ public String toString()
         return stringValue;
     }
 
-    /**
-     * Create a {@link SocketAddress} from this bolt address. This method always attempts to resolve the hostname into
-     * an {@link InetAddress}.
-     *
-     * @return new socket address.
-     * @see InetSocketAddress
-     */
-    public SocketAddress toSocketAddress()
-    {
-        return resolved == null ? new InetSocketAddress( host, port ) : new InetSocketAddress( resolved, port );
-    }
-
-    /**
-     * Resolve the host name down to an IP address
-     *
-     * @return a new address instance
-     * @throws UnknownHostException if no IP address for the host could be found
-     * @see InetAddress#getByName(String)
-     */
-    public BoltServerAddress resolve() throws UnknownHostException
-    {
-        return new BoltServerAddress( host, InetAddress.getByName( host ), port );
-    }
-
-    /**
-     * Resolve the host name down to all IP addresses that can be resolved to
-     *
-     * @return an array of new address instances that holds resolved addresses
-     * @throws UnknownHostException if no IP address for the host could be found
-     * @see InetAddress#getAllByName(String)
-     */
-    public List<BoltServerAddress> resolveAll() throws UnknownHostException
-    {
-        return Stream.of( InetAddress.getAllByName( host ) )
-                .map( address -> new BoltServerAddress( host, address, port ) )
-                .collect( toList() );
-    }
-
     @Override
     public String host()
     {
@@ -153,9 +110,21 @@ public int port()
         return port;
     }
 
-    public boolean isResolved()
+    public String connectionHost()
+    {
+        return connectionHost;
+    }
+
+    /**
+     * Create a stream of unicast addresses.
+     * <p>
+     * While this implementation just returns a stream of itself, the subclasses may provide multiple addresses.
+     *
+     * @return stream of unicast addresses.
+     */
+    public Stream<BoltServerAddress> unicastStream()
     {
-        return resolved != null;
+        return Stream.of( this );
     }
 
     private static String hostFrom( URI uri )

diff --git a/driver/src/main/java/org/neo4j/driver/internal/DefaultDomainNameResolver.java b/driver/src/main/java/org/neo4j/driver/internal/DefaultDomainNameResolver.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.driver.internal;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+public class DefaultDomainNameResolver implements DomainNameResolver
+{
+    private static final DefaultDomainNameResolver INSTANCE = new DefaultDomainNameResolver();
+
+    public static DefaultDomainNameResolver getInstance()
+    {
+        return INSTANCE;
+    }
+
+    private DefaultDomainNameResolver()
+    {
+    }
+
+    @Override
+    public InetAddress[] resolve( String name ) throws UnknownHostException
+    {
+        return InetAddress.getAllByName( name );
+    }
+}
diff --git a/driver/src/main/java/org/neo4j/driver/internal/DomainNameResolver.java b/driver/src/main/java/org/neo4j/driver/internal/DomainNameResolver.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.driver.internal;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * A resolver function used by the driver to resolve domain names.
+ */
+@FunctionalInterface
+public interface DomainNameResolver
+{
+    /**
+     * Resolve the given domain name to a set of addresses.
+     *
+     * @param name the name to resolve.
+     * @return the resolved addresses.
+     * @throws UnknownHostException must be thrown if the given name can not be resolved to at least one address.
+     */
+    InetAddress[] resolve( String name ) throws UnknownHostException;
+}
diff --git a/driver/src/main/java/org/neo4j/driver/internal/DriverFactory.java b/driver/src/main/java/org/neo4j/driver/internal/DriverFactory.java
@@ -127,7 +127,7 @@ protected static MetricsProvider createDriverMetrics( Config config, Clock clock
     protected ChannelConnector createConnector( ConnectionSettings settings, SecurityPlan securityPlan,
             Config config, Clock clock, RoutingContext routingContext )
     {
-        return new ChannelConnectorImpl( settings, securityPlan, config.logging(), clock, routingContext );
+        return new ChannelConnectorImpl( settings, securityPlan, config.logging(), clock, routingContext, getDomainNameResolver() );
     }
 
     private InternalDriver createDriver( URI uri, SecurityPlan securityPlan, BoltServerAddress address, ConnectionPool connectionPool,
@@ -210,7 +210,7 @@ protected LoadBalancer createLoadBalancer( BoltServerAddress address, Connection
         LoadBalancingStrategy loadBalancingStrategy = new LeastConnectedLoadBalancingStrategy( connectionPool, config.logging() );
         ServerAddressResolver resolver = createResolver( config );
         return new LoadBalancer( address, routingSettings, connectionPool, eventExecutorGroup, createClock(),
-                config.logging(), loadBalancingStrategy, resolver );
+                                 config.logging(), loadBalancingStrategy, resolver, getDomainNameResolver() );
     }
 
     private static ServerAddressResolver createResolver( Config config )
@@ -271,6 +271,17 @@ protected Bootstrap createBootstrap( EventLoopGroup eventLoopGroup )
         return BootstrapFactory.newBootstrap( eventLoopGroup );
     }
 
+    /**
+     * Provides an instance of {@link DomainNameResolver} that is used for domain name resolution.
+     * <p>
+     * <b>This method is protected only for testing</b>
+     *
+     * @return the instance of {@link DomainNameResolver}.
+     */
+    protected DomainNameResolver getDomainNameResolver()
+    {
+        return DefaultDomainNameResolver.getInstance();
+    }
 
     private static void assertNoRoutingContext( URI uri, RoutingSettings routingSettings )
     {

diff --git a/driver/src/main/java/org/neo4j/driver/internal/ResolvedBoltServerAddress.java b/driver/src/main/java/org/neo4j/driver/internal/ResolvedBoltServerAddress.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.driver.internal;
+
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.joining;
+
+/**
+ * An explicitly resolved version of {@link BoltServerAddress} that always contains one or more resolved IP addresses.
+ */
+public class ResolvedBoltServerAddress extends BoltServerAddress
+{
+    private static final String HOST_ADDRESSES_FORMAT = "%s%s:%d";
+    private static final int MAX_HOST_ADDRESSES_IN_STRING_VALUE = 5;
+    private static final String HOST_ADDRESS_DELIMITER = ",";
+    private static final String HOST_ADDRESSES_PREFIX = "(";
+    private static final String HOST_ADDRESSES_SUFFIX = ")";
+    private static final String TRIMMED_HOST_ADDRESSES_SUFFIX = ",..." + HOST_ADDRESSES_SUFFIX;
+
+    private final Set<InetAddress> resolvedAddresses;
+    private final String stringValue;
+
+    public ResolvedBoltServerAddress( String host, int port, InetAddress[] resolvedAddressesArr )
+    {
+        super( host, port );
+        requireNonNull( resolvedAddressesArr, "resolvedAddressesArr" );
+        if ( resolvedAddressesArr.length == 0 )
+        {
+            throw new IllegalArgumentException(
+                    "The resolvedAddressesArr must not be empty, check your DomainNameResolver is compliant with the interface contract" );
+        }
+        resolvedAddresses = Collections.unmodifiableSet( new LinkedHashSet<>( Arrays.asList( resolvedAddressesArr ) ) );
+        stringValue = createStringRepresentation();
+    }
+
+    /**
+     * Create a stream of unicast addresses.
+     * <p>
+     * The stream is created from the list of resolved IP addresses. Each unicast address is given a unique IP address as the connectionHost value.
+     *
+     * @return stream of unicast addresses.
+     */
+    @Override
+    public Stream<BoltServerAddress> unicastStream()
+    {
+        return resolvedAddresses.stream().map( address -> new BoltServerAddress( host, address.getHostAddress(), port ) );
+    }
+
+    @Override
+    public boolean equals( Object o )
+    {
+        if ( this == o )
+        {
+            return true;
+        }
+        if ( o == null || getClass() != o.getClass() )
+        {
+            return false;
+        }
+        if ( !super.equals( o ) )
+        {
+            return false;
+        }
+        ResolvedBoltServerAddress that = (ResolvedBoltServerAddress) o;
+        return resolvedAddresses.equals( that.resolvedAddresses );
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash( super.hashCode(), resolvedAddresses );
+    }
+
+    @Override
+    public String toString()
+    {
+        return stringValue;
+    }
+
+    private String createStringRepresentation()
+    {
+        String hostAddresses = resolvedAddresses.stream()
+                                                .limit( MAX_HOST_ADDRESSES_IN_STRING_VALUE )
+                                                .map( InetAddress::getHostAddress )
+                                                .collect( joining( HOST_ADDRESS_DELIMITER, HOST_ADDRESSES_PREFIX,
+                                                                   resolvedAddresses.size() > MAX_HOST_ADDRESSES_IN_STRING_VALUE
+                                                                   ? TRIMMED_HOST_ADDRESSES_SUFFIX
+                                                                   : HOST_ADDRESSES_SUFFIX ) );
+        return String.format( HOST_ADDRESSES_FORMAT, host, hostAddresses, port );
+    }
+}