Use default fallback hosts when custom environment set

lib/ably/modules/ably.rb

@@ -6,8 +6,18 @@
 module Ably
   # Fallback hosts to use when a connection to rest/realtime.ably.io is not possible due to
   # network failures either at the client, between the client and Ably, within an Ably data center, or at the IO domain registrar
+  # see https://docs.ably.io/client-lib-development-guide/features/#RSC15a
   #
-  FALLBACK_HOSTS = %w(A.ably-realtime.com B.ably-realtime.com C.ably-realtime.com D.ably-realtime.com E.ably-realtime.com).freeze
+  FALLBACK_DOMAIN = 'ably-realtime.com'.freeze
+  FALLBACK_IDS = %w(a b c d e).freeze
+
+  # Default production fallbacks a.ably-realtime.com ... e.ably-realtime.com
+  FALLBACK_HOSTS = FALLBACK_IDS.map { |host| "#{host}.#{FALLBACK_DOMAIN}".freeze }.freeze
+
+  # Custom environment default fallbacks {ENV}-a-fallback.ably-realtime.com ... {ENV}-a-fallback.ably-realtime.com
+  CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES = FALLBACK_IDS.map do |host|
+    "-#{host}-fallback.#{FALLBACK_DOMAIN}".freeze
+  end.freeze
 
   INTERNET_CHECK = {
     url:     '//internet-up.ably-realtime.com/is-the-internet-up.txt',

lib/ably/realtime/client.rb

@@ -74,6 +74,7 @@ class Client
       def_delegators :@rest_client, :use_tls?, :protocol, :protocol_binary?
       def_delegators :@rest_client, :environment, :custom_host, :custom_port, :custom_tls_port
       def_delegators :@rest_client, :log_level
+      def_delegators :@rest_client, :options
 
       # Creates a {Ably::Realtime::Client Realtime Client} and configures the {Ably::Auth} object for the connection.
       #

lib/ably/rest/client.rb

@@ -181,16 +181,18 @@ def initialize(options)
         @idempotent_rest_publishing = options.delete(:idempotent_rest_publishing) || Ably.major_minor_version_numeric > 1.1
 
 
-        if options[:fallback_hosts_use_default] && options[:fallback_jhosts]
-          raise ArgumentError, "fallback_hosts_use_default cannot be set to trye when fallback_jhosts is also provided"
+        if options[:fallback_hosts_use_default] && options[:fallback_hosts]
+          raise ArgumentError, "fallback_hosts_use_default cannot be set to try when fallback_hosts is also provided"
         end
         @fallback_hosts = case
         when options.delete(:fallback_hosts_use_default)
           Ably::FALLBACK_HOSTS
         when options_fallback_hosts = options.delete(:fallback_hosts)
           options_fallback_hosts
-        when environment || custom_host || options[:realtime_host] || custom_port || custom_tls_port
+        when custom_host || options[:realtime_host] || custom_port || custom_tls_port
           []
+        when environment
+          CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |host| "#{environment}#{host}" }
         else
           Ably::FALLBACK_HOSTS
         end

spec/acceptance/realtime/connection_failures_spec.rb

@@ -156,10 +156,20 @@
 
               stub_request(:get, auth_url).
                 to_return do |request|
-                  sleep Ably::Rest::Client::HTTP_DEFAULTS.fetch(:request_timeout)
-                  { status: [500, "Internal Server Error"] }
-                end.then.
-                to_return(:status => 201, :body => token_response.to_json, :headers => { 'Content-Type' => 'application/json' })
+                sleep Ably::Rest::Client::HTTP_DEFAULTS.fetch(:request_timeout)
+                { status: [500, "Internal Server Error"] }
+              end.then.
+              to_return(:status => 201, :body => token_response.to_json, :headers => { 'Content-Type' => 'application/json' })
+
+              stub_request(:get, 'https://internet-up.ably-realtime.com/is-the-internet-up.txt')
+                .with(
+                  headers: {
+                    'Accept-Encoding' => 'gzip, compressed',
+                    'Connection' => 'close',
+                    'Host' => 'internet-up.ably-realtime.com',
+                    'User-Agent' => 'EventMachine HttpClient'
+                  }
+                ).to_return(status: 200, body: 'yes\n', headers: { 'Content-Type' => 'text/plain' })
             end
 
             specify 'the connection moves to the disconnected state and tries again, returning again to the disconnected state (#RSA4c, #RSA4c1, #RSA4c2)' do
@@ -1423,14 +1433,19 @@ def kill_connection_transport_and_prevent_valid_resume
         let(:expected_host)  { "#{environment}-#{Ably::Realtime::Client::DOMAIN}" }
         let(:client_options) { timeout_options.merge(environment: environment) }
 
-        it 'does not use a fallback host by default' do
-          expect(connection).to receive(:create_transport).exactly(retry_count_for_all_states).times do |host|
-            expect(host).to eql(expected_host)
-            raise EventMachine::ConnectionError
-          end
+        context ':fallback_hosts_use_default is unset' do
+          let(:max_time_in_state_for_tests) { 8 }
+          let(:expected_hosts) { Ably::CUSTOM_ENVIRONMENT_FALLBACKS_SUFFIXES.map { |suffix| "#{environment}#{suffix}" } + [expected_host] }
+          let(:fallback_hosts_used) { Array.new }
+
+          it 'uses fallback hosts by default' do
+            allow(connection).to receive(:create_transport) do |host|
+              fallback_hosts_used << host
+              raise EventMachine::ConnectionError
+            end
 
-          connection.once(:suspended) do
             connection.once(:suspended) do
+              expect(fallback_hosts_used.uniq).to match_array(expected_hosts)
               stop_reactor
             end
           end
@@ -1508,7 +1523,7 @@ def kill_connection_transport_and_prevent_valid_resume
       end
 
       context 'with production environment' do
-        let(:custom_hosts)   { %w(A.ably-realtime.com B.ably-realtime.com) }
+        let(:custom_hosts)   { %w(a.ably-realtime.com b.ably-realtime.com) }
         before do
           stub_const 'Ably::FALLBACK_HOSTS', custom_hosts
         end

spec/acceptance/realtime/connection_spec.rb

@@ -122,7 +122,7 @@
                 end
               end
 
-              context 'with immediately expired token' do
+              context 'with immediately expired token and no fallback hosts' do
                 let(:ttl) { 0.001 }
                 let(:auth_requests) { [] }
                 let(:token_callback) do
@@ -131,7 +131,7 @@
                     Ably::Rest::Client.new(default_options).auth.request_token(ttl: ttl).token
                   end
                 end
-                let(:client_options) { default_options.merge(auth_callback: token_callback) }
+                let(:client_options) { default_options.merge(auth_callback: token_callback, fallback_hosts: []) }
 
                 it 'renews the token on connect, and makes one immediate subsequent attempt to obtain a new token (#RSA4b)' do
                   started_at = Time.now.to_f
@@ -146,7 +146,7 @@
                 end
 
                 context 'when disconnected_retry_timeout is 0.5 seconds' do
-                  let(:client_options) { default_options.merge(disconnected_retry_timeout: 0.5, auth_callback: token_callback) }
+                  let(:client_options) { default_options.merge(disconnected_retry_timeout: 0.5, auth_callback: token_callback, fallback_hosts: []) }
 
                   it 'renews the token on connect, and continues to attempt renew based on the retry schedule' do
                     disconnect_count = 0
@@ -172,7 +172,7 @@
                 end
 
                 context 'using implicit token auth' do
-                  let(:client_options) { default_options.merge(use_token_auth: true, default_token_params: { ttl: ttl }) }
+                  let(:client_options) { default_options.merge(use_token_auth: true, default_token_params: { ttl: ttl }, fallback_hosts: []) }
 
                   before do
                     stub_const 'Ably::Models::TokenDetails::TOKEN_EXPIRY_BUFFER', -10 # ensure client lib thinks token is still valid
@@ -441,7 +441,9 @@ def expect_ordered_phases
       end
     end
 
-    context '#connect' do
+    context '#connect with no fallbacks' do
+      let(:client_options)   { default_options.merge(fallback_hosts: []) }
+
       it 'returns a SafeDeferrable that catches exceptions in callbacks and logs them' do
         expect(connection.connect).to be_a(Ably::Util::SafeDeferrable)
         stop_reactor
@@ -1167,6 +1169,7 @@ def self.available_states
                 host:   'this.host.does.not.exist.com'
               )
             )
+            allow(client).to receive(:fallback_hosts).and_return([])
 
             connection.transition_state_machine! :disconnected
           end

spec/acceptance/rest/base_spec.rb

@@ -87,8 +87,10 @@
         let(:error_response) { '{ "error": { "statusCode": 500, "code": 50000, "message": "Internal error" } }' }
 
         before do
-          stub_request(:get, "#{client.endpoint}/time").
-            to_return(:status => 500, :body => error_response, :headers => { 'Content-Type' => 'application/json' })
+          (client.fallback_hosts.map { |host| "https://#{host}" } + [client.endpoint]).each do |host|
+            stub_request(:get, "#{host}/time")
+              .to_return(:status => 500, :body => error_response, :headers => { 'Content-Type' => 'application/json' })
+          end
         end
 
         it 'should raise a ServerError exception' do
@@ -98,8 +100,10 @@
 
       describe '500 server error without a valid JSON response body', :webmock do
         before do
-          stub_request(:get, "#{client.endpoint}/time").
-          to_return(:status => 500, :headers => { 'Content-Type' => 'application/json' })
+          (client.fallback_hosts.map { |host| "https://#{host}" } + [client.endpoint]).each do |host|
+            stub_request(:get, "#{host}/time").
+            to_return(:status => 500, :headers => { 'Content-Type' => 'application/json' })
+          end
         end
 
         it 'should raise a ServerError exception' do

spec/acceptance/rest/client_spec.rb

@@ -301,30 +301,44 @@ def encode64(text)
       context 'configured' do
         let(:client_options) { default_options.merge(key: api_key, environment: 'production') }
 
-        it 'should make connection attempts to A.ably-realtime.com, B.ably-realtime.com, C.ably-realtime.com, D.ably-realtime.com, E.ably-realtime.com (#RSC15a)' do
+        it 'should make connection attempts to a.ably-realtime.com, b.ably-realtime.com, c.ably-realtime.com, d.ably-realtime.com, e.ably-realtime.com (#RSC15a)' do
           hosts = []
           5.times do
             hosts << client.fallback_connection.host
           end
-          expect(hosts).to match_array(%w(A.ably-realtime.com B.ably-realtime.com C.ably-realtime.com D.ably-realtime.com E.ably-realtime.com))
+          expect(hosts).to match_array(%w(a.ably-realtime.com b.ably-realtime.com c.ably-realtime.com d.ably-realtime.com e.ably-realtime.com))
         end
       end
 
       context 'when environment is NOT production (#RSC15b)' do
-        let(:client_options) { default_options.merge(environment: 'sandbox', key: api_key) }
-        let!(:default_host_request_stub) do
-          stub_request(:post, "https://#{environment}-#{Ably::Rest::Client::DOMAIN}#{path}").to_return do
-            raise Faraday::TimeoutError.new('timeout error message')
+        context 'and custom fallback hosts are empty' do
+          let(:client_options) { default_options.merge(environment: 'sandbox', key: api_key, fallback_hosts: []) }
+          let!(:default_host_request_stub) do
+            stub_request(:post, "https://#{environment}-#{Ably::Rest::Client::DOMAIN}#{path}").to_return do
+              raise Faraday::TimeoutError.new('timeout error message')
+            end
+          end
+
+          it 'does not retry failed requests with fallback hosts when there is a connection error' do
+            expect { publish_block.call }.to raise_error Ably::Exceptions::ConnectionTimeout
           end
         end
 
-        it 'does not retry failed requests with fallback hosts when there is a connection error' do
-          expect { publish_block.call }.to raise_error Ably::Exceptions::ConnectionTimeout
+        context 'and no custom fallback hosts are provided' do
+          let(:client_options) { default_options.merge(environment: 'sandbox', key: api_key) }
+
+          it 'should make connection attempts to sandbox-a-fallback.ably-realtime.com, sandbox-b-fallback.ably-realtime.com, sandbox-c-fallback.ably-realtime.com, sandbox-d-fallback.ably-realtime.com, sandbox-e-fallback.ably-realtime.com (#RSC15a)' do
+            hosts = []
+            5.times do
+              hosts << client.fallback_connection.host
+            end
+            expect(hosts).to match_array(%w(a b c d e).map { |id| "sandbox-#{id}-fallback.ably-realtime.com" })
+          end
         end
       end
 
       context 'when environment is production' do
-        let(:custom_hosts)       { %w(A.ably-realtime.com B.ably-realtime.com) }
+        let(:custom_hosts)       { %w(a.ably-realtime.com b.ably-realtime.com) }
         let(:max_retry_count)    { 2 }
         let(:max_retry_duration) { 0.5 }
         let(:fallback_block)     { proc { raise Faraday::SSLError.new('ssl error message') } }
@@ -823,11 +837,12 @@ def encode64(text)
       end
 
       context 'when environment is not production and server returns a 50x error' do
+        let(:env)                { 'custom-env' }
+        let(:default_fallbacks)  { %w(a b c d e).map { |id| "#{env}-#{id}-fallback.ably-realtime.com" } }
         let(:custom_hosts)       { %w(A.foo.com B.foo.com) }
         let(:max_retry_count)    { 2 }
         let(:max_retry_duration) { 0.5 }
         let(:fallback_block)     { proc { raise Faraday::SSLError.new('ssl error message') } }
-        let(:env)                { 'custom-env' }
         let(:production_options) do
           default_options.merge(
             environment: env,
@@ -851,6 +866,26 @@ def encode64(text)
           stub_request(:post, "https://#{env}-#{Ably::Rest::Client::DOMAIN}#{path}").to_return(&fallback_block)
         end
 
+        context 'with no fallback hosts provided (#TBC, see https://github.com/ably/wiki/issues/361)' do
+          let(:client_options) {
+            production_options.merge(log_level: :fatal)
+          }
+
+          it 'uses the default fallback hosts for that environment as this is not an authentication failure' do
+            fallbacks_called_count = 0
+            default_fallbacks.each do |host|
+              counting_fallback_proc = proc do
+                fallbacks_called_count += 1
+                fallback_block.call
+              end
+              stub_request(:post, "https://#{host}#{path}").to_return(&counting_fallback_proc)
+            end
+            expect { publish_block.call }.to raise_error(Ably::Exceptions::ServerError)
+            expect(default_host_request_stub).to have_been_requested
+            expect(fallbacks_called_count).to be >= 2
+          end
+        end
+
         context 'with custom fallback hosts provided (#RSC15b, #TO3k6)' do
           let!(:first_fallback_request_stub) do
             stub_request(:post, "https://#{custom_hosts[0]}#{path}").to_return(&fallback_block)
@@ -1194,9 +1229,9 @@ def encode64(text)
 
     context 'request_id generation' do
       context 'Timeout error' do
-        context 'with option add_request_ids: true', :webmock, :prevent_log_stubbing do
+        context 'with option add_request_ids: true and no fallback hosts', :webmock, :prevent_log_stubbing do
           let(:custom_logger_object) { TestLogger.new }
-          let(:client_options) { default_options.merge(key: api_key, logger: custom_logger_object, add_request_ids: true) }
+          let(:client_options) { default_options.merge(key: api_key, logger: custom_logger_object, add_request_ids: true, fallback_hosts: []) }
 
           before do
             @request_id = nil
@@ -1286,8 +1321,8 @@ def encode64(text)
           end
         end
 
-        context 'without request_id' do
-          let(:client_options) { default_options.merge(key: api_key, http_request_timeout: 0) }
+        context 'without request_id and no fallback hosts' do
+          let(:client_options) { default_options.merge(key: api_key, http_request_timeout: 0, fallback_hosts: []) }
 
           it 'does not include request_id in ConnectionTimeout error' do
             begin