Merge pull request #465 from Shopify/dalli-support

Add support for the default MemCacheStore from ActiveSupport

.gitignore

@@ -16,3 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 .rubocop-http*
+.byebug_history

.travis.yml

@@ -13,6 +13,7 @@ gemfile:
 env:
   - DB=mysql2
   - DB=postgresql
+  - DB=mysql2 ADAPTER=memcached
 
 jobs:
   exclude:

README.md

@@ -12,7 +12,10 @@ Add this line to your application's Gemfile:
 ```ruby
 gem 'identity_cache'
 gem 'cityhash'        # optional, for faster hashing (C-Ruby only)
-gem 'memcached_store' # for CAS support, needed for cache consistency
+
+gem 'dalli' # To use :mem_cache_store
+# alternatively
+gem 'memcached_store' # to use the old libmemcached based client
 ```
 
 And then execute:
@@ -22,6 +25,24 @@ And then execute:
 
 Add the following to all your environment/*.rb files (production/development/test):
 
+### If you use Dalli (recommended)
+
+```ruby
+config.identity_cache_store = :mem_cache_store, "mem1.server.com", "mem2.server.com", {
+  expires_in: 6.hours.to_i, # in case of network errors when sending a delete
+  failover: false, # avoids more cache consistency issues
+}
+```
+
+Add an initializer with this code:
+
+```ruby
+IdentityCache.cache_backend = ActiveSupport::Cache.lookup_store(*Rails.configuration.identity_cache_store)
+```
+
+
+### If you use Memcached (old client)
+
 ```ruby
 config.identity_cache_store = :memcached_store,
   Memcached.new(["mem1.server.com"],

identity_cache.gemspec

@@ -38,6 +38,7 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency('memcached', '~> 1.8.0')
   gem.add_development_dependency('memcached_store', '~> 1.0.0')
+  gem.add_development_dependency('dalli')
   gem.add_development_dependency('rake')
   gem.add_development_dependency('mocha', '0.14.0')
   gem.add_development_dependency('spy')

lib/identity_cache.rb

@@ -44,6 +44,8 @@
 module IdentityCache
   extend ActiveSupport::Concern
 
+  autoload :MemCacheStoreCAS, 'identity_cache/mem_cache_store_cas'
+
   include WithPrimaryIndex
 
   CACHED_NIL = :idc_cached_nil

lib/identity_cache/mem_cache_store_cas.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+require 'dalli/cas/client'
+
+module IdentityCache
+  module MemCacheStoreCAS
+    def cas(name, options = nil)
+      options = merged_options(options)
+      key = normalize_key(name, options)
+
+      rescue_error_with(false) do
+        instrument(:cas, key, options) do
+          @data.with do |connection|
+            connection.cas(key, options[:expires_in].to_i, options) do |raw_value|
+              entry = deserialize_entry(raw_value)
+              value = yield entry.value
+              entry = ActiveSupport::Cache::Entry.new(value, **options)
+              options[:raw] ? entry.value.to_s : entry
+            end
+          end
+        end
+      end
+    end
+
+    def cas_multi(*names, **options)
+      return if names.empty?
+
+      options = merged_options(options)
+      keys_to_names = names.each_with_object({}) { |name, hash| hash[normalize_key(name, options)] = name }
+      keys = keys_to_names.keys
+      rescue_error_with(false) do
+        instrument(:cas_multi, keys, options) do
+          raw_values = @data.get_multi_cas(keys)
+
+          values = {}
+          raw_values.each do |key, raw_value|
+            entry = deserialize_entry(raw_value.first)
+            values[keys_to_names[key]] = entry.value unless entry.expired?
+          end
+
+          updates = yield values
+
+          updates.each do |name, value|
+            key = normalize_key(name, options)
+            cas_id = raw_values[key].last
+            entry = ActiveSupport::Cache::Entry.new(value, **options)
+            payload = options[:raw] ? entry.value.to_s : entry
+            @data.replace_cas(key, payload, options[:expires_in].to_i, cas_id, options)
+          end
+        end
+      end
+    end
+  end
+end

lib/identity_cache/memoized_cache_proxy.rb

@@ -12,6 +12,16 @@ def initialize(cache_adaptor = nil)
     end
 
     def cache_backend=(cache_adaptor)
+      if cache_adaptor.class.name == 'ActiveSupport::Cache::MemCacheStore'
+        if cache_adaptor.respond_to?(:cas) || cache_adaptor.respond_to?(:cas_multi)
+          unless cache_adaptor.is_a?(MemCacheStoreCAS)
+            raise "#{cache_adaptor} respond to :cas or :cas_multi, that's unexpected"
+          end
+        else
+          cache_adaptor.extend(MemCacheStoreCAS)
+        end
+      end
+
       if cache_adaptor.respond_to?(:cas) && cache_adaptor.respond_to?(:cas_multi)
         @cache_fetcher = CacheFetcher.new(cache_adaptor)
       else

test/helpers/cache_connection.rb

@@ -2,12 +2,32 @@
 module CacheConnection
   extend self
 
+  # This patches AR::MemcacheStore to notify AS::Notifications upon read_multis like the rest of rails does
+  module MemcachedStoreInstrumentation
+    def read_multi(*args, &block)
+      instrument('read_multi', 'MULTI', keys: args) do
+        super(*args, &block)
+      end
+    end
+  end
+
   def host
     ENV['MEMCACHED_HOST'] || "127.0.0.1"
   end
 
   def backend
-    @backend ||= ActiveSupport::Cache::MemcachedStore.new("#{host}:11211", support_cas: true)
+    @backend ||= case ENV['ADAPTER']
+    when nil, 'dalli'
+      require 'active_support/cache/mem_cache_store'
+      ActiveSupport::Cache::MemCacheStore.new("#{host}:11211", failover: false)
+    when 'memcached'
+      require 'memcached_store'
+      require 'active_support/cache/memcached_store'
+      ActiveSupport::Cache::MemcachedStore.prepend(MemcachedStoreInstrumentation)
+      ActiveSupport::Cache::MemcachedStore.new("#{host}:11211", support_cas: true, auto_eject_hosts: false)
+    else
+      raise "Unknown adapter: #{ENV['ADAPTER']}"
+    end
   end
 
   def setup

test/helpers/database_connection.rb

@@ -35,9 +35,14 @@ def self.create_tables
     TABLES.each do |table, fields|
       fields = fields.dup
       options = fields.last.is_a?(Hash) ? fields.pop : {}
-      ActiveRecord::Base.connection.create_table(table, options) do |t|
+      ActiveRecord::Base.connection.create_table(table, **options) do |t|
         fields.each do |column_type, *args|
-          t.send(column_type, *args)
+          if args.last.is_a?(Hash)
+            kwargs = args.pop
+            t.send(column_type, *args, **kwargs)
+          else
+            t.send(column_type, *args)
+          end
         end
       end
     end

test/test_helper.rb

@@ -7,24 +7,12 @@
 require 'helpers/cache_connection'
 require 'helpers/active_record_objects'
 require 'spy/integration'
-require 'memcached_store'
-require 'active_support/cache/memcached_store'
 
 require File.dirname(__FILE__) + '/../lib/identity_cache'
 
 DatabaseConnection.setup
 CacheConnection.setup
 
-# This patches AR::MemcacheStore to notify AS::Notifications upon read_multis like the rest of rails does
-module MemcachedStoreInstrumentation
-  def read_multi(*args, &block)
-    instrument('read_multi', 'MULTI', keys: args) do
-      super(*args, &block)
-    end
-  end
-end
-ActiveSupport::Cache::MemcachedStore.prepend(MemcachedStoreInstrumentation)
-
 MiniTest::Test = MiniTest::Unit::TestCase unless defined?(MiniTest::Test)
 module IdentityCache
   class TestCase < Minitest::Test