From c1b67bdb5246273787625434a61e0c15ffa0d9fc Mon Sep 17 00:00:00 2001 From: m4cteru Date: Tue, 24 Oct 2017 17:09:50 +0900 Subject: [PATCH 01/11] add redis sequencer --- lib/active_record/turntable/sequencer.rb | 17 +++++++--- .../turntable/sequencer/redis.rb | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 lib/active_record/turntable/sequencer/redis.rb diff --git a/lib/active_record/turntable/sequencer.rb b/lib/active_record/turntable/sequencer.rb index f8046519..92ea2813 100644 --- a/lib/active_record/turntable/sequencer.rb +++ b/lib/active_record/turntable/sequencer.rb @@ -7,9 +7,11 @@ module ActiveRecord::Turntable class Sequencer autoload :Api, "active_record/turntable/sequencer/api" autoload :Mysql, "active_record/turntable/sequencer/mysql" + autoload :Redis, "active_record/turntable/sequencer/redis" @@sequence_types = { :api => Api, - :mysql => Mysql + :mysql => Mysql, + :redis => Redis } @@sequences = {} @@ -17,10 +19,15 @@ class Sequencer cattr_reader :sequences, :tables def self.build(klass) - seq_config_name = ActiveRecord::Base.turntable_config["clusters"][klass.turntable_cluster_name.to_s]["seq"]["connection"] - seq_config = ActiveRecord::Base.configurations[ActiveRecord::Turntable::RackupFramework.env]["seq"][seq_config_name] + seq_config = ActiveRecord::Base.turntable_config["clusters"][klass.turntable_cluster_name.to_s]["seq"] seq_type = (seq_config["seq_type"] ? seq_config["seq_type"].to_sym : :mysql) - @@tables[klass.table_name] ||= (@@sequences[sequence_name(klass.table_name, klass.primary_key)] ||= @@sequence_types[seq_type].new(klass, seq_config)) + seq_config_name = seq_config["connection"] + if seq_type == :mysql + seq_connection_config = ActiveRecord::Base.configurations[ActiveRecord::Turntable::RackupFramework.env]["seq"][seq_config_name] + else + seq_connection_config = ActiveRecord::Base.turntable_config[seq_type][seq_config_name] + end + @@tables[klass.table_name] ||= (@@sequences[sequence_name(klass.table_name, klass.primary_key)] ||= @@sequence_types[seq_type].new(klass, seq_connection_config)) end def self.has_sequencer?(table_name) @@ -28,7 +35,7 @@ def self.has_sequencer?(table_name) end def self.sequence_name(table_name, pk) - "#{ table_name}_#{pk || 'id'}_seq" + "#{table_name}_#{pk || 'id'}_seq" end def self.table_name(seq_name) diff --git a/lib/active_record/turntable/sequencer/redis.rb b/lib/active_record/turntable/sequencer/redis.rb new file mode 100644 index 00000000..0838847e --- /dev/null +++ b/lib/active_record/turntable/sequencer/redis.rb @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# redisを利用しての採番 +# +module ActiveRecord::Turntable + class Sequencer + class Redis < Sequencer + @@clients = {} + + def initialize(klass, options = {}) + require "redis" + @klass = klass + @option = options + end + + def next_sequence_value(sequence_name, offset = 1) + client.incrby(sequence_name, offset) + end + + def current_sequence_value(sequence_name) + id = client.get(sequence_name) + raise SequenceNotFoundError if id.nil? + return id + end + + private + + def client + @@clients[@option] ||= ::Redis.new(@option) + end + end + end +end From 11b21744922cf8cb0a92dadfd4cb66055607ec5d Mon Sep 17 00:00:00 2001 From: m4cteru Date: Tue, 7 Nov 2017 21:28:22 +0900 Subject: [PATCH 02/11] check the key exists before execute incrby --- lib/active_record/turntable/sequencer/redis.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/active_record/turntable/sequencer/redis.rb b/lib/active_record/turntable/sequencer/redis.rb index 0838847e..0e92284e 100644 --- a/lib/active_record/turntable/sequencer/redis.rb +++ b/lib/active_record/turntable/sequencer/redis.rb @@ -14,7 +14,9 @@ def initialize(klass, options = {}) end def next_sequence_value(sequence_name, offset = 1) - client.incrby(sequence_name, offset) + id = client.evalsha(lua_script_sha, argv: [sequence_name, offset] ) + raise SequenceNotFoundError if id.nil? + return id end def current_sequence_value(sequence_name) @@ -28,6 +30,11 @@ def current_sequence_value(sequence_name) def client @@clients[@option] ||= ::Redis.new(@option) end + + def lua_script_sha + @lua ||= client.script(:load, "return redis.call('EXISTS', ARGV[1]) == 1 and redis.call('INCRBY', ARGV[1], ARGV[2]) or nil") + end end end end + From d266ae3c0c58170dea5577ef4df64bc918be675e Mon Sep 17 00:00:00 2001 From: m4cteru Date: Thu, 9 Nov 2017 16:06:05 +0900 Subject: [PATCH 03/11] fix spec --- spec/active_record/turntable_spec.rb | 2 +- spec/spec_helper.rb | 2 ++ spec/turntable_helper.rb | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/spec/active_record/turntable_spec.rb b/spec/active_record/turntable_spec.rb index 5ec243be..91ae4a8c 100644 --- a/spec/active_record/turntable_spec.rb +++ b/spec/active_record/turntable_spec.rb @@ -8,7 +8,7 @@ context "#config_file" do it "should return Rails.root/config/turntable.yml default" do unless defined?(::Rails); class ::Rails; end; end - stub(Rails).root { "/path/to/rails_root" } + stub(ActiveRecord::Turntable::RackupFramework).root { "/path/to/rails_root" } ActiveRecord::Base.turntable_config_file = nil ActiveRecord::Base.turntable_config_file.should == "/path/to/rails_root/config/turntable.yml" end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 242bec03..ab837714 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -24,3 +24,5 @@ config.before(:each) do end end + + diff --git a/spec/turntable_helper.rb b/spec/turntable_helper.rb index fdf4f22e..5bba9b5d 100644 --- a/spec/turntable_helper.rb +++ b/spec/turntable_helper.rb @@ -27,3 +27,15 @@ def truncate_shard def migrate(version) ActiveRecord::Migrator.run(:up, MIGRATIONS_ROOT, version) end + +module ActiveRecord::Turntable + module TestRackupFramework + def self.env + "test" + end + def self.root + File.dirname(File.dirname(__FILE__)) + end + end + RackupFramework = TestRackupFramework +end \ No newline at end of file From 5931da7dd939345b2b6f3a428f7b3c64f9d04727 Mon Sep 17 00:00:00 2001 From: m4cteru Date: Thu, 9 Nov 2017 16:44:17 +0900 Subject: [PATCH 04/11] add redis sequencer spec --- activerecord-turntable.gemspec | 1 + .../turntable/sequencer/redis_spec.rb | 71 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 spec/active_record/turntable/sequencer/redis_spec.rb diff --git a/activerecord-turntable.gemspec b/activerecord-turntable.gemspec index 81fe5d64..676df09b 100644 --- a/activerecord-turntable.gemspec +++ b/activerecord-turntable.gemspec @@ -41,6 +41,7 @@ Gem::Specification.new do |s| s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) + s.add_development_dependency(%q, [">= 3.1.0"]) if RUBY_PLATFORM =~ /darwin/ s.add_development_dependency(%q, [">= 0"]) diff --git a/spec/active_record/turntable/sequencer/redis_spec.rb b/spec/active_record/turntable/sequencer/redis_spec.rb new file mode 100644 index 00000000..a72b9788 --- /dev/null +++ b/spec/active_record/turntable/sequencer/redis_spec.rb @@ -0,0 +1,71 @@ +require 'spec_helper' +require 'redis' + +describe ActiveRecord::Turntable::Sequencer::Redis do + let (:sequencer) { ActiveRecord::Turntable::Sequencer::Redis.new(Object, {host: "127.0.0.1"})} + let (:sequencer_name) { "test_sequencer" } + + context "exists sequencer key" do + before do + client = Redis.new(host: "127.0.0.1") + client.set(sequencer_name, 100) + end + + describe "#next_sequence_value" do + context "when use default offset" do + it "return 101" do + expect(sequencer.next_sequence_value(sequencer_name)).to eq 101 + end + end + context "when offset 100" do + it "return 200" do + expect(sequencer.next_sequence_value(sequencer_name, 100)).to eq 200 + end + end + end + + describe "#current_sequence_value" do + it do + expect(sequencer.current_sequence_value(sequencer_name)).to eq 100 + end + end + end + + context "not exists sequencer key" do + before do + client = Redis.new(host: "127.0.0.1") + client.del(sequencer_name) + end + + describe "#next_sequence_value" do + it "raise ActiveRecord::Turntable::SequenceNotFoundError" do + expect { sequencer.next_sequence_value(sequencer_name) }.to raise_error(ActiveRecord::Turntable::SequenceNotFoundError) + end + end + + describe "#current_sequence_value" do + it "raise ActiveRecord::Turntable::SequenceNotFoundError" do + expect { sequencer.current_sequence_value(sequencer_name) }.to raise_error(ActiveRecord::Turntable::SequenceNotFoundError) + end + end + end + + context "sequencer value is not number" do + before do + client = Redis.new(host: "127.0.0.1") + client.set(sequencer_name, "abc") + end + + describe "#next_sequence_value" do + it "raise Redis::CommandError" do + expect { sequencer.next_sequence_value(sequencer_name) }.to raise_error(Redis::CommandError) + end + end + + describe "#current_sequence_value" do + it "raise ActiveRecord::Turntable::SequenceValueBrokenError" do + expect { sequencer.current_sequence_value(sequencer_name) }.to raise_error(ActiveRecord::Turntable::SequenceValueBrokenError) + end + end + end +end From 3ef2572a5c5161b335652799e3d6b0799da56ba4 Mon Sep 17 00:00:00 2001 From: m4cteru Date: Thu, 9 Nov 2017 16:44:51 +0900 Subject: [PATCH 05/11] fix spec --- lib/active_record/turntable/error.rb | 1 + lib/active_record/turntable/sequencer/redis.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/active_record/turntable/error.rb b/lib/active_record/turntable/error.rb index 952ba120..54ff1dcd 100644 --- a/lib/active_record/turntable/error.rb +++ b/lib/active_record/turntable/error.rb @@ -2,6 +2,7 @@ module ActiveRecord::Turntable class Error < StandardError; end class NotImplementedError < Error; end class SequenceNotFoundError < Error; end + class SequenceValueBrokenError < Error; end class CannotSpecifyShardError < Error; end class MasterShardNotConnected < Error; end class UnknownOperatorError < Error; end diff --git a/lib/active_record/turntable/sequencer/redis.rb b/lib/active_record/turntable/sequencer/redis.rb index 0e92284e..c94f3760 100644 --- a/lib/active_record/turntable/sequencer/redis.rb +++ b/lib/active_record/turntable/sequencer/redis.rb @@ -22,7 +22,7 @@ def next_sequence_value(sequence_name, offset = 1) def current_sequence_value(sequence_name) id = client.get(sequence_name) raise SequenceNotFoundError if id.nil? - return id + return Integer(id) rescue raise SequenceValueBrokenError end private From 442d9784bc3d68d539f30c9bf2bd79f682130234 Mon Sep 17 00:00:00 2001 From: m4cteru Date: Thu, 9 Nov 2017 17:49:49 +0900 Subject: [PATCH 06/11] add sequencer spec --- .travis.yml | 2 ++ Rakefile | 8 ++++++ .../active_record/turntable/sequencer_spec.rb | 26 +++++++++++++++++++ spec/config/database.yml | 17 ++++++++++++ spec/config/turntable.yml | 12 +++++++++ spec/test_models.rb | 6 +++++ 6 files changed, 71 insertions(+) create mode 100644 spec/active_record/turntable/sequencer_spec.rb diff --git a/.travis.yml b/.travis.yml index bd29344d..7a5b6c62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ gemfile: - gemfiles/rails3_0.gemfile - gemfiles/rails3_1.gemfile - gemfiles/rails3_2.gemfile +services: + - redis-server before_script: - bundle exec rake turntable:db:reset script: bundle exec rake spec diff --git a/Rakefile b/Rakefile index 6dcbb6b0..98ee2975 100644 --- a/Rakefile +++ b/Rakefile @@ -89,6 +89,14 @@ namespace :turntable do t.datetime :deleted_at, :default => nil end ActiveRecord::Base.connection.create_sequence_for :archived_cards_users + + ActiveRecord::Base.connection.create_table :friends do |t| + t.string :nickname + t.string :thumbnail_url + t.datetime :joined_at + t.datetime :deleted_at + t.timestamps + end end end diff --git a/spec/active_record/turntable/sequencer_spec.rb b/spec/active_record/turntable/sequencer_spec.rb new file mode 100644 index 00000000..38ebeb49 --- /dev/null +++ b/spec/active_record/turntable/sequencer_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' +require 'pry' + +describe ActiveRecord::Turntable::Sequencer do + before(:all) do + reload_turntable!(File.join(File.dirname(__FILE__), "../../config/turntable.yml")) + end + + before do + establish_connection_to("test") + end + + describe "#build" do + context "when sequencer type mysql" do + it "return ActiveRecord::Turntable::Sequencer::Mysql" do + expect(ActiveRecord::Turntable::Sequencer.build(User)).to be_an_instance_of(ActiveRecord::Turntable::Sequencer::Mysql) + end + end + + context "when sequencer type redis" do + it "return ActiveRecord::Turntable::Sequencer::Redis" do + expect(ActiveRecord::Turntable::Sequencer.build(Friend)).to be_an_instance_of(ActiveRecord::Turntable::Sequencer::Redis) + end + end + end +end diff --git a/spec/config/database.yml b/spec/config/database.yml index 7b5bc533..97b77d2b 100644 --- a/spec/config/database.yml +++ b/spec/config/database.yml @@ -43,3 +43,20 @@ test: port: 3306 encoding: utf8 database: turntable3_test + redis_shard_1: + adapter: mysql2 + username: root + password: + host: localhost + port: 3306 + encoding: utf8 + database: + database: redis_shard1_test + redis_shard_2: + adapter: mysql2 + username: root + password: + host: localhost + port: 3306 + encoding: utf8 + database: redis_shard2_test diff --git a/spec/config/turntable.yml b/spec/config/turntable.yml index 6acaf8d1..4605b9c0 100644 --- a/spec/config/turntable.yml +++ b/spec/config/turntable.yml @@ -15,3 +15,15 @@ test: less_than: 80000 - connection: user_shard_3 less_than: 10000000 + friend_cluster: + algorithm: modulo + seq: + seq_type: redis + connection: redis_sequencer + shards: + - connection: redis_shard_1 + - connection: redis_shard_2 + redis: + redis_sequencer: + host: localhost + diff --git a/spec/test_models.rb b/spec/test_models.rb index 3db9a274..9b9183ef 100644 --- a/spec/test_models.rb +++ b/spec/test_models.rb @@ -25,3 +25,9 @@ class CardsUser < ActiveRecord::Base belongs_to :user belongs_to :card end + +class Friend < ActiveRecord::Base + turntable :friend_cluster, :id + sequencer +end + From 4885dcf3bd4fa48c624314ef12b6d0fd2d810358 Mon Sep 17 00:00:00 2001 From: m4cteru Date: Thu, 9 Nov 2017 18:01:40 +0900 Subject: [PATCH 07/11] update readme --- README.rdoc | 64 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/README.rdoc b/README.rdoc index 51c2a0f3..ae259552 100644 --- a/README.rdoc +++ b/README.rdoc @@ -183,23 +183,75 @@ add turntable [shard_key_name] to the model class: (0.2ms) [Shard: user_shard_3] SELECT COUNT(*) FROM `users` => 1 -=== Sequencer +== Sequencer + Sequencer provides generating global IDs. -Add below to the migration: +Turntable has follow 2 sequencers currently: + +* :mysql - Use database table to generate ids. +* :redis - Use redis to generate ids. + +=== Mysql example + +First, add configuration to turntable.yml and database.yml + +* database.yml + + development: + ... + seq: # <-- sequence database definition + user_seq_1: + <<: *spec + database: sample_app_user_seq_development + +* turntable.yml - create_sequence_for(:users) # <-- this line creates sequence table + development: + clusters: + user_cluster: # <-- cluster name + .... + seq: + user_seq: # <-- sequencer name + seq_type: mysql # <-- sequencer type + connection: user_seq_1 # <-- sequencer database connection -This will creates sequence table. +Add below to the migration: + + create_sequence_for(:users) # <-- this line creates sequence table named `users_id_seq` -Next, add sequencer to the model: +Next, add sequencer definition to the model: class User < ActiveRecord::Base turntable :id - sequencer # <-- this line enables sequencer module + sequencer :user_seq # <-- this line enables sequencer module has_one :status end +=== Redis example + +First, add redis gem to your Gemfile: + + gem 'redis' + +Then, add configuration to turntable.yml: + +* turntable.yml + + development: + clusters: + user_cluster: # <-- cluster name + .... + seq: + seq_type: redis + connection: redis_sequencer # <-- redis sequencer name + shards: + - connection: redis_shard_1 + - connection: redis_shard_2 + + redis: # <-- redis sequencer definition + redis_sequencer: # <-- redis sequencer name + host: 127.0.0.1 # <-- redis connection config === Transaction > user = User.find(2) From 9255e664f2c64d0feaa33985a014de50ea13a182 Mon Sep 17 00:00:00 2001 From: m4cteru Date: Thu, 9 Nov 2017 18:19:31 +0900 Subject: [PATCH 08/11] remove test code --- spec/active_record/turntable/sequencer_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/active_record/turntable/sequencer_spec.rb b/spec/active_record/turntable/sequencer_spec.rb index 38ebeb49..1d78c567 100644 --- a/spec/active_record/turntable/sequencer_spec.rb +++ b/spec/active_record/turntable/sequencer_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require 'pry' describe ActiveRecord::Turntable::Sequencer do before(:all) do From 2dcbafdefe075cbb15a062599fe27a289f4375fc Mon Sep 17 00:00:00 2001 From: m4cteru Date: Fri, 10 Nov 2017 13:47:21 +0900 Subject: [PATCH 09/11] check sequence type --- lib/active_record/turntable/cluster.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/active_record/turntable/cluster.rb b/lib/active_record/turntable/cluster.rb index 8682edc8..4efed340 100644 --- a/lib/active_record/turntable/cluster.rb +++ b/lib/active_record/turntable/cluster.rb @@ -16,8 +16,11 @@ def initialize(klass, cluster_spec, options = {}) @master_shard = MasterShard.new(klass) # setup sequencer - if (seq = (@options[:seq] || @config[:seq])) - @seq_shard = SeqShard.new(seq) + seq = (@options[:seq] || @config[:seq]) + if seq + if seq.values.size > 0 && seq.values.first["seq_type"] == "mysql" + @seq_shard = SeqShard.new(seq.values.first) + end end # setup shards From f0133127ee8d912eb12f498bc044feda72b406ef Mon Sep 17 00:00:00 2001 From: m4cteru Date: Fri, 10 Nov 2017 15:28:34 +0900 Subject: [PATCH 10/11] delete blank line --- spec/spec_helper.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ab837714..242bec03 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -24,5 +24,3 @@ config.before(:each) do end end - - From e643f3456b4c00fb55924a37ef0a11ec26e6517a Mon Sep 17 00:00:00 2001 From: m4cteru Date: Fri, 10 Nov 2017 16:19:51 +0900 Subject: [PATCH 11/11] rename redis shard cluster (readme) --- README.rdoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rdoc b/README.rdoc index ae259552..a441c9cd 100644 --- a/README.rdoc +++ b/README.rdoc @@ -246,8 +246,8 @@ Then, add configuration to turntable.yml: seq_type: redis connection: redis_sequencer # <-- redis sequencer name shards: - - connection: redis_shard_1 - - connection: redis_shard_2 + - connection: user_shard_1 + - connection: user_shard_2 redis: # <-- redis sequencer definition redis_sequencer: # <-- redis sequencer name