diff --git a/.travis.yml b/.travis.yml index bd29344d..7a5b6c62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ gemfile: - gemfiles/rails3_0.gemfile - gemfiles/rails3_1.gemfile - gemfiles/rails3_2.gemfile +services: + - redis-server before_script: - bundle exec rake turntable:db:reset script: bundle exec rake spec diff --git a/README.rdoc b/README.rdoc index 51c2a0f3..a441c9cd 100644 --- a/README.rdoc +++ b/README.rdoc @@ -183,23 +183,75 @@ add turntable [shard_key_name] to the model class: (0.2ms) [Shard: user_shard_3] SELECT COUNT(*) FROM `users` => 1 -=== Sequencer +== Sequencer + Sequencer provides generating global IDs. -Add below to the migration: +Turntable has follow 2 sequencers currently: + +* :mysql - Use database table to generate ids. +* :redis - Use redis to generate ids. + +=== Mysql example + +First, add configuration to turntable.yml and database.yml + +* database.yml + + development: + ... + seq: # <-- sequence database definition + user_seq_1: + <<: *spec + database: sample_app_user_seq_development + +* turntable.yml - create_sequence_for(:users) # <-- this line creates sequence table + development: + clusters: + user_cluster: # <-- cluster name + .... + seq: + user_seq: # <-- sequencer name + seq_type: mysql # <-- sequencer type + connection: user_seq_1 # <-- sequencer database connection -This will creates sequence table. +Add below to the migration: + + create_sequence_for(:users) # <-- this line creates sequence table named `users_id_seq` -Next, add sequencer to the model: +Next, add sequencer definition to the model: class User < ActiveRecord::Base turntable :id - sequencer # <-- this line enables sequencer module + sequencer :user_seq # <-- this line enables sequencer module has_one :status end +=== Redis example + +First, add redis gem to your Gemfile: + + gem 'redis' + +Then, add configuration to turntable.yml: + +* turntable.yml + + development: + clusters: + user_cluster: # <-- cluster name + .... + seq: + seq_type: redis + connection: redis_sequencer # <-- redis sequencer name + shards: + - connection: user_shard_1 + - connection: user_shard_2 + + redis: # <-- redis sequencer definition + redis_sequencer: # <-- redis sequencer name + host: 127.0.0.1 # <-- redis connection config === Transaction > user = User.find(2) diff --git a/Rakefile b/Rakefile index 6dcbb6b0..98ee2975 100644 --- a/Rakefile +++ b/Rakefile @@ -89,6 +89,14 @@ namespace :turntable do t.datetime :deleted_at, :default => nil end ActiveRecord::Base.connection.create_sequence_for :archived_cards_users + + ActiveRecord::Base.connection.create_table :friends do |t| + t.string :nickname + t.string :thumbnail_url + t.datetime :joined_at + t.datetime :deleted_at + t.timestamps + end end end diff --git a/activerecord-turntable.gemspec b/activerecord-turntable.gemspec index 81fe5d64..676df09b 100644 --- a/activerecord-turntable.gemspec +++ b/activerecord-turntable.gemspec @@ -41,6 +41,7 @@ Gem::Specification.new do |s| s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) + s.add_development_dependency(%q, [">= 3.1.0"]) if RUBY_PLATFORM =~ /darwin/ s.add_development_dependency(%q, [">= 0"]) diff --git a/lib/active_record/turntable/cluster.rb b/lib/active_record/turntable/cluster.rb index 8682edc8..4efed340 100644 --- a/lib/active_record/turntable/cluster.rb +++ b/lib/active_record/turntable/cluster.rb @@ -16,8 +16,11 @@ def initialize(klass, cluster_spec, options = {}) @master_shard = MasterShard.new(klass) # setup sequencer - if (seq = (@options[:seq] || @config[:seq])) - @seq_shard = SeqShard.new(seq) + seq = (@options[:seq] || @config[:seq]) + if seq + if seq.values.size > 0 && seq.values.first["seq_type"] == "mysql" + @seq_shard = SeqShard.new(seq.values.first) + end end # setup shards diff --git a/lib/active_record/turntable/error.rb b/lib/active_record/turntable/error.rb index 952ba120..54ff1dcd 100644 --- a/lib/active_record/turntable/error.rb +++ b/lib/active_record/turntable/error.rb @@ -2,6 +2,7 @@ module ActiveRecord::Turntable class Error < StandardError; end class NotImplementedError < Error; end class SequenceNotFoundError < Error; end + class SequenceValueBrokenError < Error; end class CannotSpecifyShardError < Error; end class MasterShardNotConnected < Error; end class UnknownOperatorError < Error; end diff --git a/lib/active_record/turntable/sequencer.rb b/lib/active_record/turntable/sequencer.rb index f8046519..92ea2813 100644 --- a/lib/active_record/turntable/sequencer.rb +++ b/lib/active_record/turntable/sequencer.rb @@ -7,9 +7,11 @@ module ActiveRecord::Turntable class Sequencer autoload :Api, "active_record/turntable/sequencer/api" autoload :Mysql, "active_record/turntable/sequencer/mysql" + autoload :Redis, "active_record/turntable/sequencer/redis" @@sequence_types = { :api => Api, - :mysql => Mysql + :mysql => Mysql, + :redis => Redis } @@sequences = {} @@ -17,10 +19,15 @@ class Sequencer cattr_reader :sequences, :tables def self.build(klass) - seq_config_name = ActiveRecord::Base.turntable_config["clusters"][klass.turntable_cluster_name.to_s]["seq"]["connection"] - seq_config = ActiveRecord::Base.configurations[ActiveRecord::Turntable::RackupFramework.env]["seq"][seq_config_name] + seq_config = ActiveRecord::Base.turntable_config["clusters"][klass.turntable_cluster_name.to_s]["seq"] seq_type = (seq_config["seq_type"] ? seq_config["seq_type"].to_sym : :mysql) - @@tables[klass.table_name] ||= (@@sequences[sequence_name(klass.table_name, klass.primary_key)] ||= @@sequence_types[seq_type].new(klass, seq_config)) + seq_config_name = seq_config["connection"] + if seq_type == :mysql + seq_connection_config = ActiveRecord::Base.configurations[ActiveRecord::Turntable::RackupFramework.env]["seq"][seq_config_name] + else + seq_connection_config = ActiveRecord::Base.turntable_config[seq_type][seq_config_name] + end + @@tables[klass.table_name] ||= (@@sequences[sequence_name(klass.table_name, klass.primary_key)] ||= @@sequence_types[seq_type].new(klass, seq_connection_config)) end def self.has_sequencer?(table_name) @@ -28,7 +35,7 @@ def self.has_sequencer?(table_name) end def self.sequence_name(table_name, pk) - "#{ table_name}_#{pk || 'id'}_seq" + "#{table_name}_#{pk || 'id'}_seq" end def self.table_name(seq_name) diff --git a/lib/active_record/turntable/sequencer/redis.rb b/lib/active_record/turntable/sequencer/redis.rb new file mode 100644 index 00000000..c94f3760 --- /dev/null +++ b/lib/active_record/turntable/sequencer/redis.rb @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# redisを利用しての採番 +# +module ActiveRecord::Turntable + class Sequencer + class Redis < Sequencer + @@clients = {} + + def initialize(klass, options = {}) + require "redis" + @klass = klass + @option = options + end + + def next_sequence_value(sequence_name, offset = 1) + id = client.evalsha(lua_script_sha, argv: [sequence_name, offset] ) + raise SequenceNotFoundError if id.nil? + return id + end + + def current_sequence_value(sequence_name) + id = client.get(sequence_name) + raise SequenceNotFoundError if id.nil? + return Integer(id) rescue raise SequenceValueBrokenError + end + + private + + def client + @@clients[@option] ||= ::Redis.new(@option) + end + + def lua_script_sha + @lua ||= client.script(:load, "return redis.call('EXISTS', ARGV[1]) == 1 and redis.call('INCRBY', ARGV[1], ARGV[2]) or nil") + end + end + end +end + diff --git a/spec/active_record/turntable/sequencer/redis_spec.rb b/spec/active_record/turntable/sequencer/redis_spec.rb new file mode 100644 index 00000000..a72b9788 --- /dev/null +++ b/spec/active_record/turntable/sequencer/redis_spec.rb @@ -0,0 +1,71 @@ +require 'spec_helper' +require 'redis' + +describe ActiveRecord::Turntable::Sequencer::Redis do + let (:sequencer) { ActiveRecord::Turntable::Sequencer::Redis.new(Object, {host: "127.0.0.1"})} + let (:sequencer_name) { "test_sequencer" } + + context "exists sequencer key" do + before do + client = Redis.new(host: "127.0.0.1") + client.set(sequencer_name, 100) + end + + describe "#next_sequence_value" do + context "when use default offset" do + it "return 101" do + expect(sequencer.next_sequence_value(sequencer_name)).to eq 101 + end + end + context "when offset 100" do + it "return 200" do + expect(sequencer.next_sequence_value(sequencer_name, 100)).to eq 200 + end + end + end + + describe "#current_sequence_value" do + it do + expect(sequencer.current_sequence_value(sequencer_name)).to eq 100 + end + end + end + + context "not exists sequencer key" do + before do + client = Redis.new(host: "127.0.0.1") + client.del(sequencer_name) + end + + describe "#next_sequence_value" do + it "raise ActiveRecord::Turntable::SequenceNotFoundError" do + expect { sequencer.next_sequence_value(sequencer_name) }.to raise_error(ActiveRecord::Turntable::SequenceNotFoundError) + end + end + + describe "#current_sequence_value" do + it "raise ActiveRecord::Turntable::SequenceNotFoundError" do + expect { sequencer.current_sequence_value(sequencer_name) }.to raise_error(ActiveRecord::Turntable::SequenceNotFoundError) + end + end + end + + context "sequencer value is not number" do + before do + client = Redis.new(host: "127.0.0.1") + client.set(sequencer_name, "abc") + end + + describe "#next_sequence_value" do + it "raise Redis::CommandError" do + expect { sequencer.next_sequence_value(sequencer_name) }.to raise_error(Redis::CommandError) + end + end + + describe "#current_sequence_value" do + it "raise ActiveRecord::Turntable::SequenceValueBrokenError" do + expect { sequencer.current_sequence_value(sequencer_name) }.to raise_error(ActiveRecord::Turntable::SequenceValueBrokenError) + end + end + end +end diff --git a/spec/active_record/turntable/sequencer_spec.rb b/spec/active_record/turntable/sequencer_spec.rb new file mode 100644 index 00000000..1d78c567 --- /dev/null +++ b/spec/active_record/turntable/sequencer_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe ActiveRecord::Turntable::Sequencer do + before(:all) do + reload_turntable!(File.join(File.dirname(__FILE__), "../../config/turntable.yml")) + end + + before do + establish_connection_to("test") + end + + describe "#build" do + context "when sequencer type mysql" do + it "return ActiveRecord::Turntable::Sequencer::Mysql" do + expect(ActiveRecord::Turntable::Sequencer.build(User)).to be_an_instance_of(ActiveRecord::Turntable::Sequencer::Mysql) + end + end + + context "when sequencer type redis" do + it "return ActiveRecord::Turntable::Sequencer::Redis" do + expect(ActiveRecord::Turntable::Sequencer.build(Friend)).to be_an_instance_of(ActiveRecord::Turntable::Sequencer::Redis) + end + end + end +end diff --git a/spec/active_record/turntable_spec.rb b/spec/active_record/turntable_spec.rb index 5ec243be..91ae4a8c 100644 --- a/spec/active_record/turntable_spec.rb +++ b/spec/active_record/turntable_spec.rb @@ -8,7 +8,7 @@ context "#config_file" do it "should return Rails.root/config/turntable.yml default" do unless defined?(::Rails); class ::Rails; end; end - stub(Rails).root { "/path/to/rails_root" } + stub(ActiveRecord::Turntable::RackupFramework).root { "/path/to/rails_root" } ActiveRecord::Base.turntable_config_file = nil ActiveRecord::Base.turntable_config_file.should == "/path/to/rails_root/config/turntable.yml" end diff --git a/spec/config/database.yml b/spec/config/database.yml index 7b5bc533..97b77d2b 100644 --- a/spec/config/database.yml +++ b/spec/config/database.yml @@ -43,3 +43,20 @@ test: port: 3306 encoding: utf8 database: turntable3_test + redis_shard_1: + adapter: mysql2 + username: root + password: + host: localhost + port: 3306 + encoding: utf8 + database: + database: redis_shard1_test + redis_shard_2: + adapter: mysql2 + username: root + password: + host: localhost + port: 3306 + encoding: utf8 + database: redis_shard2_test diff --git a/spec/config/turntable.yml b/spec/config/turntable.yml index 6acaf8d1..4605b9c0 100644 --- a/spec/config/turntable.yml +++ b/spec/config/turntable.yml @@ -15,3 +15,15 @@ test: less_than: 80000 - connection: user_shard_3 less_than: 10000000 + friend_cluster: + algorithm: modulo + seq: + seq_type: redis + connection: redis_sequencer + shards: + - connection: redis_shard_1 + - connection: redis_shard_2 + redis: + redis_sequencer: + host: localhost + diff --git a/spec/test_models.rb b/spec/test_models.rb index 3db9a274..9b9183ef 100644 --- a/spec/test_models.rb +++ b/spec/test_models.rb @@ -25,3 +25,9 @@ class CardsUser < ActiveRecord::Base belongs_to :user belongs_to :card end + +class Friend < ActiveRecord::Base + turntable :friend_cluster, :id + sequencer +end + diff --git a/spec/turntable_helper.rb b/spec/turntable_helper.rb index fdf4f22e..5bba9b5d 100644 --- a/spec/turntable_helper.rb +++ b/spec/turntable_helper.rb @@ -27,3 +27,15 @@ def truncate_shard def migrate(version) ActiveRecord::Migrator.run(:up, MIGRATIONS_ROOT, version) end + +module ActiveRecord::Turntable + module TestRackupFramework + def self.env + "test" + end + def self.root + File.dirname(File.dirname(__FILE__)) + end + end + RackupFramework = TestRackupFramework +end \ No newline at end of file