From 58bfaf70b89e8122e4a2093daf7ea856b6e948f7 Mon Sep 17 00:00:00 2001 From: Tim Perkins Date: Mon, 29 Aug 2016 10:13:25 -0400 Subject: [PATCH 1/2] Add support for logical types --- Appraisals | 2 +- CHANGELOG.md | 4 ++ avromatic.gemspec | 2 +- gemfiles/avro_salsify_fork.gemfile | 2 +- lib/avromatic/model.rb | 1 + lib/avromatic/model/attributes.rb | 5 +++ lib/avromatic/model/logical_types.rb | 16 +++++++ lib/avromatic/version.rb | 2 +- spec/avro/dsl/test/logical_types.rb | 6 +++ spec/avro/schema/test/logical_types.avsc | 35 +++++++++++++++ spec/avromatic/model/builder_spec.rb | 6 +++ .../model/messaging_serialization_spec.rb | 5 +++ .../avromatic/model/raw_serialization_spec.rb | 5 +++ spec/spec_helper.rb | 9 +++- .../contexts/logical_types_serialization.rb | 45 +++++++++++++++++++ spec/support/helpers/logical_types_helper.rb | 16 +++++++ 16 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 lib/avromatic/model/logical_types.rb create mode 100644 spec/avro/dsl/test/logical_types.rb create mode 100644 spec/avro/schema/test/logical_types.avsc create mode 100644 spec/support/contexts/logical_types_serialization.rb create mode 100644 spec/support/helpers/logical_types_helper.rb diff --git a/Appraisals b/Appraisals index 333e0da..7991a24 100644 --- a/Appraisals +++ b/Appraisals @@ -11,7 +11,7 @@ appraise 'rails4_2' do end appraise 'avro-salsify-fork' do - gem 'avro-salsify-fork', '1.9.0.0', require: 'avro' + gem 'avro-salsify-fork', '1.9.0.1', require: 'avro' gem 'activesupport', '~> 4.2.6' gem 'activemodel', '~> 4.2.6' end diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c73097..58ebeb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # avromatic changelog +## v0.8.0 +- Add support for logical types. Currently this requires using the + `avro-salsify-fork` gem for logical support with Ruby. + ## v0.7.1 - Raise a more descriptive error when attempting to generate a model for a non-record Avro type. diff --git a/avromatic.gemspec b/avromatic.gemspec index 80ee909..c18ac2e 100644 --- a/avromatic.gemspec +++ b/avromatic.gemspec @@ -30,7 +30,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'rspec', '~> 3.0' spec.add_development_dependency 'simplecov' spec.add_development_dependency 'webmock' - spec.add_development_dependency 'avro-builder', '>= 0.7.0' + spec.add_development_dependency 'avro-builder', '>= 0.11.0' # For FakeSchemaRegistryServer spec.add_development_dependency 'sinatra' spec.add_development_dependency 'salsify_rubocop', '~> 0.42.0' diff --git a/gemfiles/avro_salsify_fork.gemfile b/gemfiles/avro_salsify_fork.gemfile index 6233412..f71551e 100644 --- a/gemfiles/avro_salsify_fork.gemfile +++ b/gemfiles/avro_salsify_fork.gemfile @@ -2,7 +2,7 @@ source "https://rubygems.org" -gem "avro-salsify-fork", "1.9.0.0", :require => "avro" +gem "avro-salsify-fork", "1.9.0.1", :require => "avro" gem "activesupport", "~> 4.2.6" gem "activemodel", "~> 4.2.6" diff --git a/lib/avromatic/model.rb b/lib/avromatic/model.rb index 733def6..4bafcd5 100644 --- a/lib/avromatic/model.rb +++ b/lib/avromatic/model.rb @@ -1,6 +1,7 @@ require 'avromatic/model/builder' require 'avromatic/model/message_decoder' require 'avromatic/model/type_registry' +require 'avromatic/model/logical_types' module Avromatic module Model diff --git a/lib/avromatic/model/attributes.rb b/lib/avromatic/model/attributes.rb index bbc568c..215f990 100644 --- a/lib/avromatic/model/attributes.rb +++ b/lib/avromatic/model/attributes.rb @@ -100,6 +100,11 @@ def avro_field_class(field_type) custom_type = Avromatic.type_registry.fetch(field_type) return custom_type.value_class if custom_type.value_class + if field_type.respond_to?(:logical_type) + value_class = Avromatic::Model::LogicalTypes.value_class(field_type.logical_type) + return value_class if value_class + end + case field_type.type_sym when :string, :bytes, :fixed String diff --git a/lib/avromatic/model/logical_types.rb b/lib/avromatic/model/logical_types.rb new file mode 100644 index 0000000..b52374b --- /dev/null +++ b/lib/avromatic/model/logical_types.rb @@ -0,0 +1,16 @@ +module Avromatic + module Model + module LogicalTypes + + LOGICAL_TYPE_MAP = { + 'date' => Date, + 'timestamp-micros' => Time, + 'timestamp-millis' => Time + }.freeze + + def self.value_class(logical_type) + LOGICAL_TYPE_MAP.fetch(logical_type, nil) + end + end + end +end diff --git a/lib/avromatic/version.rb b/lib/avromatic/version.rb index 068f872..b99762d 100644 --- a/lib/avromatic/version.rb +++ b/lib/avromatic/version.rb @@ -1,3 +1,3 @@ module Avromatic - VERSION = '0.7.1'.freeze + VERSION = '0.8.0'.freeze end diff --git a/spec/avro/dsl/test/logical_types.rb b/spec/avro/dsl/test/logical_types.rb new file mode 100644 index 0000000..b32f379 --- /dev/null +++ b/spec/avro/dsl/test/logical_types.rb @@ -0,0 +1,6 @@ +record :logical_types, namespace: :test do + required :date, :int, logical_type: 'date' + required :ts_msec, :long, logical_type: 'timestamp-millis' + required :ts_usec, :long, logical_type: 'timestamp-micros' + required :unknown, :int, logical_type: 'foobar' +end diff --git a/spec/avro/schema/test/logical_types.avsc b/spec/avro/schema/test/logical_types.avsc new file mode 100644 index 0000000..e17fca8 --- /dev/null +++ b/spec/avro/schema/test/logical_types.avsc @@ -0,0 +1,35 @@ +{ + "type": "record", + "name": "logical_types", + "namespace": "test", + "fields": [ + { + "name": "date", + "type": { + "type": "int", + "logicalType": "date" + } + }, + { + "name": "ts_msec", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + }, + { + "name": "ts_usec", + "type": { + "type": "long", + "logicalType": "timestamp-micros" + } + }, + { + "name": "unknown", + "type": { + "type": "int", + "logicalType": "foobar" + } + } + ] +} diff --git a/spec/avromatic/model/builder_spec.rb b/spec/avromatic/model/builder_spec.rb index 54cd6b3..883091f 100644 --- a/spec/avromatic/model/builder_spec.rb +++ b/spec/avromatic/model/builder_spec.rb @@ -181,6 +181,12 @@ end end end + + context "logical types" do + let(:schema_name) { 'test.logical_types' } + + it_behaves_like "a generated model" + end end context "validation" do diff --git a/spec/avromatic/model/messaging_serialization_spec.rb b/spec/avromatic/model/messaging_serialization_spec.rb index ffb9817..b20c707 100644 --- a/spec/avromatic/model/messaging_serialization_spec.rb +++ b/spec/avromatic/model/messaging_serialization_spec.rb @@ -170,6 +170,11 @@ end end + it_behaves_like "logical type encoding and decoding" do + let(:encoded_value) { instance.avro_message_value } + let(:decoded) { test_class.avro_message_decode(encoded_value) } + end + context "custom types" do let(:schema_name) { 'test.named_type' } let(:test_class) do diff --git a/spec/avromatic/model/raw_serialization_spec.rb b/spec/avromatic/model/raw_serialization_spec.rb index 92fed9f..0be049f 100644 --- a/spec/avromatic/model/raw_serialization_spec.rb +++ b/spec/avromatic/model/raw_serialization_spec.rb @@ -130,6 +130,11 @@ end end + it_behaves_like "logical type encoding and decoding" do + let(:encoded_value) { instance.avro_raw_value } + let(:decoded) { test_class.avro_raw_decode(value: encoded_value) } + end + context "custom types" do let(:schema_name) { 'test.named_type' } let(:test_class) do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 5871c7e..76d16f7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,11 +1,18 @@ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) require 'simplecov' -SimpleCov.start +SimpleCov.start do + add_filter 'spec' + minimum_coverage 99 +end require 'avromatic' +Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f } + RSpec.configure do |config| + config.extend LogicalTypesHelper + config.before do Avromatic.logger = Logger.new('log/test.log') Avromatic.registry_url = 'http://registry.example.com' diff --git a/spec/support/contexts/logical_types_serialization.rb b/spec/support/contexts/logical_types_serialization.rb new file mode 100644 index 0000000..d06c028 --- /dev/null +++ b/spec/support/contexts/logical_types_serialization.rb @@ -0,0 +1,45 @@ +# This examples expects let-variables to be defined for: +# decoded: a model instance based on the encoded_value +shared_examples_for "logical type encoding and decoding" do + context "logical types" do + let(:schema_name) { 'test.logical_types' } + let(:test_class) do + Avromatic::Model.model(schema_name: schema_name) + end + let(:now) { Time.now } + + with_logical_types do + context "supported" do + let(:values) do + { + date: Date.today, + ts_msec: Time.at(now.to_i, now.usec / 1000 * 1000), + ts_usec: now, + unknown: 42 + } + end + + it "encodes and decodes instances" do + expect(decoded).to eq(instance) + end + end + end + + without_logical_types do + context "unsupported" do + let(:values) do + { + date: (Date.today - Date.new(1970, 1, 1)).to_i, + ts_msec: now.to_i + now.usec / 1000 * 1000, + ts_usec: now.to_i * 1_000_000 + now.usec, + unknown: 42 + } + end + + it "encodes and decodes instances" do + expect(decoded).to eq(instance) + end + end + end + end +end diff --git a/spec/support/helpers/logical_types_helper.rb b/spec/support/helpers/logical_types_helper.rb new file mode 100644 index 0000000..ca54eef --- /dev/null +++ b/spec/support/helpers/logical_types_helper.rb @@ -0,0 +1,16 @@ +module LogicalTypesHelper + + def with_logical_types + yield if logical_types? + end + + def without_logical_types + yield unless logical_types? + end + + private + + def logical_types? + Avro::Schema.instance_methods.include?(:logical_type) + end +end From da4dfb8ecada45a8c2d98425eb99e6c298081d5d Mon Sep 17 00:00:00 2001 From: Tim Perkins Date: Mon, 29 Aug 2016 11:18:29 -0400 Subject: [PATCH 2/2] PR feedback --- CHANGELOG.md | 2 +- lib/avromatic/model/logical_types.rb | 2 +- spec/spec_helper.rb | 2 +- spec/support/contexts/logical_types_serialization.rb | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ebeb6..ddee84f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v0.8.0 - Add support for logical types. Currently this requires using the - `avro-salsify-fork` gem for logical support with Ruby. + `avro-salsify-fork` gem for logical types support with Ruby. ## v0.7.1 - Raise a more descriptive error when attempting to generate a model for a diff --git a/lib/avromatic/model/logical_types.rb b/lib/avromatic/model/logical_types.rb index b52374b..de4e1f2 100644 --- a/lib/avromatic/model/logical_types.rb +++ b/lib/avromatic/model/logical_types.rb @@ -9,7 +9,7 @@ module LogicalTypes }.freeze def self.value_class(logical_type) - LOGICAL_TYPE_MAP.fetch(logical_type, nil) + LOGICAL_TYPE_MAP[logical_type] end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 76d16f7..a85ae7a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -3,7 +3,7 @@ SimpleCov.start do add_filter 'spec' - minimum_coverage 99 + minimum_coverage 98 end require 'avromatic' diff --git a/spec/support/contexts/logical_types_serialization.rb b/spec/support/contexts/logical_types_serialization.rb index d06c028..0f6f7be 100644 --- a/spec/support/contexts/logical_types_serialization.rb +++ b/spec/support/contexts/logical_types_serialization.rb @@ -27,9 +27,10 @@ without_logical_types do context "unsupported" do + let(:epoch_start) { Date.new(1970, 1, 1) } let(:values) do { - date: (Date.today - Date.new(1970, 1, 1)).to_i, + date: (Date.today - epoch_start).to_i, ts_msec: now.to_i + now.usec / 1000 * 1000, ts_usec: now.to_i * 1_000_000 + now.usec, unknown: 42