From 22ddc25ba492e03720aa9e0dbe4a67320b7e0fcd Mon Sep 17 00:00:00 2001 From: Andrew Konchin Date: Thu, 20 Apr 2023 19:29:06 +0300 Subject: [PATCH] Use FilterExpression and KeyConditionExpression in Query operation --- lib/dynamoid/adapter_plugin/aws_sdk_v3.rb | 34 ------- .../adapter_plugin/aws_sdk_v3/query.rb | 96 +++++++++++-------- spec/dynamoid/criteria/chain_spec.rb | 43 +++++++++ 3 files changed, 101 insertions(+), 72 deletions(-) diff --git a/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb b/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb index 18ac1b2d..e1959e70 100644 --- a/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb +++ b/lib/dynamoid/adapter_plugin/aws_sdk_v3.rb @@ -34,21 +34,6 @@ class AwsSdkV3 range_eq: 'EQ' }.freeze - FIELD_MAP = { - eq: 'EQ', - ne: 'NE', - gt: 'GT', - lt: 'LT', - gte: 'GE', - lte: 'LE', - begins_with: 'BEGINS_WITH', - between: 'BETWEEN', - in: 'IN', - contains: 'CONTAINS', - not_contains: 'NOT_CONTAINS', - null: 'NULL', - not_null: 'NOT_NULL', - }.freeze HASH_KEY = 'HASH' RANGE_KEY = 'RANGE' STRING_TYPE = 'S' @@ -133,25 +118,6 @@ class AwsSdkV3 attr_reader :table_cache - # Build an array of values for Condition - # Is used in ScanFilter and QueryFilter - # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html - # @param [String] operator value of RANGE_MAP or FIELD_MAP hash, e.g. "EQ", "LT" etc - # @param [Object] value scalar value or array/set - def self.attribute_value_list(operator, value) - # For BETWEEN and IN operators we should keep value as is (it should be already an array) - # NULL and NOT_NULL require absence of attribute list - # For all the other operators we wrap the value with array - # https://docs.aws.amazon.com/en_us/amazondynamodb/latest/developerguide/LegacyConditionalParameters.Conditions.html - if %w[BETWEEN IN].include?(operator) - [value].flatten - elsif %w[NULL NOT_NULL].include?(operator) - nil - else - [value] - end - end - # Establish the connection to DynamoDB. # # @return [Aws::DynamoDB::Client] the DynamoDB connection diff --git a/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb b/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb index 48598cce..9c0fd401 100644 --- a/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb +++ b/lib/dynamoid/adapter_plugin/aws_sdk_v3/query.rb @@ -3,6 +3,8 @@ require_relative 'middleware/backoff' require_relative 'middleware/limit' require_relative 'middleware/start_key' +require_relative 'filter_expression_convertor' +require_relative 'projection_expression_convertor' module Dynamoid # @private @@ -53,6 +55,37 @@ def call private def build_request + # expressions + name_placeholder = "#_a0".dup + value_placeholder = ":_a0".dup + + name_placeholder_sequence = -> { name_placeholder.next!.dup } + value_placeholder_sequence = -> { value_placeholder.next!.dup } + + name_placeholders = {} + value_placeholders = {} + + # Deal with various limits and batching + batch_size = options[:batch_size] + limit = [record_limit, scan_limit, batch_size].compact.min + + # key condition expression + convertor = FilterExpressionConvertor.new(key_conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence) + key_condition_expression = convertor.expression + value_placeholders = convertor.value_placeholders + name_placeholders = convertor.name_placeholders + + # filter expression + convertor = FilterExpressionConvertor.new(non_key_conditions, name_placeholders, value_placeholders, name_placeholder_sequence, value_placeholder_sequence) + filter_expression = convertor.expression + value_placeholders = convertor.value_placeholders + name_placeholders = convertor.name_placeholders + + # projection expression + convertor = ProjectionExpressionConvertor.new(options[:project], name_placeholders, name_placeholder_sequence) + projection_expression = convertor.expression + name_placeholders = convertor.name_placeholders + request = options.slice( :consistent_read, :scan_index_forward, @@ -61,15 +94,13 @@ def build_request :exclusive_start_key ).compact - # Deal with various limits and batching - batch_size = options[:batch_size] - limit = [record_limit, scan_limit, batch_size].compact.min - - request[:limit] = limit if limit - request[:table_name] = table.name - request[:key_conditions] = key_conditions - request[:query_filter] = query_filter - request[:attributes_to_get] = attributes_to_get + request[:table_name] = table.name + request[:limit] = limit if limit + request[:key_condition_expression] = key_condition_expression if key_condition_expression.present? + request[:filter_expression] = filter_expression if filter_expression.present? + request[:expression_attribute_values] = value_placeholders if value_placeholders.present? + request[:expression_attribute_names] = name_placeholders if name_placeholders.present? + request[:projection_expression] = projection_expression if projection_expression.present? request end @@ -91,40 +122,29 @@ def range_key_name end def key_conditions - result = { - hash_key_name => { - comparison_operator: AwsSdkV3::EQ, - attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::EQ, options[:hash_value].freeze) - } - } - - conditions.slice(*AwsSdkV3::RANGE_MAP.keys).each do |k, _v| - op = AwsSdkV3::RANGE_MAP[k] - - result[range_key_name] = { - comparison_operator: op, - attribute_value_list: AwsSdkV3.attribute_value_list(op, conditions[k].freeze) - } + result = {} + result[hash_key_name] = { eq: options[:hash_value].freeze } + + conditions.slice(*AwsSdkV3::RANGE_MAP.keys).each do |k, v| + op = { + range_greater_than: :gt, + range_less_than: :lt, + range_gte: :gte, + range_lte: :lte, + range_begins_with: :begins_with, + range_between: :between, + range_eq: :eq + }[k] + + result[range_key_name] ||= {} + result[range_key_name][op] = v end result end - def query_filter - conditions.except(*AwsSdkV3::RANGE_MAP.keys).reduce({}) do |result, (attr, cond)| - condition = { - comparison_operator: AwsSdkV3::FIELD_MAP[cond.keys[0]], - attribute_value_list: AwsSdkV3.attribute_value_list(AwsSdkV3::FIELD_MAP[cond.keys[0]], cond.values[0].freeze) - } - result[attr] = condition - result - end - end - - def attributes_to_get - return if options[:project].nil? - - options[:project].map(&:to_s) + def non_key_conditions + conditions.except(*AwsSdkV3::RANGE_MAP.keys) end end end diff --git a/spec/dynamoid/criteria/chain_spec.rb b/spec/dynamoid/criteria/chain_spec.rb index 0843445b..70708892 100644 --- a/spec/dynamoid/criteria/chain_spec.rb +++ b/spec/dynamoid/criteria/chain_spec.rb @@ -192,6 +192,18 @@ def request_params expect(model.where(name: 'Bob', 'age.between': [19, 31]).all).to contain_exactly(customer2, customer3) end + + it 'allows conditions with attribute names conflicting with DynamoDB reserved words' do + model = new_class do + range :size # SIZE is reserved word + end + + model.create_table + put_attributes(model.table_name, id: '1', size: 'c') + + documents = model.where(id: '1', size: 'c').to_a + expect(documents.map(&:id)).to eql ['1'] + end end # http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LegacyConditionalParameters.QueryFilter.html?shortFooter=true @@ -369,6 +381,21 @@ def request_params documents = model.where('age.not_null': false).to_a expect(documents.map(&:last_name)).to contain_exactly('cc') end + + it 'allows conditions with attribute names conflicting with DynamoDB reserved words' do + model = new_class do + # SCAN, SET and SIZE are reserved words + field :scan + field :set + field :size + end + + model.create_table + put_attributes(model.table_name, id: '1', scan: 'a', set: 'b', size: 'c') + + documents = model.where(id: '1', scan: 'a', set: 'b', size: 'c').to_a + expect(documents.map(&:id)).to eql ['1'] + end end # http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LegacyConditionalParameters.ScanFilter.html?shortFooter=true @@ -1841,6 +1868,16 @@ def request_params obj, = chain.project(:bucket).to_a expect(obj.attributes).to eq(bucket: 2) end + + it 'works with Query' do + object = model.create(name: 'Alex', bucket: 2) + + chain = described_class.new(model) + expect(chain).to receive(:raw_pages_via_query).and_call_original + + obj, = chain.where(id: object.id).project(:bucket).to_a + expect(obj.attributes).to eq(bucket: 2) + end end end @@ -1939,6 +1976,12 @@ def request_params expect(model.pluck(:bucket)).to contain_exactly(1001, 1002) end + + it 'works with Query' do + object = model.create(name: 'Alice', bucket: 1001) + + expect(model.where(id: object.id).pluck(:bucket)).to eq([1001]) + end end end