Enable compound indices and queries to work regardless of ordering

README.md

@@ -511,7 +511,7 @@ ResultSet<Car> results = cars.retrieve(query, queryOptions(orderBy(descending(Ca
 ResultSet<Car> results = cars.retrieve(query, queryOptions(orderBy(descending(Car.PRICE), ascending(Car.DOORS))));
 ```
 
-Note that ordering results as above uses the default _materialize_ ordering strategy. This is relatively expensive, dependent on the number of objects matching the query, and can cause latency in accessing the first object. It requires all results to be materialized into a sorted set up-front _before iteration can begin_. However ordering results in this way also implicitly eliminates duplicates.
+Note that ordering results as above uses the default _materialize_ ordering strategy. This is relatively expensive, dependent on the number of objects matching the query, and can cause latency in accessing the first object. It requires all results to be materialized into a sorted set up-front _before iteration can begin_.
 
 ### Index-accelerated ordering ###
 

code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundAttribute.java

@@ -78,7 +78,7 @@ public CompoundAttribute(List<Attribute<O, ?>> attributes) {
         if (attributes.size() < 2) {
             throw new IllegalStateException("Cannot create a compound index on fewer than two attributes: " + attributes.size());
         }
-        this.attributes = attributes;
+        this.attributes = deduplicateAndSort(attributes);
     }
 
     public int size() {
@@ -131,15 +131,25 @@ public Iterable<CompoundValueTuple<O>> getValues(O object, QueryOptions queryOpt
         // Values for first attribute:  1
         // Values for second attribute: "bar", "baz"
         // Values for third attribute:  2.0, 3.0, 4.0
+        //
+        // or in list form :
+        // [[1], [bar, baz], [2.0, 3.0, 4.0]]
+        //
         // ...then we should generate and index the object against the following tuples:
         // [[1, bar, 2.0], [1, bar, 3.0], [1, bar, 4.0], [1, baz, 2.0], [1, baz, 3.0], [1, baz, 4.0]]
+        // note that ordering of attributes is preserved, we take advantage of this below
         List<List<Object>> listsOfValueCombinations = TupleCombinationGenerator.generateCombinations(attributeValueLists);
 
         // STEP 3.
         // Wrap each of the unique combinations in a CompoundValueTuple object...
         List<CompoundValueTuple<O>> tuples = new ArrayList<CompoundValueTuple<O>>(listsOfValueCombinations.size());
         for (List<Object> valueCombination : listsOfValueCombinations) {
-            tuples.add(new CompoundValueTuple<O>(valueCombination));
+            Map<Attribute<O, ?>, Object> mappedTuples = new HashMap<Attribute<O, ?>, Object>();
+            // here we take advantage of the fact that ordering of attributes is preserved when the tuples are generated
+            for (int i = 0; i < attributes.size(); i++) {
+                mappedTuples.put(attributes.get(i), valueCombination.get(i));
+            }
+            tuples.add(new CompoundValueTuple<O>(mappedTuples));
         }
         // Return the list of CompoundValueTuple objects...
         return tuples;
@@ -169,4 +179,15 @@ public String toString() {
                 '}';
     }
 
+    private static <O> List<Attribute<O, ?>> deduplicateAndSort(List<Attribute<O, ?>> attributes) {
+        final List<Attribute<O, ?>> deduplicatedAttributes = new ArrayList<Attribute<O, ?>>(new HashSet<Attribute<O, ?>>(attributes));
+
+        Collections.sort(deduplicatedAttributes, new Comparator<Attribute<O, ?>>() {
+            @Override
+            public int compare(Attribute<O, ?> o1, Attribute<O, ?> o2) {
+                return o1.getAttributeName().compareTo(o2.getAttributeName());
+            }
+        });
+        return deduplicatedAttributes;
+    }
 }

code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundQuery.java

@@ -17,13 +17,18 @@
 
 import com.googlecode.cqengine.attribute.Attribute;
 import com.googlecode.cqengine.query.Query;
+import com.googlecode.cqengine.query.logical.And;
+import com.googlecode.cqengine.query.logical.LogicalQuery;
 import com.googlecode.cqengine.query.option.QueryOptions;
 import com.googlecode.cqengine.query.simple.Equal;
 import com.googlecode.cqengine.query.simple.SimpleQuery;
-import com.googlecode.cqengine.query.logical.And;
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * A query which wraps a {@link CompoundAttribute}, used only in the query engine's internal communication
@@ -71,16 +76,18 @@ public CompoundAttribute<O> getCompoundAttribute() {
     }
 
     public CompoundValueTuple<O> getCompoundValueTuple() {
-        List<Object> attributeValues = new ArrayList<Object>(andQuery.getSimpleQueries().size());
+        Map<Attribute<O, ?>, Object> attributeValues = new HashMap<Attribute<O, ?>, Object>();
         for (SimpleQuery<O, ?> simpleQuery : andQuery.getSimpleQueries()) {
             Equal<O, ?> equal = (Equal<O, ?>) simpleQuery;
-            attributeValues.add(equal.getValue());
+            attributeValues.put(equal.getAttribute(), equal.getValue());
         }
         return new CompoundValueTuple<O>(attributeValues);
     }
 
     public static <O> CompoundQuery<O> fromAndQueryIfSuitable(And<O> andQuery) {
-        if (andQuery.hasLogicalQueries()) {
+        andQuery = flatten(andQuery);
+
+        if (andQuery == null) {
             return null;
         }
         List<Attribute<O, ?>> attributeList = new ArrayList<Attribute<O, ?>>(andQuery.getSimpleQueries().size());
@@ -95,4 +102,26 @@ public static <O> CompoundQuery<O> fromAndQueryIfSuitable(And<O> andQuery) {
         return new CompoundQuery<O>(andQuery, compoundAttribute);
     }
 
+    /**
+     * Flatten an And query, bringing any nested And queries up the top level query if possible
+     *
+     * @param andQuery the And query to flatten
+     * @return the flattened And query, or null if it cannot be converted into a flat And query
+     */
+    private static <O> And<O> flatten(And<O> andQuery) {
+        final Set<Query<O>> flatQuerySet = new HashSet<Query<O>>();
+        for (LogicalQuery<O> childQuery : andQuery.getLogicalQueries()) {
+            if (childQuery instanceof And) {
+                And<O> flatQuery = flatten((And<O>) childQuery);
+                if (flatQuery == null) {
+                    return null;
+                }
+                flatQuerySet.addAll(flatQuery.getSimpleQueries());
+            } else {
+                return null;
+            }
+        }
+        flatQuerySet.addAll(andQuery.getSimpleQueries());
+        return new And<O>(flatQuerySet);
+    }
 }

code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundValueTuple.java

@@ -1,12 +1,12 @@
 /**
  * Copyright 2012-2015 Niall Gallagher
- *
+ * <p>
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,8 +17,12 @@
 
 import com.googlecode.cqengine.attribute.Attribute;
 
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 
 /**
  * A tuple (ordered list) of values, extracted from the fields of an object, according to, and in the same order as, the
@@ -35,8 +39,8 @@ public class CompoundValueTuple<O> {
     private final List<?> attributeValues;
     private final int hashCode;
 
-    public CompoundValueTuple(List<?> attributeValues) {
-        this.attributeValues = attributeValues;
+    public CompoundValueTuple(Map<Attribute<O, ?>, ?> attributeToValues) {
+        this.attributeValues = getOrderedAttributeValues(attributeToValues);
         this.hashCode = attributeValues.hashCode();
     }
 
@@ -69,4 +73,25 @@ public String toString() {
                 ", hashCode=" + hashCode +
                 '}';
     }
+
+    /**
+     * Given attribute values, sort them to ensure that compound index entries in the index engine
+     * work regardless of how the given CompoundQuery is ordered
+     * <p>
+     * Note: to be more specific, without this, the {@code equals} method will return false when
+     * two CompoundValueTuples contain the same values but are ordered differently
+     *
+     * @param attributeToValues a map of attributes and their corresponding values that were extracted from an object or query
+     * @return a list of ordered attribute values that will be used in this CompoundValueTuple
+     */
+    private static <O> List<?> getOrderedAttributeValues(Map<Attribute<O, ?>, ?> attributeToValues) {
+        final TreeMap<Attribute<O, ?>, Object> attributeValuesSortedByAttributeName = new TreeMap<Attribute<O, ?>, Object>(new Comparator<Attribute<O, ?>>() {
+            @Override
+            public int compare(Attribute<O, ?> o1, Attribute<O, ?> o2) {
+                return o1.getAttributeName().compareTo(o2.getAttributeName());
+            }
+        });
+        attributeValuesSortedByAttributeName.putAll(attributeToValues);
+        return new ArrayList<Object>(attributeValuesSortedByAttributeName.values());
+    }
 }

code/src/test/java/com/googlecode/cqengine/IndexedCollectionFunctionalTest.java

@@ -15,6 +15,7 @@
  */
 package com.googlecode.cqengine;
 
+import com.google.common.collect.ImmutableMap;
 import com.googlecode.cqengine.attribute.Attribute;
 import com.googlecode.cqengine.attribute.SimpleAttribute;
 import com.googlecode.cqengine.attribute.StandingQueryAttribute;
@@ -524,7 +525,7 @@ public CompoundValueTuple<Car> getQuantizedValue(CompoundValueTuple<Car> tuple)
                                             String manufacturer = (String) tupleValues.next();
                                             String model = (String) tupleValues.next();
                                             String quantizedModel = "Focus".equals(model) ? "Focus" : "Other";
-                                            return new CompoundValueTuple<Car>(Arrays.asList(manufacturer, quantizedModel));
+                                            return new CompoundValueTuple<Car>(ImmutableMap.<Attribute<Car, ?>, Object>of(Car.MANUFACTURER, manufacturer, Car.MODEL, quantizedModel));
                                         }
                                     }, Car.MANUFACTURER, Car.MODEL)
                             ),

code/src/test/java/com/googlecode/cqengine/index/compound/support/ScrambledQueriesTest.java

@@ -0,0 +1,71 @@
+package com.googlecode.cqengine.index.compound.support;
+
+import com.google.common.collect.ImmutableList;
+import com.googlecode.cqengine.ConcurrentIndexedCollection;
+import com.googlecode.cqengine.IndexedCollection;
+import com.googlecode.cqengine.examples.introduction.Car;
+import com.googlecode.cqengine.index.compound.CompoundIndex;
+import com.googlecode.cqengine.query.Query;
+import com.googlecode.cqengine.query.logical.And;
+import com.googlecode.cqengine.resultset.ResultSet;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static com.googlecode.cqengine.query.QueryFactory.and;
+import static com.googlecode.cqengine.query.QueryFactory.equal;
+
+/**
+ * @author Eduardo Barrios
+ */
+public class ScrambledQueriesTest {
+
+    private final IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();
+
+    @Before
+    public void setUp() {
+        // add the index we want to test
+        cars.addIndex(CompoundIndex.onAttributes(Car.CAR_ID, Car.NAME, Car.DESCRIPTION, Car.FEATURES));
+
+        // Add some objects to the collection...
+        cars.add(new Car(1, "ford", "focus", ImmutableList.of("fwd", "hatchback", "black")));
+        cars.add(new Car(2, "honda", "civic", ImmutableList.of("fwd", "sedan", "silver")));
+        cars.add(new Car(3, "toyota", "prius", ImmutableList.of("hybrid", "fwd", "blue")));
+        cars.add(new Car(4, "ford", "explorer", ImmutableList.of("awd", "green")));
+        cars.add(new Car(5, "honda", "crv", ImmutableList.of("awd", "black")));
+        cars.add(new Car(6, "toyota", "tundra", ImmutableList.of("4x4", "red")));
+    }
+
+    @Test
+    public void testUnscrambleQuery() {
+
+        Query<Car> scrambledQuery = and(
+                equal(Car.NAME, "ford"),
+                equal(Car.FEATURES, "black"),
+                equal(Car.CAR_ID, 1),
+                equal(Car.DESCRIPTION, "focus")
+        );
+        ResultSet<Car> flatQueryResult = cars.retrieve(scrambledQuery);
+        Assert.assertEquals(1, flatQueryResult.size());
+        Assert.assertEquals(20, flatQueryResult.getRetrievalCost());
+    }
+
+    @Test
+    public void testFlattenQuery() {
+        And<Car> nestedQuery = and(
+                equal(Car.CAR_ID, 1),
+                and(
+                        equal(Car.NAME, "ford"),
+                        and(
+                                equal(Car.DESCRIPTION, "focus"),
+                                equal(Car.FEATURES, "black")
+                        )
+                )
+        );
+
+        ResultSet<Car> nestedQueryResult = cars.retrieve(nestedQuery);
+        Assert.assertEquals(1, nestedQueryResult.size());
+        Assert.assertEquals(20, nestedQueryResult.getRetrievalCost());
+
+    }
+}