Universal Query Routing

README.md

@@ -8,20 +8,16 @@ _Polypheny-DB_ is a self-adaptive Polystore that provides cost- and workload awa
 
 If you want to learn more about Polypheny-DB, we recommend having a look on our [vision paper](https://ieeexplore.ieee.org/document/8622353).
 
-
 ## Getting Started ##
-The easiest way to setup Polypheny-DB is to use [Polypheny Control](https://github.com/polypheny/Polypheny-Control). 
-
+The easiest way to setup Polypheny-DB is to use [Polypheny Control](https://github.com/polypheny/Polypheny-Control).
 
 ## Roadmap ##
 See the [open issues](https://github.com/polypheny/Polypheny-DB/issues) for a list of proposed features (and known issues).
 
-
 ## Contributing ##
 We highly welcome your contributions to Polypheny-DB. If you would like to contribute, please fork the repository and submit your changes as a pull request. Please consult our [Admin Repository](https://github.com/polypheny/Admin) for guidelines and additional information.
 
-Please note that we have a [code of conduct](https://github.com/polypheny/Admin/blob/master/CODE_OF_CONDUCT.md). Please follow it in all your interactions with the project. 
-
+Please note that we have a [code of conduct](https://github.com/polypheny/Admin/blob/master/CODE_OF_CONDUCT.md). Please follow it in all your interactions with the project.
 
 ## Credits ##
 _Polypheny-DB_ builds upon the great work of several other projects:
@@ -35,10 +31,8 @@ _Polypheny-DB_ builds upon the great work of several other projects:
 
 Except for the first two, those projects are used "as is" and integrated as a library. _Apache Avatica_ we [forked](https://github.com/polypheny/Avatica) and made some Polypheny-DB specific adjustments. From _Apache Calcite_ we use parts of the code as foundation for Polypheny-DB.
 
-
 ## Acknowledgements
 The Polypheny-DB project is supported by the Swiss National Science Foundation (SNSF) under the contract no. 200021_172763.
 
-
 ## License ##
 The Apache 2.0 License

build.gradle

@@ -58,10 +58,6 @@ allprojects {
             // DBIS Nexus
             url "https://dbis-nexus.dmi.unibas.ch/repository/maven2/"
         }
-        maven {
-            url "https://clojars.org/repo/"
-        }
-        mavenCentral()
         maven {
             url "https://oss.sonatype.org/content/repositories/snapshots/"
         }
@@ -129,6 +125,9 @@ allprojects {
         logLevel 'lifecycle'
     }
 
+    lombok.config['lombok.val.flagUsage'] = 'error'
+    lombok.config['lombok.var.flagUsage'] = 'error'
+
     idea {
         module {
             downloadJavadoc = true

cassandra-adapter/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

catalog/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

config/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

config/src/main/java/org/polypheny/db/config/ConfigClazzList.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 The Polypheny Project
+ * Copyright 2019-2021 The Polypheny Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@
 public class ConfigClazzList extends Config {
 
     @JsonAdapter(ClassesAdapter.class)
-    @SerializedName( "values" )
+    @SerializedName("values")
     private final Set<Class> classes;
     @JsonAdapter(ValueAdapter.class)
     private final List<Class> value;
@@ -56,6 +56,14 @@ public ConfigClazzList( final String key, final Class superClass ) {
     }
 
 
+    public ConfigClazzList( final String key, final Class superClass, boolean activateAll ) {
+        this( key, superClass );
+        if ( activateAll ) {
+            setClazzList( new ArrayList<>( this.classes ) );
+        }
+    }
+
+
     public ConfigClazzList( final String key, final Class superClass, final List<Class> defaultValue ) {
         this( key, superClass );
         setClazzList( defaultValue );
@@ -139,9 +147,9 @@ public boolean parseStringAndSetValue( String value ) {
         Gson gson = new Gson();
         ArrayList<String> val = gson.fromJson( value, ArrayList.class );
         List<Class> toAdd = new ArrayList<>();
-        for( Class c: classes ) {
-            if( val.contains( c.getName() )){
-                toAdd.add(c);
+        for ( Class c : classes ) {
+            if ( val.contains( c.getName() ) ) {
+                toAdd.add( c );
             }
         }
         return this.setClazzList( toAdd );
@@ -173,6 +181,7 @@ public void write( final JsonWriter out, final List<Class> classes ) throws IOEx
             out.endArray();
         }
 
+
         @Override
         public List<Class> read( final JsonReader in ) throws IOException {
             List<Class> list = new ArrayList<>();
@@ -189,6 +198,7 @@ public List<Class> read( final JsonReader in ) throws IOException {
             in.endArray();
             return ImmutableList.copyOf( list );
         }
+
     }
 
 

core/build.gradle

@@ -53,7 +53,7 @@ dependencies {
     implementation group: 'com.j256.simplemagic', name: 'simplemagic', version: simplemagic_version // ISC
 
     api (group: "org.apache.calcite", name: "calcite-linq4j", version: calcite_linq4j_version) { exclude(module: "guava") } // Apache 2.0
-    api (group: "org.apache.calcite.avatica", name: "avatica-core", version: avatica_core_version) { exclude(module: "protobuf-java") } // Apache 2.0
+    api (group: "org.polypheny.avatica", name: "avatica-core", version: avatica_core_version) { exclude(module: "protobuf-java") } // Apache 2.0
 
     implementation group: "com.drewnoakes", name: "metadata-extractor", version: metadata_extractor_version // Apache 2.0
 
@@ -67,7 +67,7 @@ dependencies {
     testImplementation group: "junit", name: "junit", version: junit_version
     testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version  // BSD 3-clause
 
-    testImplementation group: "org.apache.calcite.avatica", name: "avatica-server", version: avatica_server_version  // Apache 2.0
+    testImplementation group: "org.polypheny.avatica", name: "avatica-server", version: avatica_server_version  // Apache 2.0
     testImplementation group: "org.incava", name: "java-diff", version: java_diff_version  // Apache 2.0
     testImplementation group: "org.apache.commons", name: "commons-pool2", version: commons_pool2_version // Apache 2.0
 

core/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

core/src/main/java/org/polypheny/db/adapter/index/Index.java

@@ -99,7 +99,7 @@ public void rebuild( final Transaction transaction ) {
                 .project( cols.stream().map( builder::field ).collect( Collectors.toList() ) )
                 .build();
         final QueryProcessor processor = statement.getQueryProcessor();
-        final PolyphenyDbSignature signature = processor.prepareQuery( RelRoot.of( scan, SqlKind.SELECT ) );
+        final PolyphenyDbSignature signature = processor.prepareQuery( RelRoot.of( scan, SqlKind.SELECT ), false );
         // Execute query
         final Iterable<Object> enumerable = signature.enumerable( statement.getDataContext() );
         final Iterator<Object> iterator = enumerable.iterator();

core/src/main/java/org/polypheny/db/catalog/Catalog.java

@@ -523,7 +523,8 @@ protected final boolean isValidIdentifier( final String str ) {
     public abstract List<CatalogColumnPlacement> getColumnPlacement( long columnId );
 
     /**
-     * Get column placements of a specific table on a specific adapter
+     * Get column placements of a specific table on a specific adapter on column detail level.
+     * Only returns one ColumnPlacement per column on adapter. Ignores multiplicity due to different partitionsIds
      *
      * @param adapterId The id of the adapter
      * @return List of column placements of the table on the specified adapter
@@ -533,7 +534,8 @@ protected final boolean isValidIdentifier( final String str ) {
     public abstract List<CatalogColumnPlacement> getColumnPlacementsOnAdapterSortedByPhysicalPosition( int storeId, long tableId );
 
     /**
-     * Get column placements on a adapter
+     * Get column placements on a adapter. On column detail level
+     * Only returns one ColumnPlacement per column on adapter. Ignores multiplicity due to different partitionsIds
      *
      * @param adapterId The id of the adapter
      * @return List of column placements on the specified adapter

core/src/main/java/org/polypheny/db/catalog/entity/CatalogView.java

@@ -33,7 +33,7 @@
 import org.polypheny.db.rel.RelCollation;
 import org.polypheny.db.rel.RelNode;
 import org.polypheny.db.rel.SingleRel;
-import org.polypheny.db.rel.logical.LogicalViewTableScan;
+import org.polypheny.db.rel.logical.LogicalViewScan;
 import org.polypheny.db.view.ViewManager.ViewVisitor;
 
 public class CatalogView extends CatalogTable {
@@ -199,8 +199,8 @@ public void prepareView( RelNode viewLogicalRoot, RelOptCluster relOptCluster )
         } else if ( viewLogicalRoot instanceof SingleRel ) {
             prepareView( ((SingleRel) viewLogicalRoot).getInput(), relOptCluster );
         }
-        if ( viewLogicalRoot instanceof LogicalViewTableScan ) {
-            prepareView( ((LogicalViewTableScan) viewLogicalRoot).getRelNode(), relOptCluster );
+        if ( viewLogicalRoot instanceof LogicalViewScan ) {
+            prepareView( ((LogicalViewScan) viewLogicalRoot).getRelNode(), relOptCluster );
         }
     }
 

core/src/main/java/org/polypheny/db/config/RuntimeConfig.java

@@ -269,6 +269,18 @@ public enum RuntimeConfig {
             ConfigType.INTEGER,
             "implementationCachingGroup" ),
 
+    ROUTING_PLAN_CACHING( "runtime/routingPlanCaching",
+            "Cache routing plans.",
+            true,
+            ConfigType.BOOLEAN,
+            "routingPlanCachingGroup" ),
+
+    ROUTING_PLAN_CACHING_SIZE( "runtime/routingPlanCachingSize",
+            "Size of the routing plan cache. If the limit is reached, the least recently used entry is removed.",
+            1000,
+            ConfigType.INTEGER,
+            "routingPlanCachingGroup" ),
+
     PARAMETERIZE_DML( "runtime/parameterizeDML",
             "Whether DML queries should be parameterized.",
             true,
@@ -327,29 +339,27 @@ public enum RuntimeConfig {
             ConfigType.INSTANCE_LIST,
             "dockerGroup" ),
 
-
     FILE_HANDLE_CACHE_SIZE( "runtime/fileHandleCacheSize",
             "Size (in Bytes) up to which media files are cached in-memory instead of creating a temporary file. Needs to be >= 0 and smaller than Integer.MAX_SIZE. Setting to zero disables caching of media files.",
             0,
             ConfigType.INTEGER,
             "runtimeExecutionGroup" ),
 
-
     QUEUE_PROCESSING_INTERVAL( "runtime/queueProcessingInterval",
             "Rate of passive tracking of statistics.",
-            BackgroundTask.TaskSchedulingType.EVERY_TEN_SECONDS_FIXED,
+            TaskSchedulingType.EVERY_SECOND,
             ConfigType.ENUM,
             "monitoringSettingsQueueGroup" ),
 
     QUEUE_PROCESSING_ELEMENTS( "runtime/queueProcessingElements",
             "Number of elements in workload queue to process per time.",
-            50,
+            5000,
             ConfigType.INTEGER,
             "monitoringSettingsQueueGroup" ),
 
     TEMPERATURE_FREQUENCY_PROCESSING_INTERVAL( "runtime/partitionFrequencyProcessingInterval",
             "Time interval in seconds, how often the access frequency of all TEMPERATURE-partitioned tables is analyzed and redistributed",
-            BackgroundTask.TaskSchedulingType.EVERY_MINUTE_FIXED,
+            BackgroundTask.TaskSchedulingType.EVERY_MINUTE,
             ConfigType.ENUM,
             "temperaturePartitionProcessingSettingsGroup" );
 

core/src/main/java/org/polypheny/db/monitoring/core/MonitoringService.java

@@ -18,8 +18,10 @@
 
 import java.sql.Timestamp;
 import java.util.List;
+import lombok.NonNull;
 import org.polypheny.db.monitoring.events.MonitoringDataPoint;
 import org.polypheny.db.monitoring.events.MonitoringEvent;
+import org.polypheny.db.monitoring.events.QueryPostCost;
 
 
 /**
@@ -44,6 +46,14 @@ public interface MonitoringService {
      */
     <T extends MonitoringDataPoint> List<T> getAllDataPoints( Class<T> dataPointClass );
 
+    /**
+     * Get number of data points for given monitoring persistent type.
+     *
+     * @param dataPointClass DatapointClass of interest
+     * @return Returns number of data points
+     */
+    <T extends MonitoringDataPoint> long getNumberOfDataPoints( @NonNull Class<T> dataPointClass );
+
     /**
      * Get data before specified timestamp for given monitoring persistent type.
      *
@@ -62,4 +72,26 @@ public interface MonitoringService {
      */
     <T extends MonitoringDataPoint> List<T> getDataPointsAfter( Class<T> dataPointClass, Timestamp timestamp );
 
+    /**
+     * @param physicalQueryClass the physical query class string to identify aggregated post costs.
+     * @return The aggregated query post costs.
+     */
+    QueryPostCost getQueryPostCosts( @NonNull String physicalQueryClass );
+
+    /**
+     * @return All query post costs for printing in ui.
+     */
+    List<QueryPostCost> getAllQueryPostCosts();
+
+    /**
+     * @param physicalQueryClass the physical query class string to identify aggregated post costs.
+     * @param executionTime the measured execution time.
+     */
+    void updateQueryPostCosts( @NonNull String physicalQueryClass, long executionTime );
+
+    /**
+     * Removes all aggregates post costs from cache.
+     */
+    void resetQueryPostCosts();
+
 }

core/src/main/java/org/polypheny/db/monitoring/events/QueryDataPoint.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019-2021 The Polypheny Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.polypheny.db.monitoring.events;
+
+public interface QueryDataPoint extends MonitoringDataPoint {
+
+    long getExecutionTime();
+
+    String getQueryId();
+
+    String getPhysicalQueryId();
+
+}

core/src/main/java/org/polypheny/db/monitoring/events/QueryPostCost.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019-2021 The Polypheny Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.polypheny.db.monitoring.events;
+
+/**
+ * Aggregation for routing post costs.
+ */
+public interface QueryPostCost {
+
+    /**
+     * @return physical query class name as string.
+     */
+    String getPhysicalQueryClass();
+
+    /**
+     * @return average execution time in nano seconds.
+     */
+    long getExecutionTime();
+
+    /**
+     * @return number of samples.
+     */
+    int getNumberOfSamples();
+
+}