Universal Query Routing

README.md

@@ -8,20 +8,16 @@ _Polypheny-DB_ is a self-adaptive Polystore that provides cost- and workload awa
 
 If you want to learn more about Polypheny-DB, we recommend having a look on our [vision paper](https://ieeexplore.ieee.org/document/8622353).
 
-
 ## Getting Started ##
-The easiest way to setup Polypheny-DB is to use [Polypheny Control](https://github.com/polypheny/Polypheny-Control). 
-
+The easiest way to setup Polypheny-DB is to use [Polypheny Control](https://github.com/polypheny/Polypheny-Control).
 
 ## Roadmap ##
 See the [open issues](https://github.com/polypheny/Polypheny-DB/issues) for a list of proposed features (and known issues).
 
-
 ## Contributing ##
 We highly welcome your contributions to Polypheny-DB. If you would like to contribute, please fork the repository and submit your changes as a pull request. Please consult our [Admin Repository](https://github.com/polypheny/Admin) for guidelines and additional information.
 
-Please note that we have a [code of conduct](https://github.com/polypheny/Admin/blob/master/CODE_OF_CONDUCT.md). Please follow it in all your interactions with the project. 
-
+Please note that we have a [code of conduct](https://github.com/polypheny/Admin/blob/master/CODE_OF_CONDUCT.md). Please follow it in all your interactions with the project.
 
 ## Credits ##
 _Polypheny-DB_ builds upon the great work of several other projects:
@@ -35,10 +31,8 @@ _Polypheny-DB_ builds upon the great work of several other projects:
 
 Except for the first two, those projects are used "as is" and integrated as a library. _Apache Avatica_ we [forked](https://github.com/polypheny/Avatica) and made some Polypheny-DB specific adjustments. From _Apache Calcite_ we use parts of the code as foundation for Polypheny-DB.
 
-
 ## Acknowledgements
 The Polypheny-DB project is supported by the Swiss National Science Foundation (SNSF) under the contract no. 200021_172763.
 
-
 ## License ##
 The Apache 2.0 License

build.gradle

@@ -58,10 +58,6 @@ allprojects {
             // DBIS Nexus
             url "https://dbis-nexus.dmi.unibas.ch/repository/maven2/"
         }
-        maven {
-            url "https://clojars.org/repo/"
-        }
-        mavenCentral()
         maven {
             url "https://oss.sonatype.org/content/repositories/snapshots/"
         }
@@ -129,6 +125,9 @@ allprojects {
         logLevel 'lifecycle'
     }
 
+    lombok.config['lombok.val.flagUsage'] = 'error'
+    lombok.config['lombok.var.flagUsage'] = 'error'
+
     idea {
         module {
             downloadJavadoc = true

cassandra-adapter/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

catalog/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

catalog/src/test/java/org/polypheny/db/test/CatalogTest.java

@@ -333,7 +333,7 @@ public void testColumnPlacement() throws UnknownAdapterException {
         assertEquals( 2, catalog.getColumnPlacement( columnId ).size() );
         assertTrue( catalog.getColumnPlacement( columnId ).stream().map( p -> p.adapterId ).collect( Collectors.toList() ).containsAll( Arrays.asList( store2.id, store1.id ) ) );
 
-        catalog.deleteColumnPlacement( store1.id, columnId );
+        catalog.deleteColumnPlacement( store1.id, columnId, false );
         assertEquals( 1, catalog.getColumnPlacement( columnId ).size() );
         assertEquals( store2.id, catalog.getColumnPlacement( columnId ).get( 0 ).adapterId );
     }

config/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

config/src/main/java/org/polypheny/db/config/ConfigClazzList.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 The Polypheny Project
+ * Copyright 2019-2021 The Polypheny Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@
 public class ConfigClazzList extends Config {
 
     @JsonAdapter(ClassesAdapter.class)
-    @SerializedName( "values" )
+    @SerializedName("values")
     private final Set<Class> classes;
     @JsonAdapter(ValueAdapter.class)
     private final List<Class> value;
@@ -56,6 +56,14 @@ public ConfigClazzList( final String key, final Class superClass ) {
     }
 
 
+    public ConfigClazzList( final String key, final Class superClass, boolean activateAll ) {
+        this( key, superClass );
+        if ( activateAll ) {
+            setClazzList( new ArrayList<>( this.classes ) );
+        }
+    }
+
+
     public ConfigClazzList( final String key, final Class superClass, final List<Class> defaultValue ) {
         this( key, superClass );
         setClazzList( defaultValue );
@@ -139,9 +147,9 @@ public boolean parseStringAndSetValue( String value ) {
         Gson gson = new Gson();
         ArrayList<String> val = gson.fromJson( value, ArrayList.class );
         List<Class> toAdd = new ArrayList<>();
-        for( Class c: classes ) {
-            if( val.contains( c.getName() )){
-                toAdd.add(c);
+        for ( Class c : classes ) {
+            if ( val.contains( c.getName() ) ) {
+                toAdd.add( c );
             }
         }
         return this.setClazzList( toAdd );
@@ -173,6 +181,7 @@ public void write( final JsonWriter out, final List<Class> classes ) throws IOEx
             out.endArray();
         }
 
+
         @Override
         public List<Class> read( final JsonReader in ) throws IOException {
             List<Class> list = new ArrayList<>();
@@ -189,6 +198,7 @@ public List<Class> read( final JsonReader in ) throws IOException {
             in.endArray();
             return ImmutableList.copyOf( list );
         }
+
     }
 
 

core/build.gradle

@@ -53,7 +53,7 @@ dependencies {
     implementation group: 'com.j256.simplemagic', name: 'simplemagic', version: simplemagic_version // ISC
 
     api (group: "org.apache.calcite", name: "calcite-linq4j", version: calcite_linq4j_version) { exclude(module: "guava") } // Apache 2.0
-    api (group: "org.apache.calcite.avatica", name: "avatica-core", version: avatica_core_version) { exclude(module: "protobuf-java") } // Apache 2.0
+    api (group: "org.polypheny.avatica", name: "avatica-core", version: avatica_core_version) { exclude(module: "protobuf-java") } // Apache 2.0
 
     implementation group: "com.drewnoakes", name: "metadata-extractor", version: metadata_extractor_version // Apache 2.0
 
@@ -67,7 +67,7 @@ dependencies {
     testImplementation group: "junit", name: "junit", version: junit_version
     testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version  // BSD 3-clause
 
-    testImplementation group: "org.apache.calcite.avatica", name: "avatica-server", version: avatica_server_version  // Apache 2.0
+    testImplementation group: "org.polypheny.avatica", name: "avatica-server", version: avatica_server_version  // Apache 2.0
     testImplementation group: "org.incava", name: "java-diff", version: java_diff_version  // Apache 2.0
     testImplementation group: "org.apache.commons", name: "commons-pool2", version: commons_pool2_version // Apache 2.0
 

core/lombok.config

@@ -1,2 +1,4 @@
 # This file is generated by the 'io.freefair.lombok' Gradle plugin
 config.stopBubbling = true
+lombok.val.flagUsage = error
+lombok.var.flagUsage = error

core/src/main/java/org/polypheny/db/adapter/enumerable/EnumerableBindable.java

@@ -41,6 +41,7 @@
 import org.apache.calcite.linq4j.Enumerable;
 import org.apache.calcite.linq4j.Enumerator;
 import org.polypheny.db.adapter.DataContext;
+import org.polypheny.db.adapter.enumerable.EnumerableRel.Prefer;
 import org.polypheny.db.interpreter.BindableConvention;
 import org.polypheny.db.interpreter.BindableRel;
 import org.polypheny.db.interpreter.Node;
@@ -92,7 +93,7 @@ public Class<Object[]> getElementType() {
     @Override
     public Enumerable<Object[]> bind( DataContext dataContext ) {
         final Map<String, Object> map = new HashMap<>();
-        final Bindable bindable = EnumerableInterpretable.toBindable( map, null, (EnumerableRel) getInput(), EnumerableRel.Prefer.ARRAY, dataContext.getStatement() );
+        final Bindable bindable = EnumerableInterpretable.toBindable( map, null, (EnumerableRel) getInput(), Prefer.ARRAY, dataContext.getStatement() ).left;
         final ArrayBindable arrayBindable = EnumerableInterpretable.box( bindable );
         dataContext.addAll( map );
         return arrayBindable.bind( dataContext );

core/src/main/java/org/polypheny/db/adapter/enumerable/EnumerableInterpretable.java

@@ -49,11 +49,8 @@
 import org.codehaus.commons.compiler.IClassBodyEvaluator;
 import org.codehaus.commons.compiler.ICompilerFactory;
 import org.polypheny.db.adapter.DataContext;
+import org.polypheny.db.adapter.enumerable.EnumerableRel.Prefer;
 import org.polypheny.db.config.RuntimeConfig;
-import org.polypheny.db.information.InformationCode;
-import org.polypheny.db.information.InformationGroup;
-import org.polypheny.db.information.InformationManager;
-import org.polypheny.db.information.InformationPage;
 import org.polypheny.db.interpreter.BindableConvention;
 import org.polypheny.db.interpreter.Compiler;
 import org.polypheny.db.interpreter.InterpretableConvention;
@@ -73,6 +70,7 @@
 import org.polypheny.db.runtime.Typed;
 import org.polypheny.db.runtime.Utilities;
 import org.polypheny.db.transaction.Statement;
+import org.polypheny.db.util.Pair;
 import org.polypheny.db.util.Util;
 
 
@@ -101,15 +99,15 @@ public Node implement( final InterpreterImplementor implementor ) {
                 implementor.internalParameters,
                 implementor.spark,
                 (EnumerableRel) getInput(),
-                EnumerableRel.Prefer.ARRAY,
-                implementor.dataContext.getStatement() );
+                Prefer.ARRAY,
+                implementor.dataContext.getStatement() ).left;
         final ArrayBindable arrayBindable = box( bindable );
         final Enumerable<Object[]> enumerable = arrayBindable.bind( implementor.dataContext );
         return new EnumerableNode( enumerable, implementor.compiler, this );
     }
 
 
-    public static Bindable toBindable( Map<String, Object> parameters, SparkHandler spark, EnumerableRel rel, EnumerableRel.Prefer prefer, Statement statement ) {
+    public static Pair<Bindable<Object[]>, String> toBindable( Map<String, Object> parameters, SparkHandler spark, EnumerableRel rel, EnumerableRel.Prefer prefer, Statement statement ) {
         EnumerableRelImplementor relImplementor = new EnumerableRelImplementor( rel.getCluster().getRexBuilder(), parameters );
 
         final ClassDeclaration expr = relImplementor.implementRoot( rel, prefer );
@@ -119,24 +117,13 @@ public static Bindable toBindable( Map<String, Object> parameters, SparkHandler
             Util.debugCode( System.out, s );
         }
 
-        if ( statement != null && statement.getTransaction().isAnalyze() ) {
-            InformationManager queryAnalyzer = statement.getTransaction().getQueryAnalyzer();
-            InformationPage page = new InformationPage( "Generated Code" );
-            page.fullWidth();
-            InformationGroup group = new InformationGroup( page, "Generated Code" );
-            queryAnalyzer.addPage( page );
-            queryAnalyzer.addGroup( group );
-            InformationCode informationCode = new InformationCode( group, s );
-            queryAnalyzer.registerInformation( informationCode );
-        }
-
         Hook.JAVA_PLAN.run( s );
 
         try {
             if ( spark != null && spark.enabled() ) {
-                return spark.compile( expr, s );
+                return new Pair<>( spark.compile( expr, s ), s );
             } else {
-                return getBindable( expr, s, rel.getRowType().getFieldCount() );
+                return new Pair<>( getBindable( expr, s, rel.getRowType().getFieldCount() ), s );
             }
         } catch ( Exception e ) {
             throw Helper.INSTANCE.wrap( "Error while compiling generated Java code:\n" + s, e );

core/src/main/java/org/polypheny/db/adapter/index/Index.java

@@ -99,7 +99,7 @@ public void rebuild( final Transaction transaction ) {
                 .project( cols.stream().map( builder::field ).collect( Collectors.toList() ) )
                 .build();
         final QueryProcessor processor = statement.getQueryProcessor();
-        final PolyphenyDbSignature signature = processor.prepareQuery( RelRoot.of( scan, SqlKind.SELECT ) );
+        final PolyphenyDbSignature signature = processor.prepareQuery( RelRoot.of( scan, SqlKind.SELECT ), false );
         // Execute query
         final Iterable<Object> enumerable = signature.enumerable( statement.getDataContext() );
         final Iterator<Object> iterator = enumerable.iterator();

core/src/main/java/org/polypheny/db/catalog/Catalog.java

@@ -492,8 +492,9 @@ protected final boolean isValidIdentifier( final String str ) {
      *
      * @param adapterId The id of the adapter
      * @param columnId The id of the column
+     * @param columnOnly columnOnly If delete originates from a dropColumn
      */
-    public abstract void deleteColumnPlacement( int adapterId, long columnId );
+    public abstract void deleteColumnPlacement( int adapterId, long columnId, boolean columnOnly );
 
     /**
      * Gets a collective list of column placements per column on a adapter.
@@ -523,7 +524,8 @@ protected final boolean isValidIdentifier( final String str ) {
     public abstract List<CatalogColumnPlacement> getColumnPlacement( long columnId );
 
     /**
-     * Get column placements of a specific table on a specific adapter
+     * Get column placements of a specific table on a specific adapter on column detail level.
+     * Only returns one ColumnPlacement per column on adapter. Ignores multiplicity due to different partitionsIds
      *
      * @param adapterId The id of the adapter
      * @return List of column placements of the table on the specified adapter
@@ -533,7 +535,8 @@ protected final boolean isValidIdentifier( final String str ) {
     public abstract List<CatalogColumnPlacement> getColumnPlacementsOnAdapterSortedByPhysicalPosition( int storeId, long tableId );
 
     /**
-     * Get column placements on a adapter
+     * Get column placements on a adapter. On column detail level
+     * Only returns one ColumnPlacement per column on adapter. Ignores multiplicity due to different partitionsIds
      *
      * @param adapterId The id of the adapter
      * @return List of column placements on the specified adapter

core/src/main/java/org/polypheny/db/catalog/entity/CatalogAdapter.java

@@ -17,6 +17,7 @@
 package org.polypheny.db.catalog.entity;
 
 
+import com.google.common.collect.ImmutableMap;
 import java.io.Serializable;
 import java.util.Map;
 import lombok.EqualsAndHashCode;
@@ -32,7 +33,7 @@ public class CatalogAdapter implements CatalogEntity {
     public final String uniqueName;
     public final String adapterClazz;
     public final AdapterType type;
-    public final Map<String, String> settings;
+    public final ImmutableMap<String, String> settings;
 
 
     public enum AdapterType {STORE, SOURCE}
@@ -48,7 +49,7 @@ public CatalogAdapter(
         this.uniqueName = uniqueName;
         this.adapterClazz = adapterClazz;
         this.type = adapterType;
-        this.settings = settings;
+        this.settings = ImmutableMap.copyOf( settings );
     }