fix develop (#63)

palantir · Jan 23, 2020 · c0b0d19 · c0b0d19
1 parent e7ac344
commit c0b0d19
Show file tree

Hide file tree

Showing 23 changed files with 252 additions and 363 deletions.
diff --git a/.baseline/checkstyle/checkstyle-suppressions.xml b/.baseline/checkstyle/checkstyle-suppressions.xml
@@ -1,24 +1,7 @@
 <?xml version="1.0"?>
-
-<!--
-  ~ (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
 <!DOCTYPE suppressions PUBLIC
-    "-//Puppy Crawl//DTD Suppressions 1.1//EN"
-    "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
+    "-//Checkstyle//DTD SuppressionFilter Configuration 1.2//EN"
+    "https://checkstyle.org/dtds/suppressions_1_2.dtd">
 
 <!-- IMPORTANT ECLIPSE NOTE: If you change this file, you must restart Eclipse
  for your changes to take effect in its Checkstyle integration. -->

diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml
@@ -1,23 +1,7 @@
 <?xml version="1.0"?>
-<!--
-  ~ (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
 <!DOCTYPE module PUBLIC
-        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
-        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+        "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
+        "https://checkstyle.org/dtds/configuration_1_3.dtd">
 
 <!--
     Palantir Baseline Checkstyle configuration.
@@ -69,6 +53,9 @@
         <property name="optional" value="true"/>
     </module>
     <module name="SuppressWarningsFilter"/> <!-- baseline-gradle: README.md -->
+    <module name="BeforeExecutionExclusionFileFilter">
+        <property name="fileNamePattern" value="module\-info\.java$"/>
+    </module>
     <module name="TreeWalker">
         <module name="SuppressionCommentFilter"/> <!-- baseline-gradle: README.md -->
         <module name="SuppressionCommentFilter">
@@ -100,11 +87,13 @@
         <module name="AvoidStarImport"/> <!-- Java Style Guide: No wildcard imports -->
         <module name="AvoidStaticImport"> <!-- Java Style Guide: No static imports -->
             <property name="excludes" value="
+                com.google.common.base.Preconditions.*,
+                com.palantir.logsafe.Preconditions.*,
                 java.util.Collections.*,
                 java.util.stream.Collectors.*,
-                com.palantir.logsafe.Preconditions.*,
-                com.google.common.base.Preconditions.*,
-                org.apache.commons.lang3.Validate.*"/>
+                org.apache.commons.lang3.Validate.*,
+                org.assertj.core.api.Assertions.*,
+                org.mockito.Mockito.*"/>
         </module>
         <module name="ClassTypeParameterName"> <!-- Java Style Guide: Type variable names -->
             <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
@@ -126,7 +115,6 @@
         <module name="EmptyStatement"/> <!-- Java Style Guide: One statement per line -->
         <module name="EqualsHashCode"/>
         <module name="FallThrough"/> <!-- Java Style Guide: Fall-through: commented -->
-        <module name="FinalClass"/> <!-- Java Coding Guidelines: Private constructors -->
         <module name="GenericWhitespace"> <!-- Java Style Guide: Horizontal whitespace -->
             <message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
             <message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
@@ -143,6 +131,11 @@
             <property name="illegalPkgs" value="junit.framework"/>
             <message key="import.illegal" value="Use JUnit 4-style (org.junit.*) test classes and assertions instead of JUnit 3 (junit.framework.*)."/>
         </module>
+        <module name="IllegalImport"> <!-- Only relevant for pre-Java 11 because javafx is gone completely in Java 11 -->
+            <property name="id" value="BanJavafx"/>
+            <property name="illegalPkgs" value="javafx"/>
+            <message key="import.illegal" value="Must not import javafx classes because some OpenJDK builds do not include javafx."/>
+        </module>
         <module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
             <property name="illegalPkgs" value="org.elasticsearch.common.base, com.clearspring.analytics.util, org.spark_project.guava"/>
             <message key="import.illegal" value="Must not import repackaged classes."/>
@@ -380,10 +373,6 @@
             <property name="format" value="\bIOUtils\.toString\("/>
             <property name="message" value="Prefer Guava''s [CharStreams,Files,Resources].toString to avoid charset/stream closing issues."/>
         </module>
-        <module name="RegexpSinglelineJava">
-            <property name="format" value="static enum"/>
-            <property name="message" value="Redundant ''static'' modifier."/>
-        </module>
         <module name="RegexpSinglelineJava">
             <property name="format" value="\/\/TODO|\/\/ TODO(?!\([^()\s]+\): )"/>
             <property name="message" value="TODO format: // TODO(#issue): explanation"/>
@@ -433,6 +422,7 @@
             <property name="allowEmptyMethods" value="true"/>
             <property name="allowEmptyTypes" value="true"/>
             <property name="allowEmptyLoops" value="true"/>
+            <property name="allowEmptyLambdas" value="true"/>
             <property name="ignoreEnhancedForColon" value="false"/>
             <message key="ws.notFollowed" value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
             <message key="ws.notPreceded" value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
@@ -446,7 +436,7 @@
         </module>
         <module name="CyclomaticComplexity"/> <!-- Java Coding Guidelines: Reduce Cyclomatic Complexity -->
         <module name="DesignForExtension"> <!-- Java Coding Guidelines: Design for extension -->
-            <property name="ignoredAnnotations" value="Test, Before, BeforeEach, After, AfterEach, BeforeClass, BeforeAll, AfterClass, AfterAll"/>
+            <property name="ignoredAnnotations" value="ParameterizedTest, Test, Before, BeforeEach, After, AfterEach, BeforeClass, BeforeAll, AfterClass, AfterAll"/>
         </module>
         <module name="JavadocMethod"> <!-- Java Style Guide: Where Javadoc is used -->
             <property name="scope" value="public"/>
@@ -479,10 +469,9 @@
         <module name="NestedTryDepth"/> <!-- Java Coding Guide: Try/catch blocks: never nested -->
         <module name="NonEmptyAtclauseDescription"/> <!-- Java Style Guide: At-clauses -->
         <module name="ParameterName"> <!-- Java Style Guide: Parameter names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
+            <property name="format" value="^_?[a-z][a-zA-Z0-9]+$"/>
             <message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
         </module>
-        <module name="SingleLineJavadoc"/> <!-- Java Style Guide: General form -->
         <module name="SummaryJavadocCheck"> <!-- Java Coding Guidelines: Javadoc -->
             <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
         </module>

diff --git a/.baseline/idea/intellij-java-palantir-style.xml b/.baseline/idea/intellij-java-palantir-style.xml
@@ -1,19 +1,3 @@
-<!--
-  ~ (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
 <project version="4">
   <component name="ProjectCodeStyleSettingsManager">
     <option name="PER_PROJECT_SETTINGS">

diff --git a/build.gradle b/build.gradle
@@ -24,11 +24,15 @@ buildscript {
     }
 
     dependencies {
-        classpath 'com.palantir.gradle.consistentversions:gradle-consistent-versions:1.15.0'
-        classpath 'com.palantir.baseline:gradle-baseline-java:0.50.0'
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4'
+        classpath 'com.netflix.nebula:gradle-info-plugin:5.2.0'
+        classpath 'com.netflix.nebula:nebula-publishing-plugin:14.1.1'
+        classpath 'com.palantir.baseline:gradle-baseline-java:2.49.1'
         classpath 'com.palantir.gradle.conjure:gradle-conjure:4.15.0'
-        classpath 'com.palantir.sls-packaging:gradle-sls-packaging:3.3.0'
+        classpath 'com.palantir.gradle.consistentversions:gradle-consistent-versions:1.15.0'
         classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.11.0';
+        classpath 'com.palantir.javaformat:gradle-palantir-java-format:0.3.12'
+        classpath 'com.palantir.sls-packaging:gradle-sls-packaging:3.3.0'
         classpath 'gradle.plugin.org.inferred:gradle-processors:3.3.0'
     }
 }
@@ -44,6 +48,7 @@ apply plugin: 'com.palantir.consistent-versions'
 version gitVersion()
 
 allprojects {
+    apply plugin: 'com.palantir.java-format'
     version rootProject.version
     group 'com.palantir.spark.tpcds'
 
@@ -71,4 +76,3 @@ configure(subprojects) {
     tasks.check.dependsOn(javadoc)
     sourceCompatibility = 1.8
 }
-
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,21 +1,5 @@
-#
-# (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.4-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
@@ -1,5 +1,21 @@
 #!/usr/bin/env sh
 
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 ##############################################################################
 ##
 ##  Gradle start up script for UN*X
@@ -28,7 +44,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m"'
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
@@ -109,8 +125,8 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
     JAVACMD=`cygpath --unix "$JAVACMD"`

diff --git a/gradlew.bat b/gradlew.bat
@@ -1,3 +1,19 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
 @if "%DEBUG%" == "" @echo off
 @rem ##########################################################################
 @rem
@@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m"
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome

diff --git a/spark-tpcds-benchmark-runner/build.gradle b/spark-tpcds-benchmark-runner/build.gradle
@@ -17,12 +17,14 @@
 plugins {
     id 'com.palantir.docker' version '0.21.0'
     id 'com.palantir.sls-java-service-distribution'
+    id 'maven-publish'
 }
 
 dependencies {
     compile 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8'
     compile 'com.fasterxml.jackson.datatype:jackson-datatype-guava'
     compile 'com.google.guava:guava'
+    compile 'com.palantir.safe-logging:preconditions'
     compile 'org.apache.spark:spark-sql_2.11'
 
     testCompile 'junit:junit'

diff --git a/spark-tpcds-benchmark-runner/src/main/java/com/palantir/spark/tpcds/TpcdsBenchmark.java b/spark-tpcds-benchmark-runner/src/main/java/com/palantir/spark/tpcds/TpcdsBenchmark.java
@@ -46,14 +46,8 @@ public final class TpcdsBenchmark {
 
     private static final Logger log = LoggerFactory.getLogger(TpcdsBenchmark.class);
 
-    private static final ImmutableSet<String> BLACKLISTED_QUERIES = ImmutableSet.of(
-            "q23b.sql",
-            "q39a.sql",
-            "q39b.sql",
-            "q14b.sql",
-            "q49.sql",
-            "q64.sql",
-            "q77.sql");
+    private static final ImmutableSet<String> BLACKLISTED_QUERIES =
+            ImmutableSet.of("q23b.sql", "q39a.sql", "q39b.sql", "q14b.sql", "q49.sql", "q64.sql", "q77.sql");
 
     private final TpcdsBenchmarkConfig config;
     private final TpcdsDataGenerator dataGenerator;
@@ -87,25 +81,21 @@ public TpcdsBenchmark(
     public void run() throws IOException {
         dataGenerator.generateDataIfNecessary();
         for (int iteration = 0; iteration < config.iterations(); iteration++) {
-            log.info("Beginning benchmark iteration.",
+            log.info(
+                    "Beginning benchmark iteration.",
                     SafeArg.of("currentIteration", iteration),
                     SafeArg.of("totalNumIterations", config.iterations()));
             config.dataScalesGb().forEach(scale -> {
-                log.info("Beginning benchmarks at a new data scale.",
-                        SafeArg.of("dataScale", scale));
+                log.info("Beginning benchmarks at a new data scale.", SafeArg.of("dataScale", scale));
                 registration.registerTables(scale);
                 queries.get().forEach((queryName, query) -> {
-                    log.info("Running query.",
-                            SafeArg.of("queryName", queryName),
-                            SafeArg.of("queryStatement", query));
+                    log.info("Running query.", SafeArg.of("queryName", queryName), SafeArg.of("queryStatement", query));
                     try {
                         String resultLocation = paths.experimentResultLocation(scale, queryName);
                         Path resultPath = new Path(resultLocation);
                         if (dataFileSystem.exists(resultPath) && !dataFileSystem.delete(resultPath, true)) {
-                            throw new IllegalStateException(
-                                    String.format(
-                                            "Failed to clear experiment result destination directory at %s.",
-                                            resultPath));
+                            throw new IllegalStateException(String.format(
+                                    "Failed to clear experiment result destination directory at %s.", resultPath));
                         }
                         Dataset<Row> queryResultDataset = sanitizeColumnNames(spark.sql(query));
                         spark.sparkContext().setJobDescription(String.format("%s-benchmark", queryName));
@@ -121,23 +111,25 @@ public void run() throws IOException {
                                 metrics.abortBenchmark();
                             }
                         }
-                        log.info("Successfully ran query. Will now proceed to verify the correctness.",
+                        log.info(
+                                "Successfully ran query. Will now proceed to verify the correctness.",
                                 SafeArg.of("queryName", queryName),
                                 SafeArg.of("scale", scale));
                         correctness.verifyCorrectness(
                                 scale, queryName, query, queryResultDataset.schema(), resultLocation);
-                        log.info("Successfully verified correctness of a query.",
+                        log.info(
+                                "Successfully verified correctness of a query.",
                                 SafeArg.of("queryName", queryName),
                                 SafeArg.of("scale", scale));
                     } catch (IOException e) {
                         throw new RuntimeException(e);
                     }
                 });
-                log.info("Successfully ran benchmarks at a given scale.",
-                        SafeArg.of("scale", scale));
+                log.info("Successfully ran benchmarks at a given scale.", SafeArg.of("scale", scale));
             });
             metrics.flushMetrics();
-            log.info("Successfully finished one iteration of benchmarks at all scales.",
+            log.info(
+                    "Successfully finished one iteration of benchmarks at all scales.",
                     SafeArg.of("completedIterations", iteration));
         }
         log.info("Successfully ran all benchmarks at the requested number of iterations. Exiting.");
@@ -146,9 +138,9 @@ public void run() throws IOException {
     private static Map<String, String> getQueries() {
         ImmutableMap.Builder<String, String> queries = ImmutableMap.builder();
         try (TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(
-                TpcdsBenchmark.class.getClassLoader().getResourceAsStream("queries.tar"));
-                InputStreamReader tarArchiveReader = new InputStreamReader(
-                        tarArchiveInputStream, StandardCharsets.UTF_8)) {
+                        TpcdsBenchmark.class.getClassLoader().getResourceAsStream("queries.tar"));
+                InputStreamReader tarArchiveReader =
+                        new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)) {
             TarArchiveEntry entry;
             while ((entry = tarArchiveInputStream.getNextTarEntry()) != null) {
                 String queryString = CharStreams.toString(tarArchiveReader);