From a3a39a279a65228e9d10c63cace789e9fb11e79c Mon Sep 17 00:00:00 2001 From: Francesco Di Chiara Date: Tue, 8 May 2018 17:19:13 -0700 Subject: [PATCH] Remove itests. Fix jdbc url. Update Redshift jdbc driver --- .travis.yml | 18 ------------------ project/SparkRedshiftBuild.scala | 2 +- .../AWSCredentialsInUriIntegrationSuite.scala | 1 - .../spark/redshift/IntegrationSuiteBase.scala | 9 ++++++--- ...hiftCredentialsInConfIntegrationSuite.scala | 4 ++-- 5 files changed, 9 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2e0505b..a4cf233b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,24 +25,6 @@ matrix: - jdk: openjdk7 scala: 2.11.7 env: HADOOP_VERSION="2.2.0" SPARK_VERSION="2.0.0" SPARK_AVRO_VERSION="3.0.0" AWS_JAVA_SDK_VERSION="1.7.4" -env: - global: - # AWS_REDSHIFT_JDBC_URL - - secure: "RNkxdKcaKEYuJqxli8naazp42qO5/pgueIzs+J5rHwl39jcBvJMgW3DX8kT7duzdoBb/qrolj/ttbQ3l/30P45+djn0BEwcJMX7G/FGpZYD23yd03qeq7sOKPQl2Ni/OBttYHJMah5rI6aPmAysBZMQO7Wijdenb/RUiU2YcZp0=" - # AWS_REDSHIFT_PASSWORD - - secure: "g5li3gLejD+/2BIqIm+qHiqBUvCc5l0qnftVaVlLtL7SffErp/twDiFP4gW8eqnFqi2GEC1c9Shf7Z9cOIUunNSBQZdYIVG0f38UfBeDP14nOoIuwZ974O5yggbgZhX0cKvJzINcENGoRNk0FzRwgOdCCiF05IMnRqQxI3C24fE=" - # AWS_REDSHIFT_USER - - secure: "LIkY/ZpBXK3vSFsdpBSRXEsgfD2wDF52X8OZOlyBJOiZpS4y1/obj8b3VQABDPyPH95bGX/LOpM0vVM137rYgF0pskgVEzLMyZOPpwYqNGPf/d4BtQhBRc8f7+jmr6D4Hrox4jCl0cCKaeiTazun2+Y9E+zgCUDvQ8y9qGctR2k=" - # TEST_AWS_ACCESS_KEY_ID - - secure: "bsB6YwkscUxtzcZOKja4Y69IR3JqvCP3W/4vFftW/v33/hOC3EBz7TVNKS+ZIomBUQYJnzsMfM59bj7YEc3KZe8WxIcUdLI40hg0X5O1RhJDNPW+0oGbWshmzyua+hY1y7nRja+8/17tYTbAi1+MhscRu+O/2aWaXolA9BicuX0=" - # TEST_AWS_SECRET_ACCESS_KEY - - secure: "cGxnZh4be9XiPBOMxe9wHYwEfrWNw4zSjmvGFEC9UUV11ydHLo5wrXtcTVFmY7qxUxYeb0NB2N+CQXE0GcyUKoTviKG9sOS3cxR1q30FsdOVcWDKAzpBUmzDTMwDLAUMysziyOtMorDlNVydqYdYLMpiUN0O+eDKA+iOHlJp7fo=" - # STS_ROLE_ARN - - secure: "cuyemI1bqPkWBD5B1FqIKDJb5g/SX5x8lrzkO0J/jkyGY0VLbHxrl5j/9PrKFuvraBK3HC56HEP1Zg+IMvh+uv0D+p5y14C97fAzE33uNgR2aVkamOo92zHvxvXe7zBtqc8rztWsJb1pgkrY7SdgSXgQc88ohey+XecDh4TahTY=" - # AWS_S3_SCRATCH_SPACE - - secure: "LvndQIW6dHs6nyaMHtblGI/oL+s460lOezFs2BoD0Isenb/O/IM+nY5K9HepTXjJIcq8qvUYnojZX1FCrxxOXX2/+/Iihiq7GzJYdmdMC6hLg9bJYeAFk0dWYT88/AwadrJCBOa3ockRLhiO3dkai7Ki5+M1erfaFiAHHMpJxYQ=" - # AWS_S3_CROSS_REGION_SCRATCH_SPACE - - secure: "esYmBqt256Dc77HT68zoaE/vtsFGk2N+Kt+52RlR0cjHPY1q5801vxLbeOlpYb2On3x8YckE++HadjL40gwSBsca0ffoogq6zTlfbJYDSQkQG1evxXWJZLcafB0igfBs/UbEUo7EaxoAJQcLgiWWwUdO0a0iU1ciSVyogZPagL0=" script: - ./dev/run-tests-travis.sh diff --git a/project/SparkRedshiftBuild.scala b/project/SparkRedshiftBuild.scala index 1a5301f9..b3cab872 100644 --- a/project/SparkRedshiftBuild.scala +++ b/project/SparkRedshiftBuild.scala @@ -75,7 +75,7 @@ object SparkRedshiftBuild extends Build { // A Redshift-compatible JDBC driver must be present on the classpath for spark-redshift to work. // For testing, we use an Amazon driver, which is available from // http://docs.aws.amazon.com/redshift/latest/mgmt/configure-jdbc-connection.html - "com.amazon.redshift" % "jdbc4" % "1.1.7.1007" % "test" from "https://s3.amazonaws.com/redshift-downloads/drivers/RedshiftJDBC4-1.1.7.1007.jar", + "com.amazon.redshift" % "jdbc41" % "1.2.12.1017" % "test" from "https://s3.amazonaws.com/redshift-downloads/drivers/jdbc/1.2.12.1017/RedshiftJDBC41-1.2.12.1017.jar", // Although support for the postgres driver is lower priority than support for Amazon's // official Redshift driver, we still run basic tests with it. "postgresql" % "postgresql" % "8.3-606.jdbc4" % "test", diff --git a/src/it/scala/com/databricks/spark/redshift/AWSCredentialsInUriIntegrationSuite.scala b/src/it/scala/com/databricks/spark/redshift/AWSCredentialsInUriIntegrationSuite.scala index a5061c2a..7bf2f14c 100644 --- a/src/it/scala/com/databricks/spark/redshift/AWSCredentialsInUriIntegrationSuite.scala +++ b/src/it/scala/com/databricks/spark/redshift/AWSCredentialsInUriIntegrationSuite.scala @@ -44,7 +44,6 @@ class AWSCredentialsInUriIntegrationSuite extends IntegrationSuiteBase { // Override this method so that we do not set the credentials in sc.hadoopConf. override def beforeAll(): Unit = { assert(tempDir.contains("AKIA"), "tempdir did not contain AWS credentials") - assert(!AWS_SECRET_ACCESS_KEY.contains("/"), "AWS secret key should not contain slash") sc = new SparkContext("local", getClass.getSimpleName) conn = DefaultJDBCWrapper.getConnector(None, jdbcUrl, None) } diff --git a/src/it/scala/com/databricks/spark/redshift/IntegrationSuiteBase.scala b/src/it/scala/com/databricks/spark/redshift/IntegrationSuiteBase.scala index f635e528..4a188abf 100644 --- a/src/it/scala/com/databricks/spark/redshift/IntegrationSuiteBase.scala +++ b/src/it/scala/com/databricks/spark/redshift/IntegrationSuiteBase.scala @@ -54,16 +54,19 @@ trait IntegrationSuiteBase protected val AWS_REDSHIFT_JDBC_URL: String = loadConfigFromEnv("AWS_REDSHIFT_JDBC_URL") protected val AWS_REDSHIFT_USER: String = loadConfigFromEnv("AWS_REDSHIFT_USER") protected val AWS_REDSHIFT_PASSWORD: String = loadConfigFromEnv("AWS_REDSHIFT_PASSWORD") - protected val AWS_ACCESS_KEY_ID: String = loadConfigFromEnv("TEST_AWS_ACCESS_KEY_ID") - protected val AWS_SECRET_ACCESS_KEY: String = loadConfigFromEnv("TEST_AWS_SECRET_ACCESS_KEY") + protected val AWS_ACCESS_KEY_ID: String = loadConfigFromEnv("AWS_ACCESS_KEY_ID") + protected val AWS_SECRET_ACCESS_KEY: String = loadConfigFromEnv("AWS_SECRET_ACCESS_KEY") // Path to a directory in S3 (e.g. 's3n://bucket-name/path/to/scratch/space'). protected val AWS_S3_SCRATCH_SPACE: String = loadConfigFromEnv("AWS_S3_SCRATCH_SPACE") require(AWS_S3_SCRATCH_SPACE.contains("s3n"), "must use s3n:// URL") protected def jdbcUrl: String = { - s"$AWS_REDSHIFT_JDBC_URL?user=$AWS_REDSHIFT_USER&password=$AWS_REDSHIFT_PASSWORD" + s"$AWS_REDSHIFT_JDBC_URL?user=$AWS_REDSHIFT_USER&password=$AWS_REDSHIFT_PASSWORD&ssl=true" } + protected def jdbcUrlNoUserPassword: String = { + s"$AWS_REDSHIFT_JDBC_URL?ssl=true" + } /** * Random suffix appended appended to table and directory names in order to avoid collisions * between separate Travis builds. diff --git a/src/it/scala/com/databricks/spark/redshift/RedshiftCredentialsInConfIntegrationSuite.scala b/src/it/scala/com/databricks/spark/redshift/RedshiftCredentialsInConfIntegrationSuite.scala index c7566e79..b51bf3bf 100644 --- a/src/it/scala/com/databricks/spark/redshift/RedshiftCredentialsInConfIntegrationSuite.scala +++ b/src/it/scala/com/databricks/spark/redshift/RedshiftCredentialsInConfIntegrationSuite.scala @@ -31,14 +31,14 @@ class RedshiftCredentialsInConfIntegrationSuite extends IntegrationSuiteBase { val tableName = s"roundtrip_save_and_load_$randomSuffix" try { write(df) - .option("url", AWS_REDSHIFT_JDBC_URL) + .option("url", jdbcUrlNoUserPassword) .option("user", AWS_REDSHIFT_USER) .option("password", AWS_REDSHIFT_PASSWORD) .option("dbtable", tableName) .save() assert(DefaultJDBCWrapper.tableExists(conn, tableName)) val loadedDf = read - .option("url", AWS_REDSHIFT_JDBC_URL) + .option("url", jdbcUrlNoUserPassword) .option("user", AWS_REDSHIFT_USER) .option("password", AWS_REDSHIFT_PASSWORD) .option("dbtable", tableName)