From 0b4f5f3418e13c14b9eb232c565f9da3eae591b9 Mon Sep 17 00:00:00 2001 From: Blair Chen Date: Fri, 20 Jan 2023 15:30:58 +0800 Subject: [PATCH] Revert "Adding azure-cosmos-package to databricks spark config (#972)" (#1002) This reverts commit 3859fca9ada53dce5af36b2bfec88c8e9d180d26. --- .github/workflows/pull_request_push_test.yml | 2 +- build.gradle | 3 +-- feathr-impl/build.gradle | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 670e16a0f..5cb10fa1c 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -108,7 +108,7 @@ jobs: SPARK_CONFIG__SPARK_CLUSTER: databricks SPARK_CONFIG__DATABRICKS__WORKSPACE_INSTANCE_URL: ${{secrets.DATABRICKS_HOST}} DATABRICKS_WORKSPACE_TOKEN_VALUE: ${{secrets.DATABRICKS_WORKSPACE_TOKEN_VALUE}} - SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"${{secrets.DATABRICKS_INSTANCE_POOL_ID}}"},"libraries":[{"maven": {"coordinates": "com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.16.0"}}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' + SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"${{secrets.DATABRICKS_INSTANCE_POOL_ID}}"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' REDIS_PASSWORD: ${{secrets.REDIS_PASSWORD}} AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}} AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}} diff --git a/build.gradle b/build.gradle index b8c550258..19b9d62a7 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ dependencies { implementation 'net.snowflake:snowflake-jdbc:3.13.18' implementation 'net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2' provided 'com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21' - implementation 'com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.16.0' + provided 'com.azure.cosmos.spark:azure-cosmos-spark_3-2_2-12:4.11.1' provided 'com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8' provided 'org.eclipse.jetty:jetty-util:9.3.24.v20180605' provided 'org.apache.kafka:kafka-clients:3.1.0' @@ -130,7 +130,6 @@ project.ext.spec = [ 'avro' : "org.apache.avro:avro:1.10.2", "avroUtil": "com.linkedin.avroutil1:helper-all:0.2.100", "azure": "com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21", - "spark_cosmos": "com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.16.0", 'fastutil' : "it.unimi.dsi:fastutil:8.1.1", 'mvel' : "org.mvel:mvel2:2.2.8.Final", 'protobuf' : "com.google.protobuf:protobuf-java:2.6.1", diff --git a/feathr-impl/build.gradle b/feathr-impl/build.gradle index 7f9894727..055fcd4c0 100644 --- a/feathr-impl/build.gradle +++ b/feathr-impl/build.gradle @@ -48,7 +48,6 @@ dependencies { implementation spec.product.jackson.dataformat_hocon implementation spec.product.jackson.jackson_core implementation spec.product.spark_redis - implementation spec.product.spark_cosmos implementation spec.product.fastutil implementation spec.product.hadoop.mapreduce_client_core implementation spec.product.mvel @@ -78,7 +77,6 @@ dependencies { testImplementation spec.product.equalsverifier testImplementation spec.product.spark.spark_catalyst - testImplementation spec.product.spark_cosmos testImplementation spec.product.mockito testImplementation spec.product.scala.scalatest testImplementation spec.product.testing