From 0ae2b987914ff3a88c5140cc2ac83c78f6a7ccf9 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 16 Jan 2025 16:06:47 +0100 Subject: [PATCH] Make regression tests use default authentication ... and align realm names. --- regtests/docker-compose.yml | 9 +++------ regtests/run.sh | 15 +++++++++++++++ regtests/run_spark_sql.sh | 2 +- regtests/t_cli/src/test_cli.py | 2 +- regtests/t_pyspark/src/conftest.py | 4 ++-- regtests/t_pyspark/src/iceberg_spark.py | 2 +- .../src/test_spark_sql_s3_with_privileges.py | 8 ++++---- regtests/t_spark_sql/src/spark_sql_azure_blob.sh | 2 +- regtests/t_spark_sql/src/spark_sql_azure_dfs.sh | 2 +- regtests/t_spark_sql/src/spark_sql_basic.sh | 2 +- regtests/t_spark_sql/src/spark_sql_gcp.sh | 2 +- regtests/t_spark_sql/src/spark_sql_s3.sh | 2 +- .../t_spark_sql/src/spark_sql_s3_cross_region.sh | 2 +- regtests/t_spark_sql/src/spark_sql_views.sh | 2 +- 14 files changed, 34 insertions(+), 22 deletions(-) diff --git a/regtests/docker-compose.yml b/regtests/docker-compose.yml index 377caacdd..2a122b9b1 100644 --- a/regtests/docker-compose.yml +++ b/regtests/docker-compose.yml @@ -32,14 +32,11 @@ services: AZURE_TENANT_ID: $AZURE_TENANT_ID AZURE_CLIENT_ID: $AZURE_CLIENT_ID AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET + POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,root,secret polaris.persistence.type: in-memory - polaris.authentication.authenticator.type: test - polaris.authentication.token-service.type: test - polaris.authentication.token-broker.type: symmetric-key - polaris.authentication.token-broker.symmetric-key.secret: polaris polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES": '["FILE","S3","GCS","AZURE"]' - polaris.realm-context.realms: default-realm,realm1 - quarkus.log.file.enable: false + polaris.realm-context.realms: POLARIS + quarkus.log.file.enable: "false" quarkus.otel.sdk.disabled: "true" volumes: - ./credentials:/tmp/credentials/ diff --git a/regtests/run.sh b/regtests/run.sh index f91b2dcf3..1e396e59b 100755 --- a/regtests/run.sh +++ b/regtests/run.sh @@ -65,6 +65,21 @@ NUM_SUCCESSES=0 export AWS_ACCESS_KEY_ID='' export AWS_SECRET_ACCESS_KEY='' +if ! output=$(curl -X POST -H "Polaris-Realm: POLARIS" "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/oauth/tokens" \ + -d "grant_type=client_credentials" \ + -d "client_id=root" \ + -d "client_secret=secret" \ + -d "scope=PRINCIPAL_ROLE:ALL"); then + logred "Error: Failed to retrieve bearer token" + exit 1 +fi + +token=$(echo "$output" | awk -F\" '{print $4}') + +export REGTEST_ROOT_BEARER_TOKEN=$token + +echo "Root bearer token: ${REGTEST_ROOT_BEARER_TOKEN}" + for TEST_FILE in ${TEST_LIST}; do TEST_SUITE=$(dirname $(dirname ${TEST_FILE})) TEST_SHORTNAME=$(basename ${TEST_FILE}) diff --git a/regtests/run_spark_sql.sh b/regtests/run_spark_sql.sh index cc355095e..fdde29a13 100755 --- a/regtests/run_spark_sql.sh +++ b/regtests/run_spark_sql.sh @@ -55,7 +55,7 @@ if [ -z "${SPARK_HOME}"]; then export SPARK_HOME=$(realpath ~/${SPARK_DISTRIBUTION}) fi -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:default-realm}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" if [ $# -eq 0 ]; then # create a catalog backed by the local filesystem diff --git a/regtests/t_cli/src/test_cli.py b/regtests/t_cli/src/test_cli.py index c9af207d1..736f125ab 100644 --- a/regtests/t_cli/src/test_cli.py +++ b/regtests/t_cli/src/test_cli.py @@ -38,7 +38,7 @@ def get_salt(length=8) -> str: def root_cli(*args): - return cli('principal:root;realm:default-realm')(*args) + return cli(os.getenv('REGTEST_ROOT_BEARER_TOKEN', 'principal:root;realm:POLARIS'))(*args) def cli(access_token): diff --git a/regtests/t_pyspark/src/conftest.py b/regtests/t_pyspark/src/conftest.py index 041ef7b2c..5ce3f8178 100644 --- a/regtests/t_pyspark/src/conftest.py +++ b/regtests/t_pyspark/src/conftest.py @@ -66,7 +66,7 @@ def catalog_client(polaris_catalog_url): :return: """ client = CatalogApiClient( - Configuration(access_token=os.getenv('REGTEST_ROOT_BEARER_TOKEN', 'principal:root;realm:default-realm'), + Configuration(access_token=os.getenv('REGTEST_ROOT_BEARER_TOKEN', 'principal:root;realm:POLARIS'), host=polaris_catalog_url)) return IcebergCatalogAPI(client) @@ -143,7 +143,7 @@ def format_namespace(namespace): @pytest.fixture def root_client(polaris_host, polaris_url): - client = ApiClient(Configuration(access_token=os.getenv('REGTEST_ROOT_BEARER_TOKEN', 'principal:root;realm:default-realm'), + client = ApiClient(Configuration(access_token=os.getenv('REGTEST_ROOT_BEARER_TOKEN', 'principal:root;realm:POLARIS'), host=polaris_url)) api = PolarisDefaultApi(client) return api diff --git a/regtests/t_pyspark/src/iceberg_spark.py b/regtests/t_pyspark/src/iceberg_spark.py index db907ed2b..9b6a393d0 100644 --- a/regtests/t_pyspark/src/iceberg_spark.py +++ b/regtests/t_pyspark/src/iceberg_spark.py @@ -46,7 +46,7 @@ def __init__( aws_region: str = "us-west-2", catalog_name: str = None, polaris_url: str = None, - realm: str = 'default-realm' + realm: str = 'POLARIS' ): """Constructor for Iceberg Spark session. Sets the member variables.""" self.bearer_token = bearer_token diff --git a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py index 9ae9a1a39..ce00ce58c 100644 --- a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py +++ b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py @@ -154,7 +154,7 @@ def snowman_catalog_client(polaris_catalog_url, snowman): token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', client_id=snowman.principal.client_id, client_secret=snowman.credentials.client_secret, grant_type='client_credentials', - _headers={'realm': 'default-realm'}) + _headers={'realm': 'POLARIS'}) return IcebergCatalogAPI(CatalogApiClient(Configuration(access_token=token.access_token, host=polaris_catalog_url))) @@ -175,7 +175,7 @@ def creator_catalog_client(polaris_catalog_url, creator): token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', client_id=creator.principal.client_id, client_secret=creator.credentials.client_secret, grant_type='client_credentials', - _headers={'realm': 'default-realm'}) + _headers={'realm': 'POLARIS'}) return IcebergCatalogAPI(CatalogApiClient(Configuration(access_token=token.access_token, host=polaris_catalog_url))) @@ -230,7 +230,7 @@ def reader_catalog_client(polaris_catalog_url, reader): token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', client_id=reader.principal.client_id, client_secret=reader.credentials.client_secret, grant_type='client_credentials', - _headers={'realm': 'default-realm'}) + _headers={'realm': 'POLARIS'}) return IcebergCatalogAPI(CatalogApiClient(Configuration(access_token=token.access_token, host=polaris_catalog_url))) @@ -1015,7 +1015,7 @@ def create_principal(polaris_url, polaris_catalog_url, api, principal_name): token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', client_id=principal_result.principal.client_id, client_secret=principal_result.credentials.client_secret, grant_type='client_credentials', - _headers={'realm': 'default-realm'}) + _headers={'realm': 'POLARIS'}) rotate_client = ManagementApiClient(Configuration(access_token=token.access_token, host=polaris_url)) rotate_api = PolarisDefaultApi(rotate_client) diff --git a/regtests/t_spark_sql/src/spark_sql_azure_blob.sh b/regtests/t_spark_sql/src/spark_sql_azure_blob.sh index 21585b759..846692a9b 100755 --- a/regtests/t_spark_sql/src/spark_sql_azure_blob.sh +++ b/regtests/t_spark_sql/src/spark_sql_azure_blob.sh @@ -19,7 +19,7 @@ # under the License. # -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:realm1}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \ diff --git a/regtests/t_spark_sql/src/spark_sql_azure_dfs.sh b/regtests/t_spark_sql/src/spark_sql_azure_dfs.sh index 93ac6d7e5..d9562b849 100755 --- a/regtests/t_spark_sql/src/spark_sql_azure_dfs.sh +++ b/regtests/t_spark_sql/src/spark_sql_azure_dfs.sh @@ -19,7 +19,7 @@ # under the License. # -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:realm1}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \ diff --git a/regtests/t_spark_sql/src/spark_sql_basic.sh b/regtests/t_spark_sql/src/spark_sql_basic.sh index 2d5f1cc10..9d8d45d7a 100755 --- a/regtests/t_spark_sql/src/spark_sql_basic.sh +++ b/regtests/t_spark_sql/src/spark_sql_basic.sh @@ -19,7 +19,7 @@ # under the License. # -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:realm1}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \ diff --git a/regtests/t_spark_sql/src/spark_sql_gcp.sh b/regtests/t_spark_sql/src/spark_sql_gcp.sh index 0e124bf61..8638c12b3 100755 --- a/regtests/t_spark_sql/src/spark_sql_gcp.sh +++ b/regtests/t_spark_sql/src/spark_sql_gcp.sh @@ -19,7 +19,7 @@ # under the License. # -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:realm1}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \ diff --git a/regtests/t_spark_sql/src/spark_sql_s3.sh b/regtests/t_spark_sql/src/spark_sql_s3.sh index aebfdd585..181d68078 100755 --- a/regtests/t_spark_sql/src/spark_sql_s3.sh +++ b/regtests/t_spark_sql/src/spark_sql_s3.sh @@ -24,7 +24,7 @@ if [ -z "$AWS_TEST_ENABLED" ] || [ "$AWS_TEST_ENABLED" != "true" ]; then exit 0 fi -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:realm1}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \ diff --git a/regtests/t_spark_sql/src/spark_sql_s3_cross_region.sh b/regtests/t_spark_sql/src/spark_sql_s3_cross_region.sh index 7808dc8c3..d29c40058 100644 --- a/regtests/t_spark_sql/src/spark_sql_s3_cross_region.sh +++ b/regtests/t_spark_sql/src/spark_sql_s3_cross_region.sh @@ -24,7 +24,7 @@ if [ -z "$AWS_CROSS_REGION_TEST_ENABLED" ] || [ "$AWS_CROSS_REGION_TEST_ENABLED" exit 0 fi -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:realm1}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" BUCKET="${AWS_CROSS_REGION_BUCKET}" ROLE_ARN="${AWS_ROLE_FOR_CROSS_REGION_BUCKET}" diff --git a/regtests/t_spark_sql/src/spark_sql_views.sh b/regtests/t_spark_sql/src/spark_sql_views.sh index 50b2c59d5..ba6b66eee 100755 --- a/regtests/t_spark_sql/src/spark_sql_views.sh +++ b/regtests/t_spark_sql/src/spark_sql_views.sh @@ -19,7 +19,7 @@ # under the License. # -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:default-realm}" +SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:POLARIS}" curl -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs \