diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 22e9b5c7325..243a3478a93 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -11,11 +11,13 @@ on: options: - 'gravitino' - 'gravitino-ci-hive' + - 'gravitino-ci-kerberos-hive' - 'gravitino-ci-trino' - 'gravitino-ci-doris' + - 'gravitino-ci-ranger' - 'trino' - 'hive' - - 'gravitino-ci-kerberos-hive' + - 'ranger' tag: description: 'Docker tag to apply to this image' required: true @@ -47,6 +49,9 @@ jobs: elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-doris" ]; then echo "image_type=doris" >> $GITHUB_ENV echo "image_name=datastrato/gravitino-ci-doris" >> $GITHUB_ENV + elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-ranger" ]; then + echo "image_type=ranger" >> $GITHUB_ENV + echo "image_name=datastrato/gravitino-ci-ranger" >> $GITHUB_ENV elif [ "${{ github.event.inputs.image }}" == "gravitino" ]; then echo "image_type=gravitino" >> $GITHUB_ENV echo "image_name=datastrato/gravitino" >> $GITHUB_ENV @@ -56,6 +61,9 @@ jobs: elif [ "${{ github.event.inputs.image }}" == "hive" ]; then echo "image_type=hive" >> $GITHUB_ENV echo "image_name=datastrato/hive" >> $GITHUB_ENV + elif [ "${{ github.event.inputs.image }}" == "ranger" ]; then + echo "image_type=ranger" >> $GITHUB_ENV + echo "image_name=datastrato/ranger" >> $GITHUB_ENV fi - name: Check publish Docker token diff --git a/dev/docker/build-docker.sh b/dev/docker/build-docker.sh index 067dbd214fa..11c0b2043ed 100755 --- a/dev/docker/build-docker.sh +++ b/dev/docker/build-docker.sh @@ -13,7 +13,7 @@ usage() { cat << EOF Usage: -./build-docker.sh --platform [all|linux/amd64|linux/arm64] --type [gravitino|hive|trino|doris|kerberos-hive] --image {image_name} --tag {tag_name} --latest +./build-docker.sh --platform [all|linux/amd64|linux/arm64] --type [gravitino|hive|kerberos-hive|trino|doris|ranger] --image {image_name} --tag {tag_name} --latest Notice: You shouldn't use 'all' for the platform if you don't use the Github action to publish the Docker image. EOF @@ -84,6 +84,9 @@ elif [ "${component_type}" == "gravitino" ]; then elif [ "${component_type}" == "doris" ]; then . ${script_dir}/doris/doris-dependency.sh --platform ${platform_type} build_args="--build-arg DORIS_VERSION=${DORIS_VERSION}" +elif [ "${component_type}" == "ranger" ]; then + # Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive` + build_args="--build-arg RANGER_VERSION=2.4.0 --build-arg RANGER_PLUGINS=plugin-trino,plugin-hive" else echo "ERROR : ${component_type} is not a valid component type" usage @@ -99,7 +102,7 @@ if echo "${builders}" | grep -q "${BUILDER_NAME}"; then echo "BuildKit builder '${BUILDER_NAME}' already exists." else echo "BuildKit builder '${BUILDER_NAME}' does not exist." - docker buildx create --platform linux/amd64,linux/arm64 --use --name ${BUILDER_NAME} + docker buildx create --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=10000000 --platform linux/amd64,linux/arm64 --use --name ${BUILDER_NAME} fi cd ${script_dir}/${component_type} diff --git a/dev/docker/ranger/Dockerfile b/dev/docker/ranger/Dockerfile new file mode 100644 index 00000000000..fcd186e1c4b --- /dev/null +++ b/dev/docker/ranger/Dockerfile @@ -0,0 +1,79 @@ +# +# Copyright 2023 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. +# +# Apache Ranger compile Docker image +FROM debian:buster as compile-ranger +LABEL maintainer="support@datastrato.com" + +ARG RANGER_VERSION=2.4.0 +# Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive` +ARG RANGER_PLUGINS=plugin-trino + +WORKDIR /root + +RUN apt-get -q update && \ + apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps + +RUN wget https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz && \ + tar zxvf apache-maven-3.6.3-bin.tar.gz && \ + ln -s /root/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn + +ENV JAVA_HOME=/usr/local/jdk +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ + ln -s /usr/lib/jvm/java-11-openjdk-arm64 ${JAVA_HOME}; \ + else \ + ln -s /usr/lib/jvm/java-11-openjdk-amd64 ${JAVA_HOME}; \ + fi + +RUN wget https://downloads.apache.org/ranger/${RANGER_VERSION}/apache-ranger-${RANGER_VERSION}.tar.gz && \ + tar zxvf apache-ranger-${RANGER_VERSION}.tar.gz && \ + ln -s apache-ranger-${RANGER_VERSION} apache-ranger && \ + cd apache-ranger && \ + mvn -pl ${RANGER_PLUGINS},jisql,agents-audit,agents-common,agents-cred,agents-installer,credentialbuilder,embeddedwebserver,security-admin,ranger-util,ranger-plugin-classloader,ranger-tools,distro -am -DskipTests=true compile package + +# Apache Ranger Admin runtime Docker image +FROM debian:buster +LABEL maintainer="support@datastrato.com" + +ARG RANGER_VERSION=2.4.0 +# Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive` +ARG RANGER_PLUGINS=plugin-trino +ENV RANGER_PASSWORD=rangerR0cks! + +WORKDIR /root + +COPY init-mysql.sql.template /tmp/ +COPY start-ranger-services.sh /tmp/ +RUN chmod +x /tmp/start-ranger-services.sh + +RUN apt-get -q update && \ + apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps + +ENV JAVA_HOME=/usr/local/jdk +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ + ln -s /usr/lib/jvm/java-11-openjdk-arm64 ${JAVA_HOME}; \ + else \ + ln -s /usr/lib/jvm/java-11-openjdk-amd64 ${JAVA_HOME}; \ + fi + +COPY --from=compile-ranger /root/apache-ranger/target/ranger-${RANGER_VERSION}-admin.tar.gz /opt +RUN cd /opt && \ + tar zxvf ranger-${RANGER_VERSION}-admin.tar.gz && \ + ln -s ranger-${RANGER_VERSION}-admin ranger-admin + +# Initialize Ranger envirioment +RUN curl -L https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar --output /opt/ranger-admin/ews/webapp/WEB-INF/lib/mysql-connector-java-8.0.28.jar && \ + cp /opt/ranger-admin/ews/webapp/WEB-INF/lib/mysql-connector-java-8.0.28.jar /opt/ranger-admin/jisql/lib/ && \ + curl -L https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2/log4jdbc-1.2.jar --output /opt/ranger-admin/ews/webapp/WEB-INF/lib/log4jdbc-1.2.jar && \ + cp -r /opt/ranger-admin/ews/webapp/WEB-INF/classes/conf.dist/ /opt/ranger-admin/ews/webapp/WEB-INF/classes/conf && \ + mkdir /opt/ranger-admin/ews/logs + +# Clean up +RUN rm -rf /var/lib/apt/lists/* + +EXPOSE 6080 + +ENTRYPOINT ["/bin/bash", "-c", "/tmp/start-ranger-services.sh"] diff --git a/dev/docker/ranger/init-mysql.sql.template b/dev/docker/ranger/init-mysql.sql.template new file mode 100644 index 00000000000..2ba86ebdb83 --- /dev/null +++ b/dev/docker/ranger/init-mysql.sql.template @@ -0,0 +1,9 @@ +--- +-- Copyright 2023 Datastrato Pvt Ltd. +-- This software is licensed under the Apache License version 2. +--- +CREATE USER 'rangeradmin'@'localhost' IDENTIFIED BY 'PLACEHOLDER_RANGER_PASSWORD'; +CREATE DATABASE ranger; +GRANT ALL PRIVILEGES ON ranger.* TO 'rangeradmin'@'localhost'; +UPDATE mysql.user SET plugin='mysql_native_password' WHERE User='root'; +FLUSH PRIVILEGES; diff --git a/dev/docker/ranger/start-ranger-services.sh b/dev/docker/ranger/start-ranger-services.sh new file mode 100755 index 00000000000..2b46bf3fd27 --- /dev/null +++ b/dev/docker/ranger/start-ranger-services.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright 2023 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. +# + +# Initial Ranger database in MySQL +sed "s/PLACEHOLDER_RANGER_PASSWORD/${RANGER_PASSWORD}/g" "/tmp/init-mysql.sql.template" > "/tmp/init-mysql.sql" +service mysql start && mysql -uroot < /tmp/init-mysql.sql + +# Update Ranger Admin password and setup Ranger Admin +sed -i 's/audit_store=solr/audit_store=DB/g' /opt/ranger-admin/install.properties +sed -i "s/db_password=/db_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties +sed -i "s/rangerAdmin_password=/rangerAdmin_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties +sed -i "s/rangerTagsync_password=/rangerTagsync_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties +sed -i "s/rangerUsersync_password=/rangerUsersync_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties +sed -i "s/keyadmin_password=/keyadmin_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties +sed -i 's/check_java_version/#check_java_version/g' /opt/ranger-admin/setup.sh +sed -i 's/#check_java_version()/check_java_version()/g' /opt/ranger-admin/setup.sh +sed -i 's/check_db_connector/#check_db_connector/g' /opt/ranger-admin/setup.sh +sed -i 's/#check_db_connector()/check_db_connector()/g' /opt/ranger-admin/setup.sh +sed -i 's/copy_db_connector/#copy_db_connector/g' /opt/ranger-admin/setup.sh +sed -i 's/#copy_db_connector()/copy_db_connector()/g' /opt/ranger-admin/setup.sh +cd /opt/ranger-admin && /opt/ranger-admin/setup.sh + +# Start Ranger Admin +/opt/ranger-admin/ews/ranger-admin-services.sh start + +# persist the container +tail -f /dev/null diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index 98305096c69..fafe86bc12f 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -231,3 +231,16 @@ Changelog - Expose ports: - `8030` Doris FE HTTP port - `9030` Doris FE MySQL server port + +## Gravitino CI Apache Ranger image + +You can use this image to control Trino's permissions. + +Changelog + +- gravitino-ci-ranger:0.1.0 + - Docker image `datastrato/gravitino-ci-ranger:0.1.0` + - Support Apache Ranger 2.4.0 + - Use environment variable `RANGER_PASSWORD` to set up Apache Ranger admin password, Please notice Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. + - Expose ports: + - `6080` Apache Ranger admin port diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 09f46d70a6a..04b733be3ae 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -60,6 +60,7 @@ awaitility = "4.2.1" servlet = "3.1.0" jodd = "3.5.2" flink = "1.18.0" +ranger = "2.4.0" protobuf-plugin = "0.9.2" spotless-plugin = '6.11.0' @@ -167,7 +168,7 @@ sun-activation = { group = "com.sun.activation", name = "javax.activation", vers kafka-clients = { group = "org.apache.kafka", name = "kafka-clients", version.ref = "kafka" } kafka = { group = "org.apache.kafka", name = "kafka_2.12", version.ref = "kafka" } curator-test = { group = "org.apache.curator", name = "curator-test", version.ref = "curator"} - +ranger-intg = { group = "org.apache.ranger", name = "ranger-intg", version.ref = "ranger" } selenium = { group = "org.seleniumhq.selenium", name = "selenium-java", version.ref = "selenium" } rauschig = { group = "org.rauschig", name = "jarchivelib", version.ref = "rauschig" } mybatis = { group = "org.mybatis", name = "mybatis", version.ref = "mybatis"} diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index d8e277ad52f..68b0da08dab 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -30,6 +30,16 @@ dependencies { testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) testImplementation(libs.testcontainers.postgresql) + testImplementation(libs.ranger.intg) { + exclude("org.apache.hadoop", "hadoop-common") + exclude("org.apache.hive", "hive-storage-api") + exclude("org.apache.lucene") + exclude("org.apache.solr") + exclude("org.apache.kafka") + exclude("org.elasticsearch") + exclude("org.elasticsearch.client") + exclude("org.elasticsearch.plugin") + } testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index e0ba06c44cb..76bd402310c 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -40,6 +40,7 @@ public class ContainerSuite implements Closeable { private static volatile HiveContainer hiveContainer; private static volatile TrinoContainer trinoContainer; private static volatile TrinoITContainers trinoITContainers; + private static volatile RangerContainer rangerContainer; private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; private static volatile HiveContainer kerberosHiveContainer; @@ -304,6 +305,29 @@ public HiveContainer getHiveContainer() { return hiveContainer; } + public void startRangerContainer() { + if (rangerContainer == null) { + synchronized (ContainerSuite.class) { + if (rangerContainer == null) { + // Start Ranger container + RangerContainer.Builder rangerBuilder = RangerContainer.builder().withNetwork(network); + RangerContainer container = closer.register(rangerBuilder.build()); + try { + container.start(); + } catch (Exception e) { + LOG.error("Failed to start Ranger container", e); + throw new RuntimeException("Failed to start Ranger container", e); + } + rangerContainer = container; + } + } + } + } + + public RangerContainer getRangerContainer() { + return rangerContainer; + } + public HiveContainer getKerberosHiveContainer() { return kerberosHiveContainer; } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java new file mode 100644 index 00000000000..4eccbdfadb7 --- /dev/null +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java @@ -0,0 +1,118 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.integration.test.container; + +import static java.lang.String.format; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.ranger.RangerClient; +import org.apache.ranger.RangerServiceException; +import org.rnorth.ducttape.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Network; + +public class RangerContainer extends BaseContainer { + public static final Logger LOG = LoggerFactory.getLogger(RangerContainer.class); + + public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_RANGER_DOCKER_IMAGE"); + public static final String HOST_NAME = "gravitino-ci-ranger"; + public static final int RANGER_PORT = 6080; + public RangerClient rangerClient; + private String rangerUrl; + private static final String username = "admin"; + // Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. + private static final String password = "rangerR0cks!"; + /* for kerberos authentication: + authType = "kerberos" + username = principal + password = path of the keytab file */ + private static final String authType = "simple"; + + public static Builder builder() { + return new Builder(); + } + + protected RangerContainer( + String image, + String hostName, + Set ports, + Map extraHosts, + Map filesToMount, + Map envVars, + Optional network) { + super(image, hostName, ports, extraHosts, filesToMount, envVars, network); + } + + @Override + protected void setupContainer() { + super.setupContainer(); + withLogConsumer(new PrintingContainerLog(format("%-15s| ", "RangerContainer"))); + } + + @Override + public void start() { + super.start(); + + rangerUrl = String.format("http://localhost:%s", this.getMappedPort(6080)); + rangerClient = new RangerClient(rangerUrl, authType, username, password, null); + + Preconditions.check("Ranger container startup failed!", checkContainerStatus(10)); + } + + @Override + protected boolean checkContainerStatus(int retryLimit) { + int nRetry = 0; + boolean isRangerContainerReady = false; + int sleepTimeMillis = 3_000; + while (nRetry++ < retryLimit) { + try { + rangerClient.getPluginsInfo(); + isRangerContainerReady = true; + LOG.info("Ranger container startup success!"); + break; + } catch (RangerServiceException e) { + LOG.warn("Check Ranger startup status... {}", e.getMessage()); + } + if (!isRangerContainerReady) { + try { + Thread.sleep(sleepTimeMillis); + LOG.warn("Waiting for Ranger server to be ready... ({}ms)", nRetry * sleepTimeMillis); + } catch (InterruptedException e) { + // ignore + } + } + } + + return isRangerContainerReady; + } + + @Override + public void close() { + super.close(); + } + + public static class Builder + extends BaseContainer.Builder { + + private Builder() { + this.image = DEFAULT_IMAGE; + this.hostName = HOST_NAME; + this.exposePorts = ImmutableSet.of(RANGER_PORT); + this.envVars = + ImmutableMap.builder().put("RANGER_PASSWORD", password).build(); + } + + @Override + public RangerContainer build() { + return new RangerContainer( + image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); + } + } +} diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index afc6dad1601..a98608f999c 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -108,7 +108,16 @@ dependencies { exclude("jakarta.annotation") } testImplementation(libs.trino.jdbc) - + testImplementation(libs.ranger.intg) { + exclude("org.apache.hadoop", "hadoop-common") + exclude("org.apache.hive", "hive-storage-api") + exclude("org.apache.lucene") + exclude("org.apache.solr") + exclude("org.apache.kafka") + exclude("org.elasticsearch") + exclude("org.elasticsearch.client") + exclude("org.elasticsearch.plugin") + } testRuntimeOnly(libs.junit.jupiter.engine) } @@ -137,6 +146,7 @@ tasks.test { environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.5") environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0") environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.3") + environment("GRAVITINO_CI_RANGER_DOCKER_IMAGE", "datastrato/gravitino-ci-ranger:0.1.0") copy { from("${project.rootDir}/dev/docker/trino/conf") diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/authorization/ranger/RangerIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/authorization/ranger/RangerIT.java new file mode 100644 index 00000000000..beb505a5719 --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/authorization/ranger/RangerIT.java @@ -0,0 +1,73 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.integration.test.authorization.ranger; + +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.ranger.RangerClient; +import org.apache.ranger.RangerServiceException; +import org.apache.ranger.plugin.model.RangerService; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("gravitino-docker-it") +public class RangerIT { + private static final String serviceName = "trino-test"; + private static final String trinoType = "trino"; + private static RangerClient rangerClient; + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + + @BeforeAll + public static void setup() { + containerSuite.startRangerContainer(); + + rangerClient = containerSuite.getRangerContainer().rangerClient; + } + + @AfterAll + public static void cleanup() throws RangerServiceException { + if (rangerClient != null) { + rangerClient.deleteService(serviceName); + } + } + + @Test + public void testCreateTrinoService() throws RangerServiceException { + String usernameKey = "username"; + String usernameVal = "admin"; + String jdbcKey = "jdbc.driverClassName"; + String jdbcVal = "io.trino.jdbc.TrinoDriver"; + String jdbcUrlKey = "jdbc.url"; + String jdbcUrlVal = "http://localhost:8080"; + + RangerService service = new RangerService(); + service.setType(trinoType); + service.setName(serviceName); + service.setConfigs( + ImmutableMap.builder() + .put(usernameKey, usernameVal) + .put(jdbcKey, jdbcVal) + .put(jdbcUrlKey, jdbcUrlVal) + .build()); + + RangerService createdService = rangerClient.createService(service); + Assertions.assertNotNull(createdService); + + Map filter = Collections.emptyMap(); + List services = rangerClient.findServices(filter); + Assertions.assertEquals(services.get(0).getName(), serviceName); + Assertions.assertEquals(services.get(0).getType(), trinoType); + Assertions.assertEquals(services.get(0).getConfigs().get(usernameKey), usernameVal); + Assertions.assertEquals(services.get(0).getConfigs().get(jdbcKey), jdbcVal); + Assertions.assertEquals(services.get(0).getConfigs().get(jdbcUrlKey), jdbcUrlVal); + } +}