forked from apache/gravitino
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[apache#2411] Add Apache Ranger Docker CI image and integration test (a…
…pache#2490) ### What changes were proposed in this pull request? This future includes： 1. A Gravitino Ranger CI Docker image 2. Add Ranger integration test 3. `docker run -it -p 6080:6080 datastrato/gravitino-ci-ranger` ### Why are the changes needed? Created an Apache Ranger Docker image myself, because Apache Ranger's official Docker image has 5 child docker images, It's very complex. Fix: apache#2411 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? 1. Created [datastrato/gravitino-ci-ranger:0.1.0](https://hub.docker.com/r/datastrato/gravitino-ci-ranger/tags) Docker image and pushed hub.docker.com succeed. 2. CI Passed
- Loading branch information
Showing
12 changed files
with
383 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| # | ||
| # Copyright 2023 Datastrato Pvt Ltd. | ||
| # This software is licensed under the Apache License version 2. | ||
| # | ||
| # Apache Ranger compile Docker image | ||
| FROM debian:buster as compile-ranger | ||
| LABEL maintainer="[email protected]" | ||
|
|
||
| ARG RANGER_VERSION=2.4.0 | ||
| # Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive` | ||
| ARG RANGER_PLUGINS=plugin-trino | ||
|
|
||
| WORKDIR /root | ||
|
|
||
| RUN apt-get -q update && \ | ||
| apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps | ||
|
|
||
| RUN wget https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz && \ | ||
| tar zxvf apache-maven-3.6.3-bin.tar.gz && \ | ||
| ln -s /root/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn | ||
|
|
||
| ENV JAVA_HOME=/usr/local/jdk | ||
| RUN ARCH=$(uname -m) && \ | ||
| if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ | ||
| ln -s /usr/lib/jvm/java-11-openjdk-arm64 ${JAVA_HOME}; \ | ||
| else \ | ||
| ln -s /usr/lib/jvm/java-11-openjdk-amd64 ${JAVA_HOME}; \ | ||
| fi | ||
|
|
||
| RUN wget https://downloads.apache.org/ranger/${RANGER_VERSION}/apache-ranger-${RANGER_VERSION}.tar.gz && \ | ||
| tar zxvf apache-ranger-${RANGER_VERSION}.tar.gz && \ | ||
| ln -s apache-ranger-${RANGER_VERSION} apache-ranger && \ | ||
| cd apache-ranger && \ | ||
| mvn -pl ${RANGER_PLUGINS},jisql,agents-audit,agents-common,agents-cred,agents-installer,credentialbuilder,embeddedwebserver,security-admin,ranger-util,ranger-plugin-classloader,ranger-tools,distro -am -DskipTests=true compile package | ||
|
|
||
| # Apache Ranger Admin runtime Docker image | ||
| FROM debian:buster | ||
| LABEL maintainer="[email protected]" | ||
|
|
||
| ARG RANGER_VERSION=2.4.0 | ||
| # Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive` | ||
| ARG RANGER_PLUGINS=plugin-trino | ||
| ENV RANGER_PASSWORD=rangerR0cks! | ||
|
|
||
| WORKDIR /root | ||
|
|
||
| COPY init-mysql.sql.template /tmp/ | ||
| COPY start-ranger-services.sh /tmp/ | ||
| RUN chmod +x /tmp/start-ranger-services.sh | ||
|
|
||
| RUN apt-get -q update && \ | ||
| apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps | ||
|
|
||
| ENV JAVA_HOME=/usr/local/jdk | ||
| RUN ARCH=$(uname -m) && \ | ||
| if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ | ||
| ln -s /usr/lib/jvm/java-11-openjdk-arm64 ${JAVA_HOME}; \ | ||
| else \ | ||
| ln -s /usr/lib/jvm/java-11-openjdk-amd64 ${JAVA_HOME}; \ | ||
| fi | ||
|
|
||
| COPY --from=compile-ranger /root/apache-ranger/target/ranger-${RANGER_VERSION}-admin.tar.gz /opt | ||
| RUN cd /opt && \ | ||
| tar zxvf ranger-${RANGER_VERSION}-admin.tar.gz && \ | ||
| ln -s ranger-${RANGER_VERSION}-admin ranger-admin | ||
|
|
||
| # Initialize Ranger envirioment | ||
| RUN curl -L https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar --output /opt/ranger-admin/ews/webapp/WEB-INF/lib/mysql-connector-java-8.0.28.jar && \ | ||
| cp /opt/ranger-admin/ews/webapp/WEB-INF/lib/mysql-connector-java-8.0.28.jar /opt/ranger-admin/jisql/lib/ && \ | ||
| curl -L https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2/log4jdbc-1.2.jar --output /opt/ranger-admin/ews/webapp/WEB-INF/lib/log4jdbc-1.2.jar && \ | ||
| cp -r /opt/ranger-admin/ews/webapp/WEB-INF/classes/conf.dist/ /opt/ranger-admin/ews/webapp/WEB-INF/classes/conf && \ | ||
| mkdir /opt/ranger-admin/ews/logs | ||
|
|
||
| # Clean up | ||
| RUN rm -rf /var/lib/apt/lists/* | ||
|
|
||
| EXPOSE 6080 | ||
|
|
||
| ENTRYPOINT ["/bin/bash", "-c", "/tmp/start-ranger-services.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| --- | ||
| -- Copyright 2023 Datastrato Pvt Ltd. | ||
| -- This software is licensed under the Apache License version 2. | ||
| --- | ||
| CREATE USER 'rangeradmin'@'localhost' IDENTIFIED BY 'PLACEHOLDER_RANGER_PASSWORD'; | ||
| CREATE DATABASE ranger; | ||
| GRANT ALL PRIVILEGES ON ranger.* TO 'rangeradmin'@'localhost'; | ||
| UPDATE mysql.user SET plugin='mysql_native_password' WHERE User='root'; | ||
| FLUSH PRIVILEGES; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2023 Datastrato Pvt Ltd. | ||
| # This software is licensed under the Apache License version 2. | ||
| # | ||
|
|
||
| # Initial Ranger database in MySQL | ||
| sed "s/PLACEHOLDER_RANGER_PASSWORD/${RANGER_PASSWORD}/g" "/tmp/init-mysql.sql.template" > "/tmp/init-mysql.sql" | ||
| service mysql start && mysql -uroot < /tmp/init-mysql.sql | ||
|
|
||
| # Update Ranger Admin password and setup Ranger Admin | ||
| sed -i 's/audit_store=solr/audit_store=DB/g' /opt/ranger-admin/install.properties | ||
| sed -i "s/db_password=/db_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties | ||
| sed -i "s/rangerAdmin_password=/rangerAdmin_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties | ||
| sed -i "s/rangerTagsync_password=/rangerTagsync_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties | ||
| sed -i "s/rangerUsersync_password=/rangerUsersync_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties | ||
| sed -i "s/keyadmin_password=/keyadmin_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties | ||
| sed -i 's/check_java_version/#check_java_version/g' /opt/ranger-admin/setup.sh | ||
| sed -i 's/#check_java_version()/check_java_version()/g' /opt/ranger-admin/setup.sh | ||
| sed -i 's/check_db_connector/#check_db_connector/g' /opt/ranger-admin/setup.sh | ||
| sed -i 's/#check_db_connector()/check_db_connector()/g' /opt/ranger-admin/setup.sh | ||
| sed -i 's/copy_db_connector/#copy_db_connector/g' /opt/ranger-admin/setup.sh | ||
| sed -i 's/#copy_db_connector()/copy_db_connector()/g' /opt/ranger-admin/setup.sh | ||
| cd /opt/ranger-admin && /opt/ranger-admin/setup.sh | ||
|
|
||
| # Start Ranger Admin | ||
| /opt/ranger-admin/ews/ranger-admin-services.sh start | ||
|
|
||
| # persist the container | ||
| tail -f /dev/null |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
118 changes: 118 additions & 0 deletions
118
...on/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| /* | ||
| * Copyright 2023 Datastrato Pvt Ltd. | ||
| * This software is licensed under the Apache License version 2. | ||
| */ | ||
| package com.datastrato.gravitino.integration.test.container; | ||
|
|
||
| import static java.lang.String.format; | ||
|
|
||
| import com.google.common.collect.ImmutableMap; | ||
| import com.google.common.collect.ImmutableSet; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import org.apache.ranger.RangerClient; | ||
| import org.apache.ranger.RangerServiceException; | ||
| import org.rnorth.ducttape.Preconditions; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.testcontainers.containers.Network; | ||
|
|
||
| public class RangerContainer extends BaseContainer { | ||
| public static final Logger LOG = LoggerFactory.getLogger(RangerContainer.class); | ||
|
|
||
| public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_RANGER_DOCKER_IMAGE"); | ||
| public static final String HOST_NAME = "gravitino-ci-ranger"; | ||
| public static final int RANGER_PORT = 6080; | ||
| public RangerClient rangerClient; | ||
| private String rangerUrl; | ||
| private static final String username = "admin"; | ||
| // Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. | ||
| private static final String password = "rangerR0cks!"; | ||
| /* for kerberos authentication: | ||
| authType = "kerberos" | ||
| username = principal | ||
| password = path of the keytab file */ | ||
| private static final String authType = "simple"; | ||
|
|
||
| public static Builder builder() { | ||
| return new Builder(); | ||
| } | ||
|
|
||
| protected RangerContainer( | ||
| String image, | ||
| String hostName, | ||
| Set<Integer> ports, | ||
| Map<String, String> extraHosts, | ||
| Map<String, String> filesToMount, | ||
| Map<String, String> envVars, | ||
| Optional<Network> network) { | ||
| super(image, hostName, ports, extraHosts, filesToMount, envVars, network); | ||
| } | ||
|
|
||
| @Override | ||
| protected void setupContainer() { | ||
| super.setupContainer(); | ||
| withLogConsumer(new PrintingContainerLog(format("%-15s| ", "RangerContainer"))); | ||
| } | ||
|
|
||
| @Override | ||
| public void start() { | ||
| super.start(); | ||
|
|
||
| rangerUrl = String.format("http://localhost:%s", this.getMappedPort(6080)); | ||
| rangerClient = new RangerClient(rangerUrl, authType, username, password, null); | ||
|
|
||
| Preconditions.check("Ranger container startup failed!", checkContainerStatus(10)); | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean checkContainerStatus(int retryLimit) { | ||
| int nRetry = 0; | ||
| boolean isRangerContainerReady = false; | ||
| int sleepTimeMillis = 3_000; | ||
| while (nRetry++ < retryLimit) { | ||
| try { | ||
| rangerClient.getPluginsInfo(); | ||
| isRangerContainerReady = true; | ||
| LOG.info("Ranger container startup success!"); | ||
| break; | ||
| } catch (RangerServiceException e) { | ||
| LOG.warn("Check Ranger startup status... {}", e.getMessage()); | ||
| } | ||
| if (!isRangerContainerReady) { | ||
| try { | ||
| Thread.sleep(sleepTimeMillis); | ||
| LOG.warn("Waiting for Ranger server to be ready... ({}ms)", nRetry * sleepTimeMillis); | ||
| } catch (InterruptedException e) { | ||
| // ignore | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return isRangerContainerReady; | ||
| } | ||
|
|
||
| @Override | ||
| public void close() { | ||
| super.close(); | ||
| } | ||
|
|
||
| public static class Builder | ||
| extends BaseContainer.Builder<RangerContainer.Builder, RangerContainer> { | ||
|
|
||
| private Builder() { | ||
| this.image = DEFAULT_IMAGE; | ||
| this.hostName = HOST_NAME; | ||
| this.exposePorts = ImmutableSet.of(RANGER_PORT); | ||
| this.envVars = | ||
| ImmutableMap.<String, String>builder().put("RANGER_PASSWORD", password).build(); | ||
| } | ||
|
|
||
| @Override | ||
| public RangerContainer build() { | ||
| return new RangerContainer( | ||
| image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.