Some improvements about Docker

distribution/docker/Dockerfile

@@ -17,8 +17,14 @@
 # under the License.
 #
 
-ARG JDK_VERSION=8
-FROM maven:3.8.4-jdk-11-slim as builder
+ARG JDK_VERSION=11
+
+# The platform is explicitly specified as x64 to build the Druid distribution.
+# This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012
+# Since only java jars are shipped in the final image, it's OK to build the distribution on x64.
+# Once the web-console dependency problem is resolved, we can remove the --platform directive.
+FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder
+
 # Rebuild from source in this stage
 # This can be unset if the tarball was already built outside of Docker
 ARG BUILD_FROM_SOURCE="true"
@@ -29,28 +35,47 @@ RUN export DEBIAN_FRONTEND=noninteractive \
 
 COPY . /src
 WORKDIR /src
-RUN if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
+RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
       mvn -B -ff -q dependency:go-offline \
       install \
       -Pdist,bundle-contrib-exts \
       -Pskip-static-checks,skip-tests \
       -Dmaven.javadoc.skip=true \
       ; fi
 
-RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
+RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
       -Dexpression=project.version -DforceStdout=true \
     ) \
  && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
  && mv /opt/apache-druid-${VERSION} /opt/druid
 
-FROM amd64/busybox:1.30.0-glibc as busybox
+FROM busybox:1.35.0-glibc as busybox
 
-FROM gcr.io/distroless/java:$JDK_VERSION
+FROM gcr.io/distroless/java$JDK_VERSION-debian11
 LABEL maintainer="Apache Druid Developers <[email protected]>"
 
 COPY --from=busybox /bin/busybox /busybox/busybox
 RUN ["/busybox/busybox", "--install", "/bin"]
 
+# Predefined builtin arg, see: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
+ARG TARGETARCH
+
+#
+# Download bash-static binary to execute scripts that require bash.
+# Although bash-static supports multiple platforms, but there's no need for us to support all those platform, amd64 and arm64 are enough.
+#
+ARG BASH_URL_BASE="https://github.com/robxu9/bash-static/releases/download/5.1.016-1.2.3"
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+      BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \
+    elif [ "$TARGETARCH" = "amd64" ]; then \
+      BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \
+    else \
+      echo "Unsupported architecture ($TARGETARCH)" && exit 1; \
+    fi; \
+    echo "Downloading bash-static from ${BASH_URL}" \
+    && wget ${BASH_URL} -O /bin/bash \
+    && chmod 755 /bin/bash
+
 RUN addgroup -S -g 1000 druid \
  && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid
 

distribution/docker/README.md

@@ -19,22 +19,41 @@
 
 ## Build
 
-From the root of the repo, run `docker build -t apache/druid:tag -f distribution/docker/Dockerfile .`
+From the root of the repo, run following command:
 
-## Run
+```bash
+DOCKER_BUILDKIT=1 docker build -t apache/druid:tag -f distribution/docker/Dockerfile .
+```
+
+### Building images on Apple M1/M2
+To build images on Apple M1/M2, you need to follow the instructions in this section.
 
-Edit `environment` to suite. Run `docker-compose -f distribution/docker/docker-compose.yml up`
+1. build Druid distribution from the root of the repo
+   ```bash
+   mvn clean package -DskipTests -Pdist
+   ```
+2. build target image
+   ```
+   DOCKER_BUILDKIT=1 docker build -t apache/druid:tag -f distribution/docker/Dockerfile --build-arg BUILD_FROM_SOURCE=false .
+   ```
 
-## Java 11
+## Run
 
-From the root of the repo, run `docker build -t apache/druid:tag -f distribution/docker/Dockerfile.java11 .` which will build Druid to run in a Java 11 environment.
+1. Edit `distribution/docker/docker-compose.yml` file to change the tag of Druid's images to the tag that's used in the 'Build' phase above.
+2. Edit `environment` file to suite if necessary.
+3. Run:
+    ```bash
+    docker-compose -f distribution/docker/docker-compose.yml up
+    ```
 
 ## MySQL Database Connector
 
 This image contains solely the postgres metadata storage connector. If you
 need the mysql metadata storage connector, you can use Dockerfile.mysql to add
 it to the base image above.
 
-`docker build -t apache/druid:tag-mysql --build-arg DRUID_RELEASE=apache/druid:tag -f distribution/docker/Dockerfile.mysql .`
+```bash
+docker build -t apache/druid:tag-mysql --build-arg DRUID_RELEASE=apache/druid:tag -f distribution/docker/Dockerfile.mysql .
+```
 
-where `druid:tag` is the version to use as the base.
+where `druid:tag` is the version of Druid image to use as the base.

distribution/docker/druid.sh

@@ -196,4 +196,4 @@ then
     mkdir -p ${DRUID_DIRS_TO_CREATE}
 fi
 
-exec java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main server $@
+exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main server $@

integration-tests/script/setup_druid_on_k8s.sh

@@ -32,8 +32,8 @@ mvn -B -ff -q dependency:go-offline \
       -Pskip-static-checks,skip-tests \
       -Dmaven.javadoc.skip=true
 
-docker build --build-arg BUILD_FROM_SOURCE=0 -t druid/base:v1 -f distribution/docker/Dockerfile .
-docker build --build-arg BASE_IMAGE=druid/base:v1 -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
+DOCKER_BUILDKIT=1 docker build --build-arg BUILD_FROM_SOURCE=0 -t druid/base:v1 -f distribution/docker/Dockerfile .
+DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=druid/base:v1 -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
 
 # This tmp dir is used for MiddleManager pod and Historical Pod to cache segments.
 sudo rm -rf tmp