Merge branch 'master' of github.com:apache/pinot into logDataType

.github/workflows/build-multi-arch-pinot-docker-image.yml

@@ -68,7 +68,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
         with:
           platforms: linux/${{ matrix.arch }}
@@ -95,7 +95,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
       - uses: docker/setup-buildx-action@v2
         name: Set up Docker Buildx

.github/workflows/build-pinot-base-docker-image.yml

@@ -36,7 +36,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
       - uses: docker/setup-buildx-action@v2
         name: Set up Docker Buildx

.github/workflows/build-pinot-docker-image.yml

@@ -51,7 +51,7 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
     - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx

.github/workflows/build-presto-docker-image.yml

@@ -51,7 +51,7 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
     - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx

.github/workflows/build-superset-docker-image.yml

@@ -47,7 +47,7 @@ jobs:
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v2
+    - uses: docker/setup-qemu-action@v3
       name: Set up QEMU
     - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx

.github/workflows/pinot_compatibility_tests.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
           distribution: 'adopt'
@@ -74,7 +74,7 @@ jobs:
         if: always()
         run: |
           zip -1 -r artifacts.zip /tmp/compatibility-verifier/*
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Store compatibility verifier work directory
         if: always()
         with:

.github/workflows/pinot_tests.yml

@@ -51,7 +51,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
           distribution: 'adopt'
@@ -93,13 +93,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: ${{ matrix.distribution }}
           cache: 'maven'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
         with:
@@ -139,7 +139,7 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         timeout-minutes: 5
         with:
@@ -172,13 +172,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: ${{ matrix.distribution }}
           cache: 'maven'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
         with:
@@ -217,7 +217,7 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         timeout-minutes: 5
         with:
@@ -243,7 +243,7 @@ jobs:
         run: .github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
       - name: Upload coverage to Codecov
         if : ${{ matrix.testset == 1 }}
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         continue-on-error: true
         timeout-minutes: 5
         with:
@@ -267,7 +267,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 11
           distribution: 'adopt'
@@ -283,7 +283,7 @@ jobs:
           npm install -g [email protected]
           npm --version
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
         with:
@@ -325,13 +325,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: ${{ matrix.distribution }}
           cache: 'maven'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         env:
           SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}

.github/workflows/pinot_vuln_check.yml

@@ -37,7 +37,7 @@ jobs:
     name: Verify Docker Image
     runs-on: ubuntu-latest
     steps:
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
         name: Set up QEMU
       - uses: actions/checkout@v3
       - name: Build the Docker image

.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh

@@ -18,6 +18,8 @@
 # under the License.
 #
 
+set -e
+
 if [ -z "${BUILD_PLATFORM}" ]; then
   exit 1
 fi
@@ -33,5 +35,6 @@ docker buildx build \
   --platform=${BUILD_PLATFORM} \
   --file ${OPEN_JDK_DIST}.dockerfile \
   --tag apachepinot/pinot-base-${BASE_IMAGE_TYPE}:${TAG} \
+  --build-arg JAVA_VERSION=${JDK_VERSION:-11} \
   --push \
   .

.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh

@@ -18,6 +18,8 @@
 # under the License.
 #
 
+set -x
+
 if [ -z "${DOCKER_IMAGE_NAME}" ]; then
   DOCKER_IMAGE_NAME="apachepinot/pinot"
 fi

.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh

@@ -18,6 +18,9 @@
 # under the License.
 #
 
+set -e
+set -x
+
 if [ -z "${DOCKER_IMAGE_NAME}" ]; then
   DOCKER_IMAGE_NAME="apachepinot/pinot"
 fi
@@ -28,39 +31,59 @@ if [ -z "${JDK_VERSION}" ]; then
   JDK_VERSION="11"
 fi
 
-tags=()
-declare -a tags=($(echo ${TAGS} | tr "," " "))
 
 cd ${DOCKER_FILE_BASE_DIR}
 platformTag=${BUILD_PLATFORM/\//-}
-DOCKER_BUILD_TAGS=""
-for tag in "${tags[@]}"; do
-  DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag}-${BASE_IMAGE_TAG}-${platformTag} "
-
-  if [ "${BASE_IMAGE_TAG}" == "11-amazoncorretto" ]; then
-    if [ "${tag}" == "latest" ]; then
-      DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:latest-${platformTag} "
-    fi
-  fi
-done
 
-echo "Building docker image for platform: ${BUILD_PLATFORM} with tags: ${DOCKER_BUILD_TAGS}"
+PINOT_BUILD_IMAGE_TAG=${BASE_IMAGE_TAG}-${platformTag}
+echo "Building docker image for platform: ${BUILD_PLATFORM} with tag: pinot-build:${PINOT_BUILD_IMAGE_TAG}"
 docker build \
   --no-cache \
   --platform ${BUILD_PLATFORM} \
-  --file Dockerfile \
+  --file Dockerfile.build \
   --build-arg PINOT_GIT_URL=${PINOT_GIT_URL} \
   --build-arg PINOT_BRANCH=${PINOT_BRANCH} \
   --build-arg JDK_VERSION=${JDK_VERSION} \
-  ${DOCKER_BUILD_TAGS} \
+  --build-arg PINOT_BASE_IMAGE_TAG=${BASE_IMAGE_TAG} \
+  --tag pinot-build:${PINOT_BUILD_IMAGE_TAG} \
   .
 
-for tag in "${tags[@]}"; do
-  docker push ${DOCKER_IMAGE_NAME}:${tag}-${BASE_IMAGE_TAG}-${platformTag}
+tags=()
+declare -a tags=($(echo ${TAGS} | tr "," " "))
+
+runtimeImages=()
+declare -a runtimeImages=($(echo ${RUNTIME_IMAGE_TAGS} | tr "," " "))
+
+
+for runtimeImage in "${runtimeImages[@]}"; do
+  DOCKER_BUILD_TAGS=""
+  for tag in "${tags[@]}"; do
+    DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag}-${runtimeImage}-${platformTag} "
+
+    if [ "${runtimeImage}" == "11-amazoncorretto" ]; then
+      if [ "${tag}" == "latest" ]; then
+        DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:latest-${platformTag} "
+      fi
+    fi
+  done
+
+  echo "Building docker image for platform: ${BUILD_PLATFORM} with tags: ${DOCKER_BUILD_TAGS}"
+  docker build \
+    --no-cache \
+    --platform ${BUILD_PLATFORM} \
+    --file Dockerfile.package \
+    --build-arg PINOT_BUILD_IMAGE_TAG=${PINOT_BUILD_IMAGE_TAG} \
+    --build-arg PINOT_RUNTIME_IMAGE_TAG=${runtimeImage} \
+    ${DOCKER_BUILD_TAGS} \
+    .
+
+  for tag in "${tags[@]}"; do
+    docker push ${DOCKER_IMAGE_NAME}:${tag}-${runtimeImage}-${platformTag}
 
-  if [ "${BASE_IMAGE_TAG}" == "11-amazoncorretto" ]; then
-    if [ "${tag}" == "latest" ]; then
-      docker push ${DOCKER_IMAGE_NAME}:${tag}-${platformTag}
+    if [ "${runtimeImage}" == "11-amazoncorretto" ]; then
+      if [ "${tag}" == "latest" ]; then
+        docker push ${DOCKER_IMAGE_NAME}:${tag}-${platformTag}
+      fi
     fi
-  fi
+  done
 done

.github/workflows/scripts/docker/.presto_build_info_gen.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+if [ -z "${PRESTO_GIT_URL}" ]; then
+  PRESTO_GIT_URL="https://github.com/prestodb/presto.git"
+fi
+if [ -z "${PRESTO_BRANCH}" ]; then
+  PRESTO_BRANCH="master"
+fi
+
+# Get presto commit id
+ROOT_DIR=`pwd`
+rm -rf /tmp/presto
+git clone -b ${PRESTO_BRANCH} --single-branch ${PRESTO_GIT_URL} /tmp/presto
+cd /tmp/presto
+COMMIT_ID=`git rev-parse --short HEAD`
+./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout
+VERSION=`./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout`
+rm -rf /tmp/presto
+DATE=`date +%Y%m%d`
+
+if [ -z "${TAGS}" ]; then
+  TAGS="${VERSION}-${COMMIT_ID}-${DATE},latest"
+fi
+echo "commit-id=${COMMIT_ID}" >> "$GITHUB_OUTPUT"
+echo "tags=${TAGS}" >> "$GITHUB_OUTPUT"

.github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh

@@ -31,40 +31,31 @@ if [ -z "${BUILD_PLATFORM}" ]; then
   BUILD_PLATFORM="linux/amd64"
 fi
 
-# Get presto commit id
-ROOT_DIR=`pwd`
-rm -rf /tmp/presto
-git clone -b ${PRESTO_BRANCH} --single-branch ${PRESTO_GIT_URL} /tmp/presto
-cd /tmp/presto
-COMMIT_ID=`git rev-parse --short HEAD`
-VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
-rm -rf /tmp/presto
-DATE=`date +%Y%m%d`
-cd ${ROOT_DIR}
-
 tags=()
-if [ -z "${TAGS}" ]; then
-  tags=("${VERSION}-${COMMIT_ID}-${DATE}")
-  tags+=("latest")
-else
-  declare -a tags=($(echo ${TAGS} | tr "," " "))
-fi
+declare -a tags=($(echo ${TAGS} | tr "," " "))
 
 DOCKER_BUILD_TAGS=""
 for tag in "${tags[@]}"
 do
-  echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+  echo "Plan to build docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
   DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
 done
 
 
 cd ${DOCKER_FILE_BASE_DIR}
 
-docker buildx build \
+docker image prune --all --filter "until=1h" -f
+
+docker build \
     --no-cache \
     --platform=${BUILD_PLATFORM} \
     --file Dockerfile \
     --build-arg PRESTO_GIT_URL=${PRESTO_GIT_URL} --build-arg PRESTO_BRANCH=${PRESTO_BRANCH} \
     ${DOCKER_BUILD_TAGS} \
-    --push \
     .
+
+for tag in "${tags[@]}"
+do
+  echo "Push docker image: ${DOCKER_IMAGE_NAME}:${tag}"
+  docker push ${DOCKER_IMAGE_NAME}:${tag}
+done