Merge remote-tracking branch 'upstream/master'

.github/workflows/standard-its.yml

@@ -183,7 +183,8 @@ jobs:
         run: |
           for v in broker middlemanager router coordinator historical ; do
           echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------";
-          sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0 ||:;
+          /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0 ||:;
+          /usr/local/bin/kubectl get events | grep druid-tiny-cluster-"$v"s-0 ||:;
           done
 
   integration-other-tests:

distribution/docker/Dockerfile

@@ -40,7 +40,7 @@ RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; th
       install \
       -Pdist,bundle-contrib-exts \
       -Pskip-static-checks,skip-tests \
-      -Dmaven.javadoc.skip=true \
+      -Dmaven.javadoc.skip=true -T1C \
       ; fi
 
 RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \
@@ -82,15 +82,19 @@ RUN addgroup -S -g 1000 druid \
 COPY --from=bash-static /bin/bash /bin/bash
 RUN chmod 755 /bin/bash
 
-COPY --chown=druid:druid --from=builder /opt /opt
 COPY distribution/docker/druid.sh /druid.sh
 COPY distribution/docker/peon.sh /peon.sh
+COPY distribution/docker/deduplicate_jars.sh /deduplicate_jars.sh
 
 # create necessary directories which could be mounted as volume
+# copy and de-duplicate jars from builder in same layer to reduce image size
 #   /opt/druid/var is used to keep individual files(e.g. log) of each Druid service
 #   /opt/shared is used to keep segments and task logs shared among Druid services
-RUN mkdir /opt/druid/var /opt/shared \
- && chown druid:druid /opt/druid/var /opt/shared \
+RUN --mount=type=bind,from=builder,source=/opt,target=/builder/opt \
+ mkdir -p /opt/druid/var /opt/shared \
+ && cp -r /builder/opt/druid /opt/ \
+ && /deduplicate_jars.sh /opt/druid \
+ && chown -R druid:druid /opt/druid \
  && chmod 775 /opt/druid/var /opt/shared
 
 USER druid

distribution/docker/deduplicate_jars.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+# Check if an argument is provided
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <path-to-druid-home>"
+    exit 1
+fi
+
+# Directory containing JAR files
+JAR_DIR="$1"
+
+# Declare an associative array to hold the canonical filenames. Works on base version >= 4
+declare -A canonical
+
+# Find all JAR files, sort them to ensure duplicates are processed together
+while IFS= read -r jar; do
+    # Extract the base name and sanitize it to create a valid array key
+    key=$(basename "$jar")
+
+    # Check if this is the first occurrence of this file
+    if [ -z "${canonical[$key]}" ]; then
+        # Mark this file as the canonical one for this basename
+        canonical[$key]="$jar"
+    else
+        # This file is a duplicate, replace it with a symlink to the canonical file
+        ln -sf "${canonical[$key]}" "$jar"
+        echo "Replaced duplicate $jar with symlink to ${canonical[$key]}"
+    fi
+# Read in an order that retain core libs as original jars
+done < <(find $JAR_DIR -wholename '*/lib/*.jar' | sort ; find $JAR_DIR -wholename '*/extensions/*.jar' | sort ; find $JAR_DIR -wholename '*/hadoop-dependencies/*.jar' | sort)

distribution/pom.xml

@@ -456,6 +456,8 @@
                                         <argument>org.apache.druid.extensions.contrib:druid-deltalake-extensions</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:druid-spectator-histogram</argument>
+                                        <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions.contrib:druid-rabbit-indexing-service</argument>
                                     </arguments>
                                 </configuration>
                             </execution>

docs/configuration/index.md

@@ -378,7 +378,7 @@ You can configure Druid services to emit [metrics](../operations/metrics.md) reg
 |Property|Description|Default|
 |--------|-----------|-------|
 |`druid.monitoring.emissionPeriod`| Frequency that Druid emits metrics.|`PT1M`|
-|[`druid.monitoring.monitors`](#metrics-monitors)|Sets list of Druid monitors used by a service.|none (no monitors)|
+|[`druid.monitoring.monitors`](#metrics-monitors)|Sets list of Druid monitors used by a service.<br /><br />Because individual monitors sometimes only work on specific process types, it is best to set this property for each process type individually in e.g. `historical/runtime.properties` rather than `_common/common.runtime.properties`.|none (no monitors)|
 |[`druid.emitter`](#metrics-emitters)|Setting this value initializes one of the emitter modules.|`noop` (metric emission disabled by default)|
 
 #### Metrics monitors

docs/ingestion/native-batch-simple-task.md

@@ -90,8 +90,8 @@ A sample task is shown below:
     "tuningConfig" : {
       "type" : "index",
       "partitionsSpec": {
-        "type": "single_dim",
-        "partitionDimension": "country",
+        "type": "hashed",
+        "partitionDimensions": ["country"],
         "targetRowsPerSegment": 5000000
       }
     }