Merge remote-tracking branch 'upstream/master' into fix-markdown-syntax

.asf.yaml

@@ -1,3 +1,34 @@
+# note: documentation can be found here https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
+
 github:
   description: 'Apache Druid: a high performance real-time analytics database.'
   homepage: https://druid.apache.org/
+
+  # Attempt to make the auto-generated github emails more easily readable in email clients.
+  # Configuration adapted from https://github.com/apache/plc4x/blob/develop/.asf.yaml.
+  custom_subjects:
+    new_pr: "[PR] {title} ({repository})"
+    close_pr: "Re: [PR] {title} ({repository})"
+    comment_pr: "Re: [PR] {title} ({repository})"
+    diffcomment: "Re: [PR] {title} ({repository})"
+    merge_pr: "Re: [PR] {title} ({repository})"
+    new_issue: "[I] {title} ({repository})"
+    comment_issue: "Re: [I] {title} ({repository})"
+    close_issue: "Re: [I] {title} ({repository})"
+    catchall: "[GH] {title} ({repository})"
+    new_discussion: "[D] {title} ({repository})"
+    edit_discussion: "Re: [D] {title} ({repository})"
+    close_discussion: "Re: [D] {title} ({repository})"
+    close_discussion_with_comment: "Re: [D] {title} ({repository})"
+    reopen_discussion: "Re: [D] {title} ({repository})"
+    new_comment_discussion: "Re: [D] {title} ({repository})"
+    edit_comment_discussion: "Re: [D] {title} ({repository})"
+    delete_comment_discussion: "Re: [D] {title} ({repository})"
+
+notifications:
+  commits: [email protected]
+  issues: [email protected]
+  discussions: [email protected]
+  pullrequests_status: [email protected]
+  pullrequests_comment: [email protected]
+  pullrequests_bot_dependabot: [email protected]

.github/ISSUE_TEMPLATE/flaky_test.md

@@ -0,0 +1,10 @@
+---
+name: Flaky Test
+about: A template for Druid CI flaky test reports
+title: "Flaky Test: test name"
+labels: Flaky Test
+assignees: ''
+
+---
+
+Please replace 'test name' in the title with the actual test name. Also include a stack trace as well as link to a failing test run, along with any investigation you might have done to triage the issue.

.github/dependabot.yml

@@ -4,17 +4,14 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    open-pull-requests-limit: 20
     ignore:
-      - dependency-name: "com.google.guava:guava"
       # pin ZooKeeper dependencies to 3.5.x
       - dependency-name: "org.apache.zookeeper"
         versions: "[3.6,)"
       # pin Jetty dependencies to 9.4.x
       - dependency-name: "org.eclipse.jetty"
         versions: "[9.5,)"
-      - dependency-name: "org.apache.hadoop"
-      # We can't upgrade calcite without shading their guava dependencies. 
-      # We can't upgrade our own guava because we have to support Hadoop 2.
-      # Even then this will involve significant effort.
-      # See https://github.com/apache/druid/pull/12258
-      - dependency-name: "org.apache.calcite"
+      # pin Testng dependencies to 7.3.0
+      - dependency-name: "org.testng"
+        versions: "[7.4.0,)"

.github/labeler.yml

@@ -18,9 +18,79 @@
 #
 
 #
-# Pull Request Labeler Github Action Configuration: https://github.com/marketplace/actions/labeler
+# Pull Request Labeler GitHub Action Configuration: https://github.com/marketplace/actions/labeler
 
-"Area - Documentation":
-  - "docs/**/*"
-  - "website/**"
-  - "examples/quickstart/jupyter-notebooks/*"
+'Area - Batch Ingestion':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'indexing-hadoop/**'
+      - 'extensions-core/multi-stage-query/**'
+
+'Area - Dependencies':
+  - changed-files:
+    - any-glob-to-any-file:
+      - '**/pom.xml'
+      - 'licenses.yaml'
+
+'Area - Documentation':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'docs/**/*'
+      - 'website/**'
+      - 'examples/quickstart/jupyter-notebooks/**'
+
+'Area - Ingestion':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'indexing-service/**'
+
+'Area - Lookups':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/lookups-cached-global/**'
+      - 'extensions-core/lookups-cached-single/**'
+      - 'extensions-core/kafka-extraction-namespace/**'
+
+'Area - Metrics/Event Emitting':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'processing/src/main/java/org/apache/druid/java/util/metrics/**'
+      - 'processing/src/main/java/org/apache/druid/java/util/emitter/**'
+      - 'extensions-contrib/*-emitter/**'
+
+'Area - MSQ':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/multi-stage-query/**'
+
+'Area - Querying':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'sql/**'
+      - 'extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/**'
+
+'Area - Segment Format and Ser/De':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'processing/src/main/java/org/apache/druid/segment/**'
+
+'Area - Streaming Ingestion':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/kafka-indexing-service/**'
+      - 'extensions-core/kinesis-indexing-service/**'
+
+'Area - Web Console':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'web-console/**'
+
+'Kubernetes':
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'extensions-core/kubernetes-overlord-extensions/**'
+
+'GHA':
+  - changed-files:
+      - any-glob-to-any-file:
+          - '.github/**'

.github/scripts/analyze_dependencies_script.sh

@@ -15,7 +15,7 @@
 
 #!bin/bash
 
-${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} ||
+${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ||
 { echo "
     The dependency analysis has found a dependency that is either:
 

.github/scripts/license_checks_script.sh

@@ -20,7 +20,7 @@ set -e
 ./.github/scripts/setup_generate_license.sh
 ${MVN} apache-rat:check -Prat --fail-at-end \
 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
--Drat.consoleOutput=true ${HADOOP_PROFILE}
+-Drat.consoleOutput=true
 # Generate dependency reports and checks they are valid.
 mkdir -p target
 distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2

.github/scripts/setup_generate_license.sh

@@ -18,6 +18,9 @@
 set -e
 
 sudo apt-get update && sudo apt-get install python3 -y
-curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+# creating python virtual env
+python3 -m venv ~/.python3venv
+source ~/.python3venv/bin/activate
+sudo apt install python3-pip
 pip3 install wheel  # install wheel first explicitly
 pip3 install --upgrade pyyaml

.github/scripts/setup_test_profiling_env.sh

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+set -e
+
+JAR_INPUT_FILE="jfr-profiler-1.0.0.jar"
+JAR_OUTPUT_FILE="jfr-profiler.jar"
+ENV_VAR="JFR_PROFILER_ARG_LINE"
+
+if [ "$#" -ne 5 ]; then
+    echo "usage: $0 <jdk_version> <run_id> <run_number> <run_attempt> <module>"
+fi
+
+if [[ "$1" -ge "17" ]];
+then
+  curl https://static.imply.io/cp/$JAR_INPUT_FILE -s -o $JAR_OUTPUT_FILE
+
+  # Run 'java -version' and capture the output
+  output=$(java -version 2>&1)
+
+  # Extract the version number using grep and awk
+  jvm_version=$(echo "$output" | grep "version" | awk -F '"' '{print $2}')
+
+
+  echo $ENV_VAR=-javaagent:"$PWD"/$JAR_OUTPUT_FILE \
+  -Djfr.profiler.http.username=druid-ci \
+  -Djfr.profiler.http.password=w3Fb6PW8LIo849mViEkbgA== \
+  -Djfr.profiler.tags.project=druid \
+  -Djfr.profiler.tags.run_id=$2 \
+  -Djfr.profiler.tags.run_number=$3 \
+  -Djfr.profiler.tags.run_attempt=$4 \
+  -Djfr.profiler.tags.module=$5 \
+  -Djfr.profiler.tags.jvm_version=$jvm_version
+else
+  echo $ENV_VAR=\"\"
+fi
+
+

.github/scripts/unit_tests_script.sh

@@ -21,7 +21,8 @@ unset _JAVA_OPTIONS
 
 # Set MAVEN_OPTS for Surefire launcher.
 MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
-${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
+${MAVEN_SKIP} \
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}" -Pci
 sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
 free -m
 ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false" >> "$GITHUB_ENV" && false; }
@@ -73,5 +74,7 @@ then
   { printf "\n\n****FAILED****\nDiff code coverage check failed. To view coverage report, run 'mvn clean test jacoco:report' and open 'target/site/jacoco/index.html'\nFor more details on how to run code coverage locally, follow instructions here - https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n" && echo "coverage_failure=true" >> "$GITHUB_ENV" && false; }
 fi
 
-{ for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done }
-{ for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done }
+# Commented out as codecov visualizations are currently not being used in the Druid community
+# and cause PR failures due to rate limiting (see https://github.com/apache/druid/pull/16347)
+# { for i in 1 2 3; do curl -o codecov.sh -s https://codecov.io/bash && break || sleep 15; done }
+# { for i in 1 2 3; do bash codecov.sh -X gcov && break || sleep 15; done }

.github/workflows/codeql.yml

@@ -2,8 +2,16 @@ name: "CodeQL"
 
 on:
   push:
+    paths-ignore:
+      - '**/*.md'
+      - 'dev/**'
+      - 'docs/**'
     branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ]
   pull_request:
+    paths-ignore:
+      - '**/*.md'
+      - 'dev/**'
+      - 'docs/**'
     # The branches below must be a subset of the branches above
     branches: [ 'master' ]
   schedule:
@@ -27,22 +35,27 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'zulu'
+        java-version: '17'
+        cache: 'maven'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
-        config-file: ./.github/workflows/codeql-config.yml
+        config-file: ./.github/config/codeql-config.yml
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
 
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         queries: +security-and-quality
 
-
     - run: |
         echo "Building using custom commands"
         mvn clean package -f "pom.xml" -B -V -e -Dfindbugs.skip -Dcheckstyle.skip -Dpmd.skip=true -Denforcer.skip -Dmaven.javadoc.skip -DskipTests -Dmaven.test.skip.exec -Dlicense.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true
@@ -58,6 +71,6 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"