Merge branch 'up/master' into website/branch-2.7.2-chapter-2

* up/master: (26 commits) [pulsar-admin] Allow setting --forward-source-message-property to false when updating a pulsar function (apache#12128) [website][upgrade]feat: docs migration - Development (apache#12320) Update delete inactive topic configuration documentation (apache#12350) [PIP 95][Issue 12040][broker] Multiple bind addresses for Pulsar protocol (apache#12056) Added Debezium Source for MS SQL Server (apache#12256) Fix: flaky oracle tests (apache#12306) [C++] Use URL encoded content type for OAuth 2.0 authentication (apache#12341) [C++] Handle OAuth 2.0 exceptional cases gracefully (apache#12335) feat(cli): add restart command to pulsar-daemon (apache#12279) [client-tools] Remove redundant initial value (apache#12296) Make AuthenticationTokenTest to run on windows (apache#12329) [offload] fix FileSystemManagedLedgerOffloader can not cleanup outdated ledger data (apache#12309) [Doc]--Update contents for Pulsar adaptor for Apache Spark (apache#12338) [PIP 95][Issue 12040][broker] Improved multi-listener in standalone mode (apache#12066) [website][upgrade]feat: docs migration - Cookbooks (apache#12319) [testclient] Make --payload-file take effect in PerformanceClient (apache#12187) [website][upgrade]feat: docs migration - adaptor (apache#12318) [pulsar-client] Add partition-change api for producer/consumer interceptors (apache#12287) [Transaction]Fix lowWaterMark of TopicTransactionBuffer (apache#12312) [pulsar-admin] New option takes precedence over deprecated option (apache#12260) ... # Conflicts: # site2/website-next/docusaurus.config.js # site2/website-next/versions.json
zeo1995 · Oct 14, 2021 · 8b9e28c · 8b9e28c
2 parents 7fdb854 + e316a53
commit 8b9e28c
Show file tree

Hide file tree

Showing 158 changed files with 12,926 additions and 400 deletions.
diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml
@@ -0,0 +1,118 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: CI - Integration - Pulsar-IO Oracle Source
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - branch-*
+
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+
+  pulsar-io:
+    name:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Detect changed files
+        id:   changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+
+      - name: Cache local Maven repository
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/pulsar
+          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+            ${{ runner.os }}-m2-dependencies-core-modules-
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        with:
+          distribution: 'adopt'
+          java-version: 11
+
+      - name: clean disk
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: |
+          sudo swapoff -a
+          sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          sudo apt clean
+          docker rmi $(docker images -q) -f
+          df -h
+
+      - name: run install by skip tests
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: mvn -q -B -ntp clean install -DskipTests
+
+      - name: build pulsar image
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
+
+      - name: build pulsar-all image
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
+
+      - name: build artifacts and docker image
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
+
+      - name: run integration tests
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: ./build/run_integration_group.sh PULSAR_IO_ORA
+
+      - name: Upload container logs
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: container-logs
+          path: tests/integration/target/container-logs
+
+      - name: Upload surefire-reports
+        uses: actions/upload-artifact@v2
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: surefire-reports
+          path: tests/integration/target/surefire-reports
diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon
@@ -20,7 +20,7 @@
 
 usage() {
     cat <<EOF
-Usage: pulsar-daemon (start|stop) <command> <args...>
+Usage: pulsar-daemon (start|stop|restart) <command> <args...>
 where command is one of:
     broker              Run a broker server
     bookie              Run a bookie server
@@ -205,6 +205,66 @@ case $startStop in
     fi
     ;;
 
+  (restart)
+   if [ -f $pid ]; then
+      TARGET_PID=$(cat $pid)
+      if kill -0 $TARGET_PID > /dev/null 2>&1; then
+        echo "stopping $command"
+        kill $TARGET_PID
+
+        count=0
+        location=$PULSAR_LOG_DIR
+        while ps -p $TARGET_PID > /dev/null;
+         do
+          echo "Shutdown is in progress... Please wait..."
+          sleep 1
+          count=`expr $count + 1`
+
+          if [ "$count" = "$PULSAR_STOP_TIMEOUT" ]; then
+                break
+          fi
+         done
+
+        if [ "$count" != "$PULSAR_STOP_TIMEOUT" ]; then
+            echo "Shutdown completed."
+        fi
+
+        if kill -0 $TARGET_PID > /dev/null 2>&1; then
+              fileName=$location/$command.out
+              $JAVA_HOME/bin/jstack $TARGET_PID > $fileName
+              echo "Thread dumps are taken for analysis at $fileName"
+              if [ "$1" == "-force" ]
+              then
+                 echo "forcefully stopping $command"
+                 kill -9 $TARGET_PID >/dev/null 2>&1
+                 echo Successfully stopped the process
+              else
+                 echo "WARNNING :  $command is not stopped completely."
+                 exit 1
+              fi
+        fi
+      else
+        echo "no $command to stop"
+      fi
+      rm $pid
+    else
+      echo no "$command to stop"
+    fi
+    sleep 3
+
+    rotate_out_log $out
+    echo restarting $command, logging to $logfile
+    echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations.
+    pulsar=$PULSAR_HOME/bin/pulsar
+    nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null &
+    echo $! > $pid
+    sleep 1; head $out
+    sleep 2;
+    if ! ps -p $! > /dev/null ; then
+      exit 1
+    fi
+    ;;
+
   (*)
     usage
     exit 1

diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf
@@ -252,6 +252,9 @@ autoRecoveryDaemonEnabled=true
 # How long to wait, in seconds, before starting auto recovery of a lost bookie
 lostBookieRecoveryDelay=0
 
+# Use older Bookkeeper wire protocol (Before Version 3) for AutoRecovery. Default is false
+useV2WireProtocol=true
+
 #############################################################################
 ## Placement settings
 #############################################################################

diff --git a/conf/broker.conf b/conf/broker.conf
@@ -40,6 +40,9 @@ webServicePortTls=
 # Hostname or IP address the service binds on, default is 0.0.0.0.
 bindAddress=0.0.0.0
 
+# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
+bindAddresses=
+
 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
 advertisedAddress=
 
@@ -155,7 +158,8 @@ allowAutoSubscriptionCreation=true
 # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
 defaultNumPartitions=1
 
-# Enable the deletion of inactive topics
+# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
+# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
 brokerDeleteInactiveTopicsEnabled=true
 
 # How often to check for inactive topics

diff --git a/conf/standalone.conf b/conf/standalone.conf
@@ -33,6 +33,9 @@ webServicePort=8080
 # Hostname or IP address the service binds on, default is 0.0.0.0.
 bindAddress=0.0.0.0
 
+# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
+bindAddresses=
+
 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
 advertisedAddress=
 
@@ -99,7 +102,8 @@ backlogQuotaDefaultLimitSecond=-1
 # Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0)
 ttlDurationDefaultInSeconds=0
 
-# Enable the deletion of inactive topics
+# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
+# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
 brokerDeleteInactiveTopicsEnabled=true
 
 # How often to check for inactive topics

diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
@@ -119,7 +119,8 @@ allowAutoSubscriptionCreation=true
 # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
 defaultNumPartitions=1
 
-# Enable the deletion of inactive topics
+# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
+# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
 brokerDeleteInactiveTopicsEnabled=true
 
 # How often to check for inactive topics

diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml
@@ -71,6 +71,7 @@
     <file><source>${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar</source></file>
     <file><source>${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar</source></file>

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -3083,9 +3083,13 @@ private void cleanupOffloaded(long ledgerId, UUID uuid, String offloadDriverName
             Map<String, String> offloadDriverMetadata, String cleanupReason) {
         log.info("[{}] Cleanup offload for ledgerId {} uuid {} because of the reason {}.",
                 name, ledgerId, uuid.toString(), cleanupReason);
+        Map<String, String> metadataMap = Maps.newHashMap();
+        metadataMap.putAll(offloadDriverMetadata);
+        metadataMap.put("ManagedLedgerName", name);
+
         Retries.run(Backoff.exponentialJittered(TimeUnit.SECONDS.toMillis(1), TimeUnit.SECONDS.toHours(1)).limit(10),
                 Retries.NonFatalPredicate,
-                () -> config.getLedgerOffloader().deleteOffloaded(ledgerId, uuid, offloadDriverMetadata),
+                () -> config.getLedgerOffloader().deleteOffloaded(ledgerId, uuid, metadataMap),
                 scheduledExecutor, name).whenComplete((ignored, exception) -> {
                     if (exception != null) {
                         log.warn("[{}] Error cleaning up offload for {}, (cleanup reason: {})",