[improve][broker] add limitStatsLogging configuration when enable boo…

…kkeeperClientExposeStatsToPrometheus (#16734) Signed-off-by: tison <[email protected]> Co-authored-by: fanjianye <[email protected]> Co-authored-by: tison <[email protected]>
apache · Dec 10, 2022 · 71bf3b2 · 71bf3b2
1 parent a734b13
commit 71bf3b2
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 1 deletion.
diff --git a/conf/broker.conf b/conf/broker.conf
@@ -1065,6 +1065,10 @@ bookkeeperExplicitLacIntervalInMills=0
 # Expose bookkeeper client managed ledger stats to prometheus. default is false
 # bookkeeperClientExposeStatsToPrometheus=false
 
+# If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
+# to limit per_channel_bookie_client metrics. default is false
+# bookkeeperClientLimitStatsLogging=false
+
 ### --- Managed Ledger --- ###
 
 # Ensemble size (E), Number of bookies to use for storing entries in a ledger.

diff --git a/conf/standalone.conf b/conf/standalone.conf
@@ -696,6 +696,10 @@ bookkeeperUseV2WireProtocol=true
 # Expose bookkeeper client managed ledger stats to prometheus. default is false
 # bookkeeperClientExposeStatsToPrometheus=false
 
+# If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
+# to limit per_channel_bookie_client metrics. default is false
+# bookkeeperClientLimitStatsLogging=false
+
 ### --- Managed Ledger --- ###
 
 # Number of bookies to use when creating a ledger

diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
@@ -744,6 +744,10 @@ bookkeeperExplicitLacIntervalInMills=0
 # Expose bookkeeper client managed ledger stats to prometheus. default is false
 # bookkeeperClientExposeStatsToPrometheus=false
 
+# If bookkeeperClientExposeStatsToPrometheus is set to true, we can set bookkeeperClientLimitStatsLogging=true
+# to limit per_channel_bookie_client metrics. default is false
+# bookkeeperClientLimitStatsLogging=false
+
 ### --- Managed Ledger --- ###
 
 # Number of bookies to use when creating a ledger

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1765,6 +1765,12 @@ The delayed message index bucket time step(in seconds) in per bucket snapshot se
     )
     private boolean bookkeeperClientExposeStatsToPrometheus = false;
 
+    @FieldContext(
+            category = CATEGORY_STORAGE_BK,
+            doc = "whether limit per_channel_bookie_client metrics of bookkeeper client stats"
+    )
+    private boolean bookkeeperClientLimitStatsLogging = false;
+
     @FieldContext(
             category = CATEGORY_STORAGE_BK,
             doc = "Throttle value for bookkeeper client"

diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java
@@ -132,8 +132,8 @@ ClientConfiguration createBkClientConfiguration(MetadataStoreExtended store, Ser
         bkConf.setStickyReadsEnabled(conf.isBookkeeperEnableStickyReads());
         bkConf.setNettyMaxFrameSizeBytes(conf.getMaxMessageSize() + Commands.MESSAGE_SIZE_FRAME_PADDING);
         bkConf.setDiskWeightBasedPlacementEnabled(conf.isBookkeeperDiskWeightBasedPlacementEnabled());
-
         bkConf.setMetadataServiceUri(conf.getBookkeeperMetadataStoreUrl());
+        bkConf.setLimitStatsLogging(conf.isBookkeeperClientLimitStatsLogging());
 
         if (!conf.isBookkeeperMetadataStoreSeparated()) {
             // If we're connecting to the same metadata service, with same config, then

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java
@@ -299,4 +299,26 @@ public void testBookKeeperIoThreadsConfiguration() throws Exception {
         assertNull(FieldUtils.readField(builder, "eventLoopGroup", true));
     }
 
+    @Test
+    public void testBookKeeperLimitStatsLoggingConfiguration() throws Exception {
+        BookKeeperClientFactoryImpl factory = new BookKeeperClientFactoryImpl();
+        ServiceConfiguration conf = new ServiceConfiguration();
+        assertFalse(
+                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf).getLimitStatsLogging());
+        EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class);
+        BookKeeper.Builder builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class),
+                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf));
+        ClientConfiguration clientConfiguration =
+                (ClientConfiguration) FieldUtils.readField(builder, "conf", true);
+        assertFalse(clientConfiguration.getLimitStatsLogging());
+
+        conf.setBookkeeperClientLimitStatsLogging(true);
+        assertTrue(factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf)
+                .getLimitStatsLogging());
+        builder = factory.getBookKeeperBuilder(conf, eventLoopGroup, mock(StatsLogger.class),
+                factory.createBkClientConfiguration(mock(MetadataStoreExtended.class), conf));
+        clientConfiguration =
+                (ClientConfiguration) FieldUtils.readField(builder, "conf", true);
+        assertTrue(clientConfiguration.getLimitStatsLogging());
+    }
 }