Bump kafka client to 3.4.1 (#145)

* Update other dependencies and remove those that aren't used anymore
* Delete logstash-integration-kafka_jars.rb since it's generated during vendor task

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.3.0
+  - Bump kafka client to 3.4.1 [#145](https://github.com/logstash-plugins/logstash-integration-kafka/pull/145)
+
 ## 11.2.1
  - Fix nil exception to empty headers of record during event metadata assignment [#140](https://github.com/logstash-plugins/logstash-integration-kafka/pull/140)
 

build.gradle

@@ -28,9 +28,9 @@ group "org.logstash.integrations"
 sourceCompatibility = JavaVersion.VERSION_1_8
 
 // given https://docs.confluent.io/current/installation/versions-interoperability.html matrix
-// Confluent Platform 7.3.x is Apache Kafka 3.3.x
-String confluentKafkaVersion = '7.3.0'
-String apacheKafkaVersion = '3.3.1'
+// Confluent Platform 7.4.x is Apache Kafka 3.4.x
+String confluentKafkaVersion = '7.4.0'
+String apacheKafkaVersion = '3.4.1'
 
 buildscript {
     repositories {
@@ -50,27 +50,22 @@ dependencies {
     implementation("io.confluent:kafka-avro-serializer:${confluentKafkaVersion}") {
         exclude group: 'org.apache.kafka', module:'kafka-clients'
     }
+    // dependency of kafka-avro-serializer
     implementation("io.confluent:kafka-schema-serializer:${confluentKafkaVersion}") {
         exclude group: 'org.apache.kafka', module:'kafka-clients'
     }
-    implementation "io.confluent:common-config:${confluentKafkaVersion}"
-    implementation 'org.apache.avro:avro:1.11.0'
+    // dependency of kafka-avro-serializer
+    implementation 'org.apache.avro:avro:1.11.1'
+    // dependency of kafka-avro-serializer
     implementation("io.confluent:kafka-schema-registry-client:${confluentKafkaVersion}") {
         exclude group: 'org.apache.kafka', module:'kafka-clients'
     }
-    implementation("org.apache.kafka:kafka_2.12:${apacheKafkaVersion}") {
-        // contains kafka.utils.VerifiableProperties used by kafka-schema-registry-client, explicitly exclude
-        // transitive dependencies
-        transitive = false
-    }
-    implementation "io.confluent:common-utils:${confluentKafkaVersion}"
-    implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1'
-    implementation 'org.glassfish.jersey.core:jersey-common:2.33'
     implementation "org.apache.kafka:kafka-clients:${apacheKafkaVersion}"
-    implementation 'com.github.luben:zstd-jni:1.5.2-2'
+    // slf4j, zstd, lz4-java, snappy are dependencies from "kafka-clients"
     implementation 'org.slf4j:slf4j-api:1.7.36'
+    implementation 'com.github.luben:zstd-jni:1.5.5-4'
     implementation 'org.lz4:lz4-java:1.8.0'
-    implementation 'org.xerial.snappy:snappy-java:1.1.8.4'
+    implementation 'org.xerial.snappy:snappy-java:1.1.10.1'
 }
 task generateGemJarRequiresFile {
     doLast {

docs/index.asciidoc

@@ -1,7 +1,7 @@
 :plugin: kafka
 :type: integration
 :no_codec:
-:kafka_client: 3.3.1
+:kafka_client: 3.4.1
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

docs/input-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: input
 :default_codec: plain
-:kafka_client: 3.3
-:kafka_client_doc: 33
+:kafka_client: 3.4
+:kafka_client_doc: 34
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

docs/output-kafka.asciidoc

@@ -2,8 +2,8 @@
 :plugin: kafka
 :type: output
 :default_codec: plain
-:kafka_client: 3.3
-:kafka_client_doc: 33
+:kafka_client: 3.4
+:kafka_client_doc: 34
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!

kafka_test_setup.sh

@@ -6,7 +6,7 @@ set -ex
 if [ -n "${KAFKA_VERSION+1}" ]; then
   echo "KAFKA_VERSION is $KAFKA_VERSION"
 else
-   KAFKA_VERSION=3.3.1
+   KAFKA_VERSION=3.4.1
 fi
 
 export _JAVA_OPTIONS="-Djava.net.preferIPv4Stack=true"
@@ -15,11 +15,11 @@ rm -rf build
 mkdir build
 
 echo "Setup Kafka version $KAFKA_VERSION"
-if [ ! -e "kafka_2.12-$KAFKA_VERSION.tgz" ]; then
+if [ ! -e "kafka_2.13-$KAFKA_VERSION.tgz" ]; then
   echo "Kafka not present locally, downloading"
-  curl -s -o "kafka_2.12-$KAFKA_VERSION.tgz" "https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.12-$KAFKA_VERSION.tgz"
+  curl -s -o "kafka_2.13-$KAFKA_VERSION.tgz" "https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
 fi
-cp kafka_2.12-$KAFKA_VERSION.tgz build/kafka.tgz
+cp kafka_2.13-$KAFKA_VERSION.tgz build/kafka.tgz
 mkdir build/kafka && tar xzf build/kafka.tgz -C build/kafka --strip-components 1
 
 echo "Starting ZooKeeper"
@@ -34,15 +34,15 @@ echo "Setup Confluent Platform"
 if [ -n "${CONFLUENT_VERSION+1}" ]; then
   echo "CONFLUENT_VERSION is $CONFLUENT_VERSION"
 else
-   CONFLUENT_VERSION=5.5.1
+   CONFLUENT_VERSION=7.4.0
 fi
-if [ ! -e confluent-community-$CONFLUENT_VERSION-2.12.tar.gz ]; then
+if [ ! -e confluent-community-$CONFLUENT_VERSION.tar.gz ]; then
   echo "Confluent Platform not present locally, downloading"
   CONFLUENT_MINOR=$(echo "$CONFLUENT_VERSION" | sed -n 's/^\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p')
   echo "CONFLUENT_MINOR is $CONFLUENT_MINOR"
-  curl -s -o confluent-community-$CONFLUENT_VERSION-2.12.tar.gz http://packages.confluent.io/archive/$CONFLUENT_MINOR/confluent-community-$CONFLUENT_VERSION-2.12.tar.gz
+  curl -s -o confluent-community-$CONFLUENT_VERSION.tar.gz http://packages.confluent.io/archive/$CONFLUENT_MINOR/confluent-community-$CONFLUENT_VERSION.tar.gz
 fi
-cp confluent-community-$CONFLUENT_VERSION-2.12.tar.gz build/confluent_platform.tar.gz
+cp confluent-community-$CONFLUENT_VERSION.tar.gz build/confluent_platform.tar.gz
 mkdir build/confluent_platform && tar xzf build/confluent_platform.tar.gz -C build/confluent_platform --strip-components 1
 
 echo "Configuring TLS on Schema registry"

logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '11.2.1'
+  s.version         = '11.3.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+