diff --git a/.github/workflows/ci-integration-test-legacy.yml b/.github/workflows/ci-integration-test-legacy.yml index 3548b21fa..37d756a0b 100644 --- a/.github/workflows/ci-integration-test-legacy.yml +++ b/.github/workflows/ci-integration-test-legacy.yml @@ -26,16 +26,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven - run: mvn -B clean integration-test -Pacls -Dcp.version=${{matrix.cpversion}} --file pom.xml \ No newline at end of file + run: mvn -B clean integration-test -Pacls -Dcp.version=${{matrix.cpversion}} --file pom.xml diff --git a/.github/workflows/ci-integration-test-main.yml b/.github/workflows/ci-integration-test-main.yml index 060c4e8bc..9ae28ad0b 100644 --- a/.github/workflows/ci-integration-test-main.yml +++ b/.github/workflows/ci-integration-test-main.yml @@ -15,21 +15,17 @@ jobs: matrix: os: [ubuntu-latest] java: [11.0.x] - cpversion: [7.5.0] + cpversion: [7.6.0] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B clean integration-test -Pacls -Dcp.version=${{matrix.cpversion}} --file pom.xml diff --git a/.github/workflows/ci-unit-test-legacy.yml b/.github/workflows/ci-unit-test-legacy.yml index ad27f4953..a36cf0e10 100644 --- a/.github/workflows/ci-unit-test-legacy.yml +++ b/.github/workflows/ci-unit-test-legacy.yml @@ -24,16 +24,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven - run: mvn -B package --file pom.xml \ No newline at end of file + run: mvn -B package --file pom.xml diff --git a/.github/workflows/ci-unit-test-main.yml b/.github/workflows/ci-unit-test-main.yml index d83e25977..58324c85f 100644 --- a/.github/workflows/ci-unit-test-main.yml +++ b/.github/workflows/ci-unit-test-main.yml @@ -19,16 +19,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven - run: mvn -B package --file pom.xml \ No newline at end of file + run: mvn -B package --file pom.xml diff --git a/.github/workflows/nightly-artifacts-build.yml b/.github/workflows/nightly-artifacts-build.yml index 28362e2df..df54cdefe 100644 --- a/.github/workflows/nightly-artifacts-build.yml +++ b/.github/workflows/nightly-artifacts-build.yml @@ -11,18 +11,14 @@ jobs: name: Build docker image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: docker/setup-buildx-action@v1 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: 11 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B package --file pom.xml - name: copy fat jar @@ -49,24 +45,21 @@ jobs: name: Build rpm/deb packages (using maven) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: 11 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B package --file pom.xml - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '11' - distribution: 'adopt' + distribution: "temurin" + cache: maven server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD diff --git a/.github/workflows/release-artifacts-build-legacy.yml b/.github/workflows/release-artifacts-build-legacy.yml index 8013002be..314497a4c 100644 --- a/.github/workflows/release-artifacts-build-legacy.yml +++ b/.github/workflows/release-artifacts-build-legacy.yml @@ -9,17 +9,13 @@ jobs: name: Build rpm/deb packages (using maven) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: 8 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B package --file pom.xml - name: Import private GPG key diff --git a/.github/workflows/release-artifacts-build.yml b/.github/workflows/release-artifacts-build.yml index 1b00e5824..3f9bf08e0 100644 --- a/.github/workflows/release-artifacts-build.yml +++ b/.github/workflows/release-artifacts-build.yml @@ -10,7 +10,7 @@ jobs: name: Build rpm/deb packages (using maven) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Import private GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@v4 @@ -22,22 +22,19 @@ jobs: - name: Import public GPG Key run: rpm --import release/keys/public.key - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: 11 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B package --file pom.xml - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '11' - distribution: 'adopt' + distribution: "temurin" + cache: maven server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD diff --git a/.github/workflows/release-docker-legacy.yml b/.github/workflows/release-docker-legacy.yml index 7e80fbbdc..30258777e 100644 --- a/.github/workflows/release-docker-legacy.yml +++ b/.github/workflows/release-docker-legacy.yml @@ -9,7 +9,7 @@ jobs: name: Build docker image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: docker/setup-buildx-action@v1 - name: Docker meta id: docker_meta @@ -21,15 +21,11 @@ jobs: {{version}} {{major}}.{{minor}} - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: 8 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B package --file pom.xml - name: copy fat jar diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 2da56bd1f..1ff52cbf0 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -10,7 +10,7 @@ jobs: name: Build docker image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: docker/setup-buildx-action@v1 - name: Docker meta id: docker_meta @@ -22,15 +22,11 @@ jobs: {{version}} {{major}}.{{minor}} - name: Set up the JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: java-version: 11 - - name: Cache dependencies - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: "temurin" + cache: maven - name: Build with Maven run: mvn -B package --file pom.xml - name: copy fat jar diff --git a/pom.xml b/pom.xml index 3a29b2bcc..36ebf7286 100644 --- a/pom.xml +++ b/pom.xml @@ -153,7 +153,7 @@ com.spotify.fmt fmt-maven-plugin - 2.20 + 2.23 @@ -408,7 +408,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.5.0 + 3.6.3 attach-javadocs @@ -428,7 +428,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.5.0 + 3.5.2 @@ -534,38 +534,38 @@ UTF-8 - 0.8.10 - 3.11.0 - 3.1.2 - 3.21.0 - 3.4.5 + 0.8.11 + 3.12.1 + 3.2.5 + 3.21.2 + 3.5.0 3.12.1 - 3.1.2 + 3.2.5 2.3.0 - 4.7.3.5 + 4.8.3.1 - 3.24.2 - 1.11.2 - 2.20.144 - 1.5.0 - 7.5.0-ce - 7.5.0 - 26.22.0 + 3.25.3 + 1.11.3 + 2.25.6 + 1.6.0 + 7.6.0-ce + 7.6.0 + 26.34.0 2.2 - 2.15.2 - 4.4.4 - 3.1.3 - 2.7.1 + 2.16.2 + 4.4.7 + 3.1.5 + 2.7.2 4.13.2 7.0.0 0.27.1 - 2.20.0 - 1.18.28 - 5.5.0 - 2.0.9 - 1.19.0 - 1.4.2 - 3.9.0 + 2.23.0 + 1.18.30 + 5.11.0 + 2.0.12 + 1.19.7 + 1.4.3 + 3.9.1 @@ -720,7 +720,7 @@ com.google.cloud google-cloud-storage - 2.26.1 + 2.35.0 org.hamcrest @@ -773,7 +773,7 @@ com.github.tomakehurst wiremock-jre8 - 2.35.1 + 2.35.2 test diff --git a/src/main/java/com/purbon/kafka/topology/AccessControlManager.java b/src/main/java/com/purbon/kafka/topology/AccessControlManager.java index c8f2d7bbd..508a84272 100644 --- a/src/main/java/com/purbon/kafka/topology/AccessControlManager.java +++ b/src/main/java/com/purbon/kafka/topology/AccessControlManager.java @@ -81,9 +81,7 @@ private Set loadActualClusterStateIfAvailable(ExecutionPlan .collect(Collectors.toSet()); if (!config.shouldVerifyRemoteState()) { - LOGGER.warn( - "Remote state verification disabled, this is not a good practice, be aware" - + "in future versions, this check is going to become mandatory."); + OnceOnlyWarningLogger.getInstance().logRemoteStateVerificationDisabledWarning(); } if (config.shouldVerifyRemoteState() && !config.fetchStateFromTheCluster()) { diff --git a/src/main/java/com/purbon/kafka/topology/ArtefactManager.java b/src/main/java/com/purbon/kafka/topology/ArtefactManager.java index eb395b0f5..9168b4c13 100644 --- a/src/main/java/com/purbon/kafka/topology/ArtefactManager.java +++ b/src/main/java/com/purbon/kafka/topology/ArtefactManager.java @@ -126,9 +126,7 @@ protected Collection loadActualClusterStateIfAvailable(Execu var currentState = config.fetchStateFromTheCluster() ? getClustersState() : getLocalState(plan); if (!config.shouldVerifyRemoteState()) { - LOGGER.warn( - "Remote state verification disabled, this is not a good practice, be aware" - + "in future versions, this check is going to become mandatory."); + OnceOnlyWarningLogger.getInstance().logRemoteStateVerificationDisabledWarning(); } if (config.shouldVerifyRemoteState() && !config.fetchStateFromTheCluster()) { diff --git a/src/main/java/com/purbon/kafka/topology/CommandLineInterface.java b/src/main/java/com/purbon/kafka/topology/CommandLineInterface.java index 2933dff5f..30c088eb0 100644 --- a/src/main/java/com/purbon/kafka/topology/CommandLineInterface.java +++ b/src/main/java/com/purbon/kafka/topology/CommandLineInterface.java @@ -40,6 +40,15 @@ public class CommandLineInterface { public static final String QUIET_OPTION = "quiet"; public static final String QUIET_DESC = "Print minimum status update"; + public static final String DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION = "accept-read-only-streams"; + public static final String DONT_WARN_FOR_READ_ONLY_STREAMS_DESC = + "Don't warn for streams that only have readers. Use this if you abuse streams to have a lazy person's consumers with state."; + + public static final String DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION = + "accept-projects-without-topics"; + public static final String DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_DESC = + "Don't warn for projects without topics."; + public static final String VALIDATE_OPTION = "validate"; public static final String VALIDATE_DESC = "Only run configured validations in your topology"; @@ -109,6 +118,22 @@ private Options buildOptions() { .required(false) .build(); + final Option dontWarnForReadOnlyStreamsOption = + Option.builder() + .longOpt(DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION) + .hasArg(false) + .desc(DONT_WARN_FOR_READ_ONLY_STREAMS_DESC) + .required(false) + .build(); + + final Option dontWarnForProjectsWithoutTopicsOption = + Option.builder() + .longOpt(DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION) + .hasArg(false) + .desc(DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_DESC) + .required(false) + .build(); + final Option quietOption = Option.builder() .longOpt(QUIET_OPTION) @@ -146,6 +171,8 @@ private Options buildOptions() { options.addOption(overridingAdminClientConfigFileOption); options.addOption(dryRunOption); options.addOption(recursiveOption); + options.addOption(dontWarnForReadOnlyStreamsOption); + options.addOption(dontWarnForProjectsWithoutTopicsOption); options.addOption(quietOption); options.addOption(validateOption); options.addOption(versionOption); @@ -185,6 +212,12 @@ private Map parseConfig(CommandLine cmd) { } config.put(DRY_RUN_OPTION, String.valueOf(cmd.hasOption(DRY_RUN_OPTION))); config.put(RECURSIVE_OPTION, String.valueOf(cmd.hasOption(RECURSIVE_OPTION))); + config.put( + DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION, + String.valueOf(cmd.hasOption(DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION))); + config.put( + DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION, + String.valueOf(cmd.hasOption(DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION))); config.put(QUIET_OPTION, String.valueOf(cmd.hasOption(QUIET_OPTION))); config.put(VALIDATE_OPTION, String.valueOf(cmd.hasOption(VALIDATE_OPTION))); config.put( diff --git a/src/main/java/com/purbon/kafka/topology/Configuration.java b/src/main/java/com/purbon/kafka/topology/Configuration.java index 015acf3e5..688cb2aff 100644 --- a/src/main/java/com/purbon/kafka/topology/Configuration.java +++ b/src/main/java/com/purbon/kafka/topology/Configuration.java @@ -396,6 +396,18 @@ public boolean isAllowDeleteKsqlArtefacts() { return config.getBoolean(ALLOW_DELETE_KSQL_ARTEFACTS); } + public boolean isWarnIfReadOnlyStreams() { + return !Boolean.parseBoolean( + cliParams.getOrDefault(DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION, "false")) + && config.getBoolean(STREAMS_WARN_IF_READ_ONLY); + } + + public boolean isWarnIfProjectsWithoutTopics() { + return !Boolean.parseBoolean( + cliParams.getOrDefault(DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION, "false")) + && config.getBoolean(PROJECTS_WARN_IF_NO_TOPICS); + } + public boolean enabledPrincipalTranslation() { return config.getBoolean(TOPOLOGY_PRINCIPAL_TRANSLATION_ENABLED_CONFIG); } diff --git a/src/main/java/com/purbon/kafka/topology/Constants.java b/src/main/java/com/purbon/kafka/topology/Constants.java index 3131e5426..f6b33021b 100644 --- a/src/main/java/com/purbon/kafka/topology/Constants.java +++ b/src/main/java/com/purbon/kafka/topology/Constants.java @@ -83,7 +83,8 @@ public class Constants { static final String ALLOW_DELETE_PRINCIPALS = "allow.delete.principals"; public static final String ALLOW_DELETE_CONNECT_ARTEFACTS = "allow.delete.artefacts.connect"; public static final String ALLOW_DELETE_KSQL_ARTEFACTS = "allow.delete.artefacts.ksql"; - + public static final String STREAMS_WARN_IF_READ_ONLY = "streams.warn-if.read-only"; + public static final String PROJECTS_WARN_IF_NO_TOPICS = "projects.warn-if.no-topics"; public static final String JULIE_ENABLE_PRINCIPAL_MANAGEMENT = "julie.enable.principal.management"; diff --git a/src/main/java/com/purbon/kafka/topology/OnceOnlyWarningLogger.java b/src/main/java/com/purbon/kafka/topology/OnceOnlyWarningLogger.java new file mode 100644 index 000000000..a610eece2 --- /dev/null +++ b/src/main/java/com/purbon/kafka/topology/OnceOnlyWarningLogger.java @@ -0,0 +1,27 @@ +package com.purbon.kafka.topology; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +class OnceOnlyWarningLogger { + + private static final OnceOnlyWarningLogger INSTANCE = new OnceOnlyWarningLogger(); + private static final Logger LOGGER = LogManager.getLogger(OnceOnlyWarningLogger.class); + private boolean remoteStateVerificationDisabledWarningShown = false; + + private OnceOnlyWarningLogger() {} + + public static OnceOnlyWarningLogger getInstance() { + return INSTANCE; + } + + public void logRemoteStateVerificationDisabledWarning() { + if (remoteStateVerificationDisabledWarningShown) { + return; + } + LOGGER.warn( + "Remote state verification is disabled. This is not a good practice, " + + "and in future versions this check may become mandatory."); + remoteStateVerificationDisabledWarningShown = true; + } +} diff --git a/src/main/java/com/purbon/kafka/topology/TopicManager.java b/src/main/java/com/purbon/kafka/topology/TopicManager.java index 900da3487..9f76bfe92 100644 --- a/src/main/java/com/purbon/kafka/topology/TopicManager.java +++ b/src/main/java/com/purbon/kafka/topology/TopicManager.java @@ -137,9 +137,7 @@ private Set loadActualClusterStateIfAvailable(ExecutionPlan plan) throws + StringUtils.join(new ArrayList<>(listOfTopics), ",")); if (!config.shouldVerifyRemoteState()) { - LOGGER.warn( - "Remote state verification disabled, this is not a good practice, be aware" - + "in future versions, this check is going to become mandatory."); + OnceOnlyWarningLogger.getInstance().logRemoteStateVerificationDisabledWarning(); } if (config.shouldVerifyRemoteState() && !config.fetchStateFromTheCluster()) { diff --git a/src/main/java/com/purbon/kafka/topology/clients/JulieHttpClient.java b/src/main/java/com/purbon/kafka/topology/clients/JulieHttpClient.java index 4b9c853ef..aa7681034 100644 --- a/src/main/java/com/purbon/kafka/topology/clients/JulieHttpClient.java +++ b/src/main/java/com/purbon/kafka/topology/clients/JulieHttpClient.java @@ -130,7 +130,10 @@ protected TrustManager[] getTrustManagersFromTrustStore(Configuration config) } protected KeyManager[] getKeyManagersFromKeyStore(Configuration config) - throws NoSuchAlgorithmException, CertificateException, KeyStoreException, IOException, + throws NoSuchAlgorithmException, + CertificateException, + KeyStoreException, + IOException, UnrecoverableKeyException { KeyManagerFactory kmf = KeyManagerFactory.getInstance("PKIX"); KeyStore ks = loadKeyStore(config.getSslKeyStoreLocation(), config.getSslKeyStorePassword()); diff --git a/src/main/java/com/purbon/kafka/topology/serdes/TopologyCustomDeserializer.java b/src/main/java/com/purbon/kafka/topology/serdes/TopologyCustomDeserializer.java index d9f388639..6e2d2dfe8 100644 --- a/src/main/java/com/purbon/kafka/topology/serdes/TopologyCustomDeserializer.java +++ b/src/main/java/com/purbon/kafka/topology/serdes/TopologyCustomDeserializer.java @@ -252,11 +252,13 @@ private Project parseProject( var topicsNode = rootNode.get(TOPICS_KEY); if (topicsNode == null) { - LOGGER.warn( - TOPICS_KEY - + " is missing for project: " - + project.getName() - + ", this might be a required field, be aware."); + if (config.isWarnIfProjectsWithoutTopics()) { + LOGGER.warn( + TOPICS_KEY + + " is missing for project: " + + project.getName() + + ", this might be a required field, be aware."); + } } else { var allowList = config.getDlqTopicsAllowList().stream() @@ -423,15 +425,15 @@ private Optional doStreamsElements(JsonParser parser, JsonNode n for (KStream ks : streams) { var topics = ks.getTopics(); - if (topics.get(KStream.READ_TOPICS).isEmpty() || topics.get(KStream.WRITE_TOPICS).isEmpty()) { + if (topics.get(KStream.WRITE_TOPICS).isEmpty() && config.isWarnIfReadOnlyStreams()) { LOGGER.warn( "A Kafka Streams application with Id (" + ks.getApplicationId() + ") and Principal (" + ks.getPrincipal() + ")" - + " might require both read and write topics as per its " - + "nature it is always reading and writing into Apache Kafka, be aware if you notice problems."); + + " might require both read and write topics, as per its " + + "nature, it is always reading and writing into Apache Kafka."); } if (topics.get(KStream.READ_TOPICS).isEmpty()) { // should have at minimum read topics defined as we could think of write topics as internal diff --git a/src/main/java/com/purbon/kafka/topology/utils/JSON.java b/src/main/java/com/purbon/kafka/topology/utils/JSON.java index e58eb4b34..aa858c0b5 100644 --- a/src/main/java/com/purbon/kafka/topology/utils/JSON.java +++ b/src/main/java/com/purbon/kafka/topology/utils/JSON.java @@ -1,8 +1,11 @@ package com.purbon.kafka.topology.utils; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.util.DefaultIndenter; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import java.util.List; @@ -11,11 +14,15 @@ public class JSON { private static final ObjectMapper mapper; + private static final ObjectWriter prettyWriter; static { mapper = new ObjectMapper(); mapper.registerModule(new Jdk8Module()); mapper.findAndRegisterModules(); + DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter(); + prettyPrinter.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE); + prettyWriter = mapper.writer(prettyPrinter); } public static Map toMap(String jsonString) throws JsonProcessingException { @@ -27,7 +34,7 @@ public static String asString(Map map) throws JsonProcessingException { } public static String asPrettyString(Map map) throws JsonProcessingException { - return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(map); + return prettyWriter.writeValueAsString(map); } public static List toArray(String jsonString) throws JsonProcessingException { @@ -39,7 +46,7 @@ public static String asString(Object object) throws JsonProcessingException { } public static String asPrettyString(Object object) throws JsonProcessingException { - return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object); + return prettyWriter.writeValueAsString(object); } public static Object toObjectList(String jsonString, Class objectClazz) diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index a9f3d6b8c..0f374feab 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -101,7 +101,6 @@ julie { verify.remote.state = false verify.remote.state = ${?JULIE_VERIFY_REMOTE_STATE} - audit { enabled = false enabled = ${?JULIE_AUDIT_ENABLED} @@ -177,4 +176,11 @@ validations { op = "gte" value = 3 } -} \ No newline at end of file +} + +streams { + warn-if.read-only = true +} +projects { + warn-if.no-topics = true +} diff --git a/src/test/java/com/purbon/kafka/topology/CLITest.java b/src/test/java/com/purbon/kafka/topology/CLITest.java index a19018dee..640463a57 100644 --- a/src/test/java/com/purbon/kafka/topology/CLITest.java +++ b/src/test/java/com/purbon/kafka/topology/CLITest.java @@ -38,12 +38,8 @@ public void testParamPassing() throws Exception { doNothing().when(cli).processTopology(eq("descriptor.yaml"), eq("default"), anyMap()); - Map config = new HashMap<>(); + Map config = getDefaultMap(); config.put(BROKERS_OPTION, "localhost:9092"); - config.put(DRY_RUN_OPTION, "false"); - config.put(RECURSIVE_OPTION, "false"); - config.put(QUIET_OPTION, "false"); - config.put(VALIDATE_OPTION, "false"); config.put(CLIENT_CONFIG_OPTION, "topology-builder-sasl-plain.properties"); config.put(OVERRIDING_CLIENT_CONFIG_OPTION, null); cli.run(args); @@ -63,16 +59,24 @@ public void testDryRun() throws Exception { doNothing().when(cli).processTopology(eq("descriptor.yaml"), eq("default"), anyMap()); - Map config = new HashMap<>(); + Map config = getDefaultMap(); config.put(BROKERS_OPTION, "localhost:9092"); config.put(DRY_RUN_OPTION, "true"); - config.put(RECURSIVE_OPTION, "false"); - config.put(QUIET_OPTION, "false"); - config.put(VALIDATE_OPTION, "false"); config.put(CLIENT_CONFIG_OPTION, "topology-builder-sasl-plain.properties"); config.put(OVERRIDING_CLIENT_CONFIG_OPTION, null); cli.run(args); verify(cli, times(1)).processTopology(eq("descriptor.yaml"), eq("default"), eq(config)); } + + private Map getDefaultMap() { + Map map = new HashMap<>(); + map.put(DRY_RUN_OPTION, "false"); + map.put(RECURSIVE_OPTION, "false"); + map.put(QUIET_OPTION, "false"); + map.put(VALIDATE_OPTION, "false"); + map.put(DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION, "false"); + map.put(DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION, "false"); + return map; + } } diff --git a/src/test/java/com/purbon/kafka/topology/ConfigurationTest.java b/src/test/java/com/purbon/kafka/topology/ConfigurationTest.java index f6fb9e3b9..8f03eee6b 100644 --- a/src/test/java/com/purbon/kafka/topology/ConfigurationTest.java +++ b/src/test/java/com/purbon/kafka/topology/ConfigurationTest.java @@ -21,6 +21,7 @@ import java.util.*; import java.util.stream.Collectors; import org.assertj.core.api.Condition; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -336,4 +337,32 @@ public void emptySaManagedPrefixConfigsShouldRaiseAnError() throws Configuration Configuration config = new Configuration(cliOps, props); config.validateWith(topology); } + + @Test + public void shouldWarnForReadOnlyStreamsByDefaultToKeepBackwardsCompatibility() { + Configuration config = new Configuration(cliOps, props); + Assert.assertTrue(config.isWarnIfReadOnlyStreams()); + } + + @Test + public void shouldOverrideWarnForReadOnlyStreamsFromCommandLine() { + Map localCliOps = new HashMap<>(); + localCliOps.put(CommandLineInterface.DONT_WARN_FOR_READ_ONLY_STREAMS_OPTION, "true"); + Configuration config = new Configuration(localCliOps, props); + Assert.assertFalse(config.isWarnIfReadOnlyStreams()); + } + + @Test + public void shouldWarnForProjectsWithoutTopicsToKeepBackwardsCompatibility() { + Configuration config = new Configuration(cliOps, props); + Assert.assertTrue(config.isWarnIfProjectsWithoutTopics()); + } + + @Test + public void shouldOverrideWarnForProjectsWithoutTopicsFromCommandLine() { + Map localCliOps = new HashMap<>(); + localCliOps.put(CommandLineInterface.DONT_WARN_FOR_PROJECTS_WITHOUT_TOPICS_OPTION, "true"); + Configuration config = new Configuration(localCliOps, props); + Assert.assertFalse(config.isWarnIfProjectsWithoutTopics()); + } } diff --git a/src/test/java/com/purbon/kafka/topology/integration/backend/RedisBackendIT.java b/src/test/java/com/purbon/kafka/topology/integration/backend/RedisBackendIT.java index 03e489cee..895a81b6c 100644 --- a/src/test/java/com/purbon/kafka/topology/integration/backend/RedisBackendIT.java +++ b/src/test/java/com/purbon/kafka/topology/integration/backend/RedisBackendIT.java @@ -41,7 +41,7 @@ public class RedisBackendIT { @Rule public GenericContainer redis = - new GenericContainer<>(DockerImageName.parse("redis:5.0.3-alpine")).withExposedPorts(6379); + new GenericContainer<>(DockerImageName.parse("redis:7.2.4")).withExposedPorts(6379); private static SaslPlaintextKafkaContainer container; private TopicManager topicManager; @@ -158,6 +158,6 @@ public void testTopicCreation() throws IOException { String content = jedis.get(bucket); assertThat(content) .contains( - "\"topics\" : [ \"testTopicCreation.project.topicB\", \"testTopicCreation.project.topicA\" ]"); + "\"topics\" : [\n \"testTopicCreation.project.topicB\",\n \"testTopicCreation.project.topicA\"\n ]"); } } diff --git a/src/test/java/com/purbon/kafka/topology/integration/containerutils/ConnectContainer.java b/src/test/java/com/purbon/kafka/topology/integration/containerutils/ConnectContainer.java index 342612d70..9cc1dc1bd 100644 --- a/src/test/java/com/purbon/kafka/topology/integration/containerutils/ConnectContainer.java +++ b/src/test/java/com/purbon/kafka/topology/integration/containerutils/ConnectContainer.java @@ -8,7 +8,7 @@ public class ConnectContainer extends GenericContainer { private static final DockerImageName DEFAULT_IMAGE = - DockerImageName.parse("confluentinc/cp-kafka-connect").withTag("7.5.0"); + DockerImageName.parse("confluentinc/cp-kafka-connect").withTag("7.6.0"); private static int CONNECT_PORT = 8083; private static int CONNECT_SSL_PORT = 8084; diff --git a/src/test/java/com/purbon/kafka/topology/integration/containerutils/ContainerTestUtils.java b/src/test/java/com/purbon/kafka/topology/integration/containerutils/ContainerTestUtils.java index fcae8f0dd..338af253c 100644 --- a/src/test/java/com/purbon/kafka/topology/integration/containerutils/ContainerTestUtils.java +++ b/src/test/java/com/purbon/kafka/topology/integration/containerutils/ContainerTestUtils.java @@ -17,7 +17,7 @@ public final class ContainerTestUtils { - static final String DEFAULT_CP_KAFKA_VERSION = "7.5.0"; + static final String DEFAULT_CP_KAFKA_VERSION = "7.6.0"; private ContainerTestUtils() {} diff --git a/src/test/java/com/purbon/kafka/topology/integration/containerutils/SchemaRegistryContainer.java b/src/test/java/com/purbon/kafka/topology/integration/containerutils/SchemaRegistryContainer.java index 389f2bbf0..73db19241 100644 --- a/src/test/java/com/purbon/kafka/topology/integration/containerutils/SchemaRegistryContainer.java +++ b/src/test/java/com/purbon/kafka/topology/integration/containerutils/SchemaRegistryContainer.java @@ -6,7 +6,7 @@ public class SchemaRegistryContainer extends GenericContainer { private static final DockerImageName DEFAULT_IMAGE = - DockerImageName.parse("confluentinc/cp-schema-registry").withTag("7.5.0"); + DockerImageName.parse("confluentinc/cp-schema-registry").withTag("7.6.0"); public static final int SR_PORT = 8081;