From f33685f2ced5733faab4cd317bd8fbb32f232137 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 10:14:13 +0200
Subject: [PATCH 01/25] Add image tag field to Kubernetes App values
---
docs/docs/schema/defaults.json | 33 ++++++++++++++++++-
docs/docs/schema/pipeline.json | 18 ++++++++++
kpops/components/base_components/kafka_app.py | 10 +++++-
.../base_components/kafka_connector.py | 9 +++++
.../base_components/kubernetes_app.py | 4 +++
.../test-distributed-defaults/defaults.yaml | 2 ++
tests/components/test_helm_app.py | 4 +--
tests/components/test_kafka_sink_connector.py | 8 -----
tests/components/test_producer_app.py | 3 ++
tests/components/test_streams_bootstrap.py | 1 +
.../pipeline-with-env-defaults/defaults.yaml | 2 ++
.../test_generate/atm-fraud/pipeline.yaml | 4 ++-
.../test_generate/word-count/pipeline.yaml | 4 ++-
.../test_default_config/pipeline.yaml | 4 +++
.../test_inflate_pipeline/pipeline.yaml | 8 ++++-
.../pipeline.yaml | 6 +++-
.../test_load_pipeline/pipeline.yaml | 2 ++
.../pipeline.yaml | 2 ++
.../pipeline.yaml | 2 ++
.../test_model_serialization/pipeline.yaml | 2 ++
.../test_no_input_topic/pipeline.yaml | 4 +++
.../pipeline.yaml | 2 ++
.../test_pipelines_with_envs/pipeline.yaml | 2 ++
.../test_read_from_component/pipeline.yaml | 26 +++++++++++++--
.../pipeline.yaml | 2 ++
.../pipeline.yaml | 4 +++
.../pipeline.yaml | 4 +++
.../test_with_env_defaults/pipeline.yaml | 6 +++-
tests/pipeline/test_generate.py | 1 +
29 files changed, 160 insertions(+), 19 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index ae4af6582..cc7d70207 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -155,6 +155,12 @@
"additionalProperties": true,
"description": "Helm app values.",
"properties": {
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
@@ -278,6 +284,12 @@
"additionalProperties": true,
"description": "Settings specific to Kafka Apps.",
"properties": {
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
@@ -725,7 +737,14 @@
"KubernetesAppValues": {
"additionalProperties": true,
"description": "Settings specific to Kubernetes apps.",
- "properties": {},
+ "properties": {
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ }
+ },
"title": "KubernetesAppValues",
"type": "object"
},
@@ -920,6 +939,12 @@
"additionalProperties": true,
"description": "Settings specific to producers.",
"properties": {
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
@@ -1273,6 +1298,12 @@
"default": null,
"description": "Kubernetes event-driven autoscaling config"
},
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index ea33470b3..9fc5bdc5b 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -155,6 +155,12 @@
"additionalProperties": true,
"description": "Helm app values.",
"properties": {
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
@@ -588,6 +594,12 @@
"additionalProperties": true,
"description": "Settings specific to producers.",
"properties": {
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
@@ -941,6 +953,12 @@
"default": null,
"description": "Kubernetes event-driven autoscaling config"
},
+ "imageTag": {
+ "default": "latest",
+ "description": "",
+ "title": "Imagetag",
+ "type": "string"
+ },
"nameOverride": {
"anyOf": [
{
diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py
index f52ca6e78..b3e3248f1 100644
--- a/kpops/components/base_components/kafka_app.py
+++ b/kpops/components/base_components/kafka_app.py
@@ -3,7 +3,7 @@
import logging
from abc import ABC
from collections.abc import Callable
-from typing import Any
+from typing import Any, Self
import pydantic
from pydantic import AliasChoices, ConfigDict, Field
@@ -131,6 +131,14 @@ class KafkaApp(PipelineComponent, ABC):
description=describe_attr("app", __doc__),
)
+ @pydantic.model_validator(mode="after")
+ def warning_for_latest_image_tag(self) -> Self:
+ if self.app.image_tag == "latest" and "$" not in self.name:
+ log.warning(
+ f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag."
+ )
+ return self
+
@override
async def deploy(self, dry_run: bool) -> None:
if self.to:
diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py
index 1d9a296b3..134e37567 100644
--- a/kpops/components/base_components/kafka_connector.py
+++ b/kpops/components/base_components/kafka_connector.py
@@ -134,6 +134,15 @@ def connector_config_should_have_component_name(
app["name"] = component_name
return KafkaConnectorConfig(**app)
+ @pydantic.model_validator(mode="after")
+ def warning_for_latest_image_tag(self) -> Self:
+ if self.resetter_values.image_tag == "latest":
+ log.warning(
+ f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. "
+ f"Please, consider providing a stable imageTag."
+ )
+ return self
+
@cached_property
def _resetter(self) -> KafkaConnectorResetter:
kwargs: dict[str, Any] = {}
diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py
index e2ad7d39c..3c7968917 100644
--- a/kpops/components/base_components/kubernetes_app.py
+++ b/kpops/components/base_components/kubernetes_app.py
@@ -21,6 +21,10 @@
class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel):
"""Settings specific to Kubernetes apps."""
+ image_tag: str = Field(
+ default="latest", description=describe_attr("imageTag", __doc__)
+ )
+
model_config = ConfigDict(
extra="allow",
)
diff --git a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml
index 567c831f3..d835cc16f 100644
--- a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml
+++ b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml
@@ -1,3 +1,5 @@
helm-app:
name: ${component.type}
namespace: example-namespace
+ app:
+ imageTag: latest
diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py
index cd9a78b66..cf819050f 100644
--- a/tests/components/test_helm_app.py
+++ b/tests/components/test_helm_app.py
@@ -49,7 +49,7 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock:
@pytest.fixture()
def app_values(self) -> HelmAppValues:
- return HelmAppValues(**{"foo": "test-value"})
+ return HelmAppValues(**{"foo": "test-value", "imageTag": "latest"})
@pytest.fixture()
def repo_config(self) -> HelmRepoConfig:
@@ -235,7 +235,7 @@ def test_helm_name_override(
name="helm-app-name-is-very-long-as-well",
config=config,
handlers=handlers,
- app=HelmAppValues(),
+ app=HelmAppValues(**{"imageTag": "latest"}),
namespace="test-namespace",
repo_config=repo_config,
)
diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py
index 51b30a61d..1b98e4ac4 100644
--- a/tests/components/test_kafka_sink_connector.py
+++ b/tests/components/test_kafka_sink_connector.py
@@ -107,14 +107,6 @@ def test_connector_config_parsing(
handlers: ComponentHandlers,
connector_config: KafkaConnectorConfig,
):
- connector = KafkaSinkConnector(
- name=CONNECTOR_NAME,
- config=config,
- handlers=handlers,
- app=connector_config,
- resetter_namespace=RESETTER_NAMESPACE,
- )
-
topic_pattern = ".*"
connector = KafkaSinkConnector(
name=CONNECTOR_NAME,
diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py
index 4f7184ead..514fb1cea 100644
--- a/tests/components/test_producer_app.py
+++ b/tests/components/test_producer_app.py
@@ -67,6 +67,7 @@ def producer_app(
"version": "2.4.2",
"namespace": "test-namespace",
"app": {
+ "imageTag": "latest",
"streams": {"brokers": "fake-broker:9092"},
},
"clean_schemas": True,
@@ -108,6 +109,7 @@ def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers):
**{
"namespace": "test-namespace",
"app": {
+ "imageTag": "latest",
"namespace": "test-namespace",
"streams": {"brokers": "fake-broker:9092"},
},
@@ -324,6 +326,7 @@ def test_get_output_topics(
**{
"namespace": "test-namespace",
"app": {
+ "imageTag": "latest",
"namespace": "test-namespace",
"streams": {"brokers": "fake-broker:9092"},
},
diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py
index a82fca8a9..caef4a904 100644
--- a/tests/components/test_streams_bootstrap.py
+++ b/tests/components/test_streams_bootstrap.py
@@ -48,6 +48,7 @@ def test_default_configs(self, config: KpopsConfig, handlers: ComponentHandlers)
)
assert streams_bootstrap.version == "2.9.0"
assert streams_bootstrap.namespace == "test-namespace"
+ assert streams_bootstrap.app.image_tag == "latest"
@pytest.mark.asyncio()
async def test_should_deploy_streams_bootstrap_app(
diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
index afc821f3c..6a77cfd61 100644
--- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
+++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
@@ -1,6 +1,8 @@
kubernetes-app:
name: ${component.type}
namespace: example-namespace
+ app:
+ imageTag: "latest"
kafka-app:
app:
streams:
diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
index 528da26bb..bc1517900 100644
--- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
@@ -393,6 +393,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: atm-fraud-postgresql-connector
connectorType: sink
+ imageTag: latest
name: postgresql-connector
namespace: ${NAMESPACE}
prefix: atm-fraud-
@@ -429,5 +430,6 @@
value.converter.schema.registry.url: http://k8kafka-cp-schema-registry.${NAMESPACE}.svc.cluster.local:8081
name: postgresql-connector
prefix: atm-fraud-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
type: kafka-sink-connector
diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
index 7e7d16fe8..1f6571865 100644
--- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
@@ -142,6 +142,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: word-count-redis-sink-connector
connectorType: sink
+ imageTag: latest
name: redis-sink-connector
namespace: ${NAMESPACE}
prefix: word-count-
@@ -164,5 +165,6 @@
value.converter: org.apache.kafka.connect.storage.StringConverter
name: redis-sink-connector
prefix: word-count-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
type: kafka-sink-connector
diff --git a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml
index cbd0e251e..4549c6209 100644
--- a/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_default_config/pipeline.yaml
@@ -1,5 +1,6 @@
- _cleaner:
app:
+ imageTag: latest
resources:
limits:
memory: 2G
@@ -21,6 +22,7 @@
type: producer-app-cleaner
version: 2.9.0
app:
+ imageTag: latest
resources:
limits:
memory: 2G
@@ -50,6 +52,7 @@
- _cleaner:
app:
image: some-image
+ imageTag: latest
labels:
pipeline: resources-custom-config
persistence:
@@ -77,6 +80,7 @@
version: 2.9.0
app:
image: some-image
+ imageTag: latest
labels:
pipeline: resources-custom-config
persistence:
diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
index e5e003376..8d5be21bf 100644
--- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
@@ -64,6 +64,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -105,6 +106,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -253,6 +255,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-pipeline-with-inflate-should-inflate-inflated-sink-connector
connectorType: sink
+ imageTag: latest
name: should-inflate-inflated-sink-connector
namespace: example-namespace
prefix: resources-pipeline-with-inflate-
@@ -280,7 +283,8 @@
transforms.changeTopic.replacement: resources-pipeline-with-inflate-should-inflate-index-v1
name: should-inflate-inflated-sink-connector
prefix: resources-pipeline-with-inflate-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
to:
models: {}
topics:
@@ -293,6 +297,7 @@
type: kafka-sink-connector
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -317,6 +322,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
index bb569e772..7a6417d01 100644
--- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
@@ -1,6 +1,7 @@
- _cleaner:
app:
image: fake-image
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -26,6 +27,7 @@
version: 2.4.2
app:
image: fake-image
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -71,6 +73,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-kafka-connect-sink-es-sink-connector
connectorType: sink
+ imageTag: latest
name: es-sink-connector
namespace: example-namespace
prefix: resources-kafka-connect-sink-
@@ -97,6 +100,7 @@
topics: example-output
name: es-sink-connector
prefix: resources-kafka-connect-sink-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
type: kafka-sink-connector
diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml
index 87a88601c..d49fa57d2 100644
--- a/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline/pipeline.yaml
@@ -64,6 +64,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -105,6 +106,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml
index 2787d7444..42cd958d5 100644
--- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_folder_path/pipeline.yaml
@@ -65,6 +65,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -104,6 +105,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
diff --git a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml
index 2787d7444..42cd958d5 100644
--- a/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_load_pipeline_with_multiple_pipeline_paths/pipeline.yaml
@@ -65,6 +65,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -104,6 +105,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
diff --git a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml
index 02e06ff34..9e85d9e01 100644
--- a/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_model_serialization/pipeline.yaml
@@ -1,5 +1,6 @@
- _cleaner:
app:
+ imageTag: latest
streams:
brokers: test
outputTopic: out
@@ -19,6 +20,7 @@
type: producer-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
streams:
brokers: test
outputTopic: out
diff --git a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml
index a77aea9be..3c57c8eb1 100644
--- a/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_no_input_topic/pipeline.yaml
@@ -2,6 +2,7 @@
app:
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -32,6 +33,7 @@
app:
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -77,6 +79,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -103,6 +106,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
diff --git a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml
index d8850383e..348443bbc 100644
--- a/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_no_user_defined_components/pipeline.yaml
@@ -1,6 +1,7 @@
- _cleaner:
app:
image: fake-image
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -26,6 +27,7 @@
version: 2.4.2
app:
image: fake-image
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
diff --git a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml
index 344e9c5b1..36d50168f 100644
--- a/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_pipelines_with_envs/pipeline.yaml
@@ -64,6 +64,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -105,6 +106,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
index 761f21e63..0ad5a8dd7 100644
--- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
@@ -1,5 +1,6 @@
- _cleaner:
app:
+ imageTag: latest
streams:
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
outputTopic: resources-read-from-component-producer1
@@ -16,6 +17,7 @@
type: producer-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
streams:
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
outputTopic: resources-read-from-component-producer1
@@ -38,6 +40,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
streams:
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
outputTopic: resources-read-from-component-producer2
@@ -54,6 +57,7 @@
type: producer-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
streams:
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
outputTopic: resources-read-from-component-producer2
@@ -168,6 +172,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-read-from-component-inflate-step-inflated-sink-connector
connectorType: sink
+ imageTag: latest
name: inflate-step-inflated-sink-connector
namespace: example-namespace
prefix: resources-read-from-component-
@@ -195,7 +200,8 @@
transforms.changeTopic.replacement: resources-read-from-component-inflate-step-index-v1
name: inflate-step-inflated-sink-connector
prefix: resources-read-from-component-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
to:
models: {}
topics:
@@ -208,6 +214,7 @@
type: kafka-sink-connector
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -232,6 +239,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -360,6 +368,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector
connectorType: sink
+ imageTag: latest
name: inflate-step-without-prefix-inflated-sink-connector
namespace: example-namespace
prefix: resources-read-from-component-
@@ -387,7 +396,8 @@
transforms.changeTopic.replacement: resources-read-from-component-inflate-step-without-prefix-index-v1
name: inflate-step-without-prefix-inflated-sink-connector
prefix: resources-read-from-component-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
to:
models: {}
topics:
@@ -400,6 +410,7 @@
type: kafka-sink-connector
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -424,6 +435,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -460,6 +472,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -484,6 +497,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -525,6 +539,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -549,6 +564,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -589,6 +605,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -613,6 +630,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -653,6 +671,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -676,6 +695,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -713,6 +733,7 @@
version: 2.4.2
- _cleaner:
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -736,6 +757,7 @@
type: streams-app-cleaner
version: 2.4.2
app:
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
diff --git a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml
index 8ca686f30..8878759fa 100644
--- a/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_substitute_in_component/pipeline.yaml
@@ -72,6 +72,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
@@ -113,6 +114,7 @@
topics: []
commandLine:
CONVERT_XML: true
+ imageTag: latest
persistence:
enabled: false
resources:
diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml
index f78e8f0d1..f96db8745 100644
--- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_absolute_defaults_path/pipeline.yaml
@@ -1,5 +1,6 @@
- _cleaner:
app:
+ imageTag: latest
resources:
limits:
memory: 2G
@@ -21,6 +22,7 @@
type: producer-app-cleaner
version: 2.9.0
app:
+ imageTag: latest
resources:
limits:
memory: 2G
@@ -50,6 +52,7 @@
- _cleaner:
app:
image: some-image
+ imageTag: latest
labels:
pipeline: resources-custom-config
persistence:
@@ -77,6 +80,7 @@
version: 2.9.0
app:
image: some-image
+ imageTag: latest
labels:
pipeline: resources-custom-config
persistence:
diff --git a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml
index f78e8f0d1..f96db8745 100644
--- a/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_with_custom_config_with_relative_defaults_path/pipeline.yaml
@@ -1,5 +1,6 @@
- _cleaner:
app:
+ imageTag: latest
resources:
limits:
memory: 2G
@@ -21,6 +22,7 @@
type: producer-app-cleaner
version: 2.9.0
app:
+ imageTag: latest
resources:
limits:
memory: 2G
@@ -50,6 +52,7 @@
- _cleaner:
app:
image: some-image
+ imageTag: latest
labels:
pipeline: resources-custom-config
persistence:
@@ -77,6 +80,7 @@
version: 2.9.0
app:
image: some-image
+ imageTag: latest
labels:
pipeline: resources-custom-config
persistence:
diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
index cea0b2660..670ee9942 100644
--- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
@@ -1,6 +1,7 @@
- _cleaner:
app:
image: fake-image
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -26,6 +27,7 @@
version: 2.9.0
app:
image: fake-image
+ imageTag: latest
persistence:
enabled: false
statefulSet: false
@@ -71,6 +73,7 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-pipeline-with-env-defaults-es-sink-connector
connectorType: sink
+ imageTag: latest
name: es-sink-connector
namespace: development-namespace
prefix: resources-pipeline-with-env-defaults-
@@ -97,6 +100,7 @@
topics: example-output
name: es-sink-connector
prefix: resources-pipeline-with-env-defaults-
- resetter_values: {}
+ resetter_values:
+ imageTag: latest
type: kafka-sink-connector
diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py
index 915ae17d7..b0fdd2db4 100644
--- a/tests/pipeline/test_generate.py
+++ b/tests/pipeline/test_generate.py
@@ -219,6 +219,7 @@ def test_kafka_connector_config_parsing(self):
str(RESOURCE_PATH / "kafka-connect-sink-config" / PIPELINE_YAML),
"--config",
str(RESOURCE_PATH / "kafka-connect-sink-config"),
+ "--verbose",
],
catch_exceptions=False,
)
From c7de8c189389d4054ca50b2f0b70afff0d07b344 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 10:17:21 +0200
Subject: [PATCH 02/25] Add image tag field to Kubernetes App values
---
kpops/components/base_components/kubernetes_app.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py
index 3c7968917..fef30c35f 100644
--- a/kpops/components/base_components/kubernetes_app.py
+++ b/kpops/components/base_components/kubernetes_app.py
@@ -19,7 +19,10 @@
class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel):
- """Settings specific to Kubernetes apps."""
+ """Settings specific to Kubernetes apps.
+
+ :param image_tag: The image tag of the Kubernetes app.
+ """
image_tag: str = Field(
default="latest", description=describe_attr("imageTag", __doc__)
From 6fd51edfede0fa042876a6eb0b2b2ec55d0ea554 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 10:19:31 +0200
Subject: [PATCH 03/25] Update files
---
tests/components/test_helm_app.py | 4 ++--
tests/components/test_producer_app.py | 3 ---
.../resources/pipeline-with-env-defaults/defaults.yaml | 2 --
tests/pipeline/test_generate.py | 1 -
4 files changed, 2 insertions(+), 8 deletions(-)
diff --git a/tests/components/test_helm_app.py b/tests/components/test_helm_app.py
index cf819050f..cd9a78b66 100644
--- a/tests/components/test_helm_app.py
+++ b/tests/components/test_helm_app.py
@@ -49,7 +49,7 @@ def log_info_mock(self, mocker: MockerFixture) -> MagicMock:
@pytest.fixture()
def app_values(self) -> HelmAppValues:
- return HelmAppValues(**{"foo": "test-value", "imageTag": "latest"})
+ return HelmAppValues(**{"foo": "test-value"})
@pytest.fixture()
def repo_config(self) -> HelmRepoConfig:
@@ -235,7 +235,7 @@ def test_helm_name_override(
name="helm-app-name-is-very-long-as-well",
config=config,
handlers=handlers,
- app=HelmAppValues(**{"imageTag": "latest"}),
+ app=HelmAppValues(),
namespace="test-namespace",
repo_config=repo_config,
)
diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py
index 514fb1cea..4f7184ead 100644
--- a/tests/components/test_producer_app.py
+++ b/tests/components/test_producer_app.py
@@ -67,7 +67,6 @@ def producer_app(
"version": "2.4.2",
"namespace": "test-namespace",
"app": {
- "imageTag": "latest",
"streams": {"brokers": "fake-broker:9092"},
},
"clean_schemas": True,
@@ -109,7 +108,6 @@ def test_output_topics(self, config: KpopsConfig, handlers: ComponentHandlers):
**{
"namespace": "test-namespace",
"app": {
- "imageTag": "latest",
"namespace": "test-namespace",
"streams": {"brokers": "fake-broker:9092"},
},
@@ -326,7 +324,6 @@ def test_get_output_topics(
**{
"namespace": "test-namespace",
"app": {
- "imageTag": "latest",
"namespace": "test-namespace",
"streams": {"brokers": "fake-broker:9092"},
},
diff --git a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
index 6a77cfd61..afc821f3c 100644
--- a/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
+++ b/tests/pipeline/resources/pipeline-with-env-defaults/defaults.yaml
@@ -1,8 +1,6 @@
kubernetes-app:
name: ${component.type}
namespace: example-namespace
- app:
- imageTag: "latest"
kafka-app:
app:
streams:
diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py
index b0fdd2db4..915ae17d7 100644
--- a/tests/pipeline/test_generate.py
+++ b/tests/pipeline/test_generate.py
@@ -219,7 +219,6 @@ def test_kafka_connector_config_parsing(self):
str(RESOURCE_PATH / "kafka-connect-sink-config" / PIPELINE_YAML),
"--config",
str(RESOURCE_PATH / "kafka-connect-sink-config"),
- "--verbose",
],
catch_exceptions=False,
)
From f54ef8450dc26f1440468615ec6bfadda44a17b9 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 10:28:35 +0200
Subject: [PATCH 04/25] Update files
---
kpops/components/base_components/kubernetes_app.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py
index fef30c35f..7c5cdb8ea 100644
--- a/kpops/components/base_components/kubernetes_app.py
+++ b/kpops/components/base_components/kubernetes_app.py
@@ -21,7 +21,7 @@
class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel):
"""Settings specific to Kubernetes apps.
- :param image_tag: The image tag of the Kubernetes app.
+ :param image_tag: Specifies the version or specific build of a container image to use.
"""
image_tag: str = Field(
From bd9e48791711036b10463af87601b81e3d60df0d Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 10:29:11 +0200
Subject: [PATCH 05/25] Update files
---
docs/docs/schema/defaults.json | 10 +++++-----
docs/docs/schema/pipeline.json | 6 +++---
kpops/components/base_components/kubernetes_app.py | 2 +-
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index cc7d70207..3dba96dff 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -157,7 +157,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -286,7 +286,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -740,7 +740,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
}
@@ -941,7 +941,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -1300,7 +1300,7 @@
},
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index 9fc5bdc5b..cfa63af47 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -157,7 +157,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -596,7 +596,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -955,7 +955,7 @@
},
"imageTag": {
"default": "latest",
- "description": "",
+ "description": "Specifies the version or speci\ufb01c build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py
index 7c5cdb8ea..9901a37d1 100644
--- a/kpops/components/base_components/kubernetes_app.py
+++ b/kpops/components/base_components/kubernetes_app.py
@@ -25,7 +25,7 @@ class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel):
"""
image_tag: str = Field(
- default="latest", description=describe_attr("imageTag", __doc__)
+ default="latest", description=describe_attr("image_tag", __doc__)
)
model_config = ConfigDict(
From af35a803a20b881921fcf932f4c571382b911cb6 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 10:29:41 +0200
Subject: [PATCH 06/25] Update files
---
docs/docs/schema/defaults.json | 10 +++++-----
docs/docs/schema/pipeline.json | 6 +++---
kpops/components/base_components/kubernetes_app.py | 2 +-
3 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index 3dba96dff..d670aa264 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -157,7 +157,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -286,7 +286,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -740,7 +740,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
}
@@ -941,7 +941,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -1300,7 +1300,7 @@
},
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index cfa63af47..1d66028e5 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -157,7 +157,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -596,7 +596,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -955,7 +955,7 @@
},
"imageTag": {
"default": "latest",
- "description": "Specifies the version or speci\ufb01c build of a container image to use.",
+ "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py
index 9901a37d1..27fb1286a 100644
--- a/kpops/components/base_components/kubernetes_app.py
+++ b/kpops/components/base_components/kubernetes_app.py
@@ -21,7 +21,7 @@
class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel):
"""Settings specific to Kubernetes apps.
- :param image_tag: Specifies the version or specific build of a container image to use.
+ :param image_tag: Specifies the version or specific build of a container image to use.
"""
image_tag: str = Field(
From 0fd9c1e9f17a07e645b259df076d2a2fcbc3c141 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 14:53:56 +0200
Subject: [PATCH 07/25] Move image tag to streamsbootstrap values
---
docs/docs/schema/defaults.json | 57 ++++++++++---------
docs/docs/schema/pipeline.json | 8 ---
.../component_handlers/kafka_connect/model.py | 2 +
kpops/components/base_components/kafka_app.py | 10 +---
.../base_components/kafka_connector.py | 17 +++---
.../base_components/kubernetes_app.py | 9 +--
.../components/streams_bootstrap/__init__.py | 29 +++++++++-
.../streams_bootstrap/producer/model.py | 3 +-
.../streams_bootstrap/streams/model.py | 3 +-
tests/components/test_streams_bootstrap.py | 1 -
.../test_generate/atm-fraud/pipeline.yaml | 3 +-
.../test_generate/word-count/pipeline.yaml | 3 +-
.../test_inflate_pipeline/pipeline.yaml | 3 +-
.../pipeline.yaml | 3 +-
.../test_read_from_component/pipeline.yaml | 6 +-
.../test_with_env_defaults/pipeline.yaml | 3 +-
16 files changed, 83 insertions(+), 77 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index d670aa264..b6644fa31 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -155,12 +155,6 @@
"additionalProperties": true,
"description": "Helm app values.",
"properties": {
- "imageTag": {
- "default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
- "title": "Imagetag",
- "type": "string"
- },
"nameOverride": {
"anyOf": [
{
@@ -284,12 +278,6 @@
"additionalProperties": true,
"description": "Settings specific to Kafka Apps.",
"properties": {
- "imageTag": {
- "default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
- "title": "Imagetag",
- "type": "string"
- },
"nameOverride": {
"anyOf": [
{
@@ -737,14 +725,7 @@
"KubernetesAppValues": {
"additionalProperties": true,
"description": "Settings specific to Kubernetes apps.",
- "properties": {
- "imageTag": {
- "default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
- "title": "Imagetag",
- "type": "string"
- }
- },
+ "properties": {},
"title": "KubernetesAppValues",
"type": "object"
},
@@ -941,7 +922,6 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -1300,7 +1280,6 @@
},
"imageTag": {
"default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -1359,10 +1338,10 @@
"app": {
"allOf": [
{
- "$ref": "#/$defs/HelmAppValues"
+ "$ref": "#/$defs/StreamsBootstrapValues"
}
],
- "description": "Helm app values"
+ "description": ""
},
"from": {
"anyOf": [
@@ -1440,12 +1419,38 @@
},
"required": [
"name",
- "namespace",
- "app"
+ "namespace"
],
"title": "StreamsBootstrap",
"type": "object"
},
+ "StreamsBootstrapValues": {
+ "additionalProperties": true,
+ "description": "Base value class for all streams bootstrap related components.",
+ "properties": {
+ "imageTag": {
+ "default": "latest",
+ "title": "Imagetag",
+ "type": "string"
+ },
+ "nameOverride": {
+ "anyOf": [
+ {
+ "maxLength": 63,
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "default": null,
+ "description": "Helm chart name override, assigned automatically",
+ "title": "Nameoverride"
+ }
+ },
+ "title": "StreamsBootstrapValues",
+ "type": "object"
+ },
"StreamsConfig": {
"additionalProperties": true,
"description": "Streams Bootstrap streams section.",
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index 1d66028e5..0d98c4e7e 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -155,12 +155,6 @@
"additionalProperties": true,
"description": "Helm app values.",
"properties": {
- "imageTag": {
- "default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
- "title": "Imagetag",
- "type": "string"
- },
"nameOverride": {
"anyOf": [
{
@@ -596,7 +590,6 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
@@ -955,7 +948,6 @@
},
"imageTag": {
"default": "latest",
- "description": "Specifies the version or specific build of a container image to use.",
"title": "Imagetag",
"type": "string"
},
diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py
index 840df06c3..c4aeafa31 100644
--- a/kpops/component_handlers/kafka_connect/model.py
+++ b/kpops/component_handlers/kafka_connect/model.py
@@ -5,6 +5,7 @@
from pydantic import (
BaseModel,
ConfigDict,
+ Field,
SerializationInfo,
field_validator,
model_serializer,
@@ -123,3 +124,4 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel):
class KafkaConnectorResetterValues(HelmAppValues):
connector_type: Literal["source", "sink"]
config: KafkaConnectorResetterConfig
+ image_tag: str = Field(default="latest", alias="imageTag")
diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py
index b3e3248f1..f52ca6e78 100644
--- a/kpops/components/base_components/kafka_app.py
+++ b/kpops/components/base_components/kafka_app.py
@@ -3,7 +3,7 @@
import logging
from abc import ABC
from collections.abc import Callable
-from typing import Any, Self
+from typing import Any
import pydantic
from pydantic import AliasChoices, ConfigDict, Field
@@ -131,14 +131,6 @@ class KafkaApp(PipelineComponent, ABC):
description=describe_attr("app", __doc__),
)
- @pydantic.model_validator(mode="after")
- def warning_for_latest_image_tag(self) -> Self:
- if self.app.image_tag == "latest" and "$" not in self.name:
- log.warning(
- f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag."
- )
- return self
-
@override
async def deploy(self, dry_run: bool) -> None:
if self.to:
diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py
index 134e37567..40e9ad1c8 100644
--- a/kpops/components/base_components/kafka_connector.py
+++ b/kpops/components/base_components/kafka_connector.py
@@ -134,14 +134,15 @@ def connector_config_should_have_component_name(
app["name"] = component_name
return KafkaConnectorConfig(**app)
- @pydantic.model_validator(mode="after")
- def warning_for_latest_image_tag(self) -> Self:
- if self.resetter_values.image_tag == "latest":
- log.warning(
- f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. "
- f"Please, consider providing a stable imageTag."
- )
- return self
+ # @pydantic.model_validator(mode="after")
+ # def warning_for_latest_image_tag(self) -> Self:
+ # resetter_image_tag = self.resetter_values.get("imageTag")
+ # if not resetter_image_tag or resetter_image_tag == "latest":
+ # log.warning(
+ # f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. "
+ # f"Please, consider providing a stable imageTag."
+ # )
+ # return self
@cached_property
def _resetter(self) -> KafkaConnectorResetter:
diff --git a/kpops/components/base_components/kubernetes_app.py b/kpops/components/base_components/kubernetes_app.py
index 27fb1286a..e2ad7d39c 100644
--- a/kpops/components/base_components/kubernetes_app.py
+++ b/kpops/components/base_components/kubernetes_app.py
@@ -19,14 +19,7 @@
class KubernetesAppValues(CamelCaseConfigModel, DescConfigModel):
- """Settings specific to Kubernetes apps.
-
- :param image_tag: Specifies the version or specific build of a container image to use.
- """
-
- image_tag: str = Field(
- default="latest", description=describe_attr("image_tag", __doc__)
- )
+ """Settings specific to Kubernetes apps."""
model_config = ConfigDict(
extra="allow",
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 1b02b091b..2501643ff 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -1,9 +1,12 @@
+import logging
from abc import ABC
+from typing import Self
+import pydantic
from pydantic import Field
from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig
-from kpops.components.base_components.helm_app import HelmApp
+from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig(
@@ -12,6 +15,17 @@
)
STREAMS_BOOTSTRAP_VERSION = "2.9.0"
+log = logging.getLogger("StreamsBootstrap")
+
+
+class StreamsBootstrapValues(HelmAppValues):
+ """Base value class for all streams bootstrap related components.
+
+ :param image_tag: Docker image tag of the Kafka Streams app.
+ """
+
+ image_tag: str = Field(default="latest")
+
class StreamsBootstrap(HelmApp, ABC):
"""Base for components with a streams-bootstrap Helm chart.
@@ -21,6 +35,11 @@ class StreamsBootstrap(HelmApp, ABC):
:param version: Helm chart version, defaults to "2.9.0"
"""
+ app: StreamsBootstrapValues = Field(
+ default_factory=StreamsBootstrapValues,
+ description=describe_attr("app", __doc__),
+ )
+
repo_config: HelmRepoConfig = Field(
default=STREAMS_BOOTSTRAP_HELM_REPO,
description=describe_attr("repo_config", __doc__),
@@ -29,3 +48,11 @@ class StreamsBootstrap(HelmApp, ABC):
default=STREAMS_BOOTSTRAP_VERSION,
description=describe_attr("version", __doc__),
)
+
+ @pydantic.model_validator(mode="after")
+ def warning_for_latest_image_tag(self) -> Self:
+ if self.app.image_tag == "latest" and "$" not in self.name:
+ log.warning(
+ f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag."
+ )
+ return self
diff --git a/kpops/components/streams_bootstrap/producer/model.py b/kpops/components/streams_bootstrap/producer/model.py
index caef8d29d..2dc3b5927 100644
--- a/kpops/components/streams_bootstrap/producer/model.py
+++ b/kpops/components/streams_bootstrap/producer/model.py
@@ -4,6 +4,7 @@
KafkaAppValues,
KafkaStreamsConfig,
)
+from kpops.components.streams_bootstrap import StreamsBootstrapValues
from kpops.utils.docstring import describe_attr
@@ -11,7 +12,7 @@ class ProducerStreamsConfig(KafkaStreamsConfig):
"""Kafka Streams settings specific to Producer."""
-class ProducerAppValues(KafkaAppValues):
+class ProducerAppValues(StreamsBootstrapValues, KafkaAppValues):
"""Settings specific to producers.
:param streams: Kafka Streams settings
diff --git a/kpops/components/streams_bootstrap/streams/model.py b/kpops/components/streams_bootstrap/streams/model.py
index 04f95b54b..1bffb84c6 100644
--- a/kpops/components/streams_bootstrap/streams/model.py
+++ b/kpops/components/streams_bootstrap/streams/model.py
@@ -11,6 +11,7 @@
KafkaStreamsConfig,
)
from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr
+from kpops.components.streams_bootstrap import StreamsBootstrapValues
from kpops.utils.docstring import describe_attr
from kpops.utils.pydantic import (
CamelCaseConfigModel,
@@ -237,7 +238,7 @@ def validate_mandatory_fields_are_set(
return self
-class StreamsAppValues(KafkaAppValues):
+class StreamsAppValues(StreamsBootstrapValues, KafkaAppValues):
"""streams-bootstrap app configurations.
The attributes correspond to keys and values that are used as values for the streams bootstrap helm chart.
diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py
index caef4a904..a82fca8a9 100644
--- a/tests/components/test_streams_bootstrap.py
+++ b/tests/components/test_streams_bootstrap.py
@@ -48,7 +48,6 @@ def test_default_configs(self, config: KpopsConfig, handlers: ComponentHandlers)
)
assert streams_bootstrap.version == "2.9.0"
assert streams_bootstrap.namespace == "test-namespace"
- assert streams_bootstrap.app.image_tag == "latest"
@pytest.mark.asyncio()
async def test_should_deploy_streams_bootstrap_app(
diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
index bc1517900..8559e1534 100644
--- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
@@ -430,6 +430,5 @@
value.converter.schema.registry.url: http://k8kafka-cp-schema-registry.${NAMESPACE}.svc.cluster.local:8081
name: postgresql-connector
prefix: atm-fraud-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
type: kafka-sink-connector
diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
index 1f6571865..a10c5b5a3 100644
--- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
@@ -165,6 +165,5 @@
value.converter: org.apache.kafka.connect.storage.StringConverter
name: redis-sink-connector
prefix: word-count-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
type: kafka-sink-connector
diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
index 8d5be21bf..5ead506c6 100644
--- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
@@ -283,8 +283,7 @@
transforms.changeTopic.replacement: resources-pipeline-with-inflate-should-inflate-index-v1
name: should-inflate-inflated-sink-connector
prefix: resources-pipeline-with-inflate-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
to:
models: {}
topics:
diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
index 7a6417d01..180b80ec0 100644
--- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
@@ -100,7 +100,6 @@
topics: example-output
name: es-sink-connector
prefix: resources-kafka-connect-sink-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
type: kafka-sink-connector
diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
index 0ad5a8dd7..63fb42b8e 100644
--- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
@@ -200,8 +200,7 @@
transforms.changeTopic.replacement: resources-read-from-component-inflate-step-index-v1
name: inflate-step-inflated-sink-connector
prefix: resources-read-from-component-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
to:
models: {}
topics:
@@ -396,8 +395,7 @@
transforms.changeTopic.replacement: resources-read-from-component-inflate-step-without-prefix-index-v1
name: inflate-step-without-prefix-inflated-sink-connector
prefix: resources-read-from-component-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
to:
models: {}
topics:
diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
index 670ee9942..edb1d4623 100644
--- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
@@ -100,7 +100,6 @@
topics: example-output
name: es-sink-connector
prefix: resources-pipeline-with-env-defaults-
- resetter_values:
- imageTag: latest
+ resetter_values: {}
type: kafka-sink-connector
From e3ea6af05b8b895fe05c96ffaf2d7b97b23b9ddd Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 14:54:48 +0200
Subject: [PATCH 08/25] Update files
---
docs/docs/schema/defaults.json | 2 +-
kpops/components/streams_bootstrap/__init__.py | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index b6644fa31..69ca8ffa7 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -1341,7 +1341,7 @@
"$ref": "#/$defs/StreamsBootstrapValues"
}
],
- "description": ""
+ "description": "Streams bootstrap app values"
},
"from": {
"anyOf": [
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 2501643ff..daef43cce 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -30,6 +30,7 @@ class StreamsBootstrapValues(HelmAppValues):
class StreamsBootstrap(HelmApp, ABC):
"""Base for components with a streams-bootstrap Helm chart.
+ :param app: Streams bootstrap app values
:param repo_config: Configuration of the Helm chart repo to be used for
deploying the component, defaults to streams-bootstrap Helm repo
:param version: Helm chart version, defaults to "2.9.0"
From 8b5a5a713ac4a862c8589bacc9e0001d5766d359 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 14:56:32 +0200
Subject: [PATCH 09/25] Update files
---
kpops/components/base_components/kafka_connector.py | 10 ----------
.../pipelines/test-distributed-defaults/defaults.yaml | 2 --
2 files changed, 12 deletions(-)
diff --git a/kpops/components/base_components/kafka_connector.py b/kpops/components/base_components/kafka_connector.py
index 40e9ad1c8..1d9a296b3 100644
--- a/kpops/components/base_components/kafka_connector.py
+++ b/kpops/components/base_components/kafka_connector.py
@@ -134,16 +134,6 @@ def connector_config_should_have_component_name(
app["name"] = component_name
return KafkaConnectorConfig(**app)
- # @pydantic.model_validator(mode="after")
- # def warning_for_latest_image_tag(self) -> Self:
- # resetter_image_tag = self.resetter_values.get("imageTag")
- # if not resetter_image_tag or resetter_image_tag == "latest":
- # log.warning(
- # f"The imageTag for the Kafka Connect resetter in component '{self.name}' is not set and defaults to 'latest'. "
- # f"Please, consider providing a stable imageTag."
- # )
- # return self
-
@cached_property
def _resetter(self) -> KafkaConnectorResetter:
kwargs: dict[str, Any] = {}
diff --git a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml
index d835cc16f..567c831f3 100644
--- a/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml
+++ b/tests/components/resources/pipelines/test-distributed-defaults/defaults.yaml
@@ -1,5 +1,3 @@
helm-app:
name: ${component.type}
namespace: example-namespace
- app:
- imageTag: latest
From 22923463a7b8b9910f14b657905f132de8bb32aa Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 14:57:30 +0200
Subject: [PATCH 10/25] Update files
---
kpops/component_handlers/kafka_connect/model.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py
index c4aeafa31..840df06c3 100644
--- a/kpops/component_handlers/kafka_connect/model.py
+++ b/kpops/component_handlers/kafka_connect/model.py
@@ -5,7 +5,6 @@
from pydantic import (
BaseModel,
ConfigDict,
- Field,
SerializationInfo,
field_validator,
model_serializer,
@@ -124,4 +123,3 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel):
class KafkaConnectorResetterValues(HelmAppValues):
connector_type: Literal["source", "sink"]
config: KafkaConnectorResetterConfig
- image_tag: str = Field(default="latest", alias="imageTag")
From 37743f38d352343b335a1114cf5149ba156b0cd9 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 14:58:31 +0200
Subject: [PATCH 11/25] Update files
---
kpops/components/streams_bootstrap/__init__.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index daef43cce..223fee9e6 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -52,8 +52,11 @@ class StreamsBootstrap(HelmApp, ABC):
@pydantic.model_validator(mode="after")
def warning_for_latest_image_tag(self) -> Self:
- if self.app.image_tag == "latest" and "$" not in self.name:
+ if self.app.image_tag == "latest" and self.is_substituted():
log.warning(
f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag."
)
return self
+
+ def is_substituted(self):
+ return "$" not in self.name
From 8fa198bec7ae710667274819e8aca4a51c3f6758 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 15:01:26 +0200
Subject: [PATCH 12/25] Update files
---
kpops/component_handlers/kafka_connect/model.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py
index 840df06c3..005d7c429 100644
--- a/kpops/component_handlers/kafka_connect/model.py
+++ b/kpops/component_handlers/kafka_connect/model.py
@@ -5,6 +5,7 @@
from pydantic import (
BaseModel,
ConfigDict,
+ Field,
SerializationInfo,
field_validator,
model_serializer,
@@ -123,3 +124,4 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel):
class KafkaConnectorResetterValues(HelmAppValues):
connector_type: Literal["source", "sink"]
config: KafkaConnectorResetterConfig
+ image_tag: str = Field(default="latest")
From ce461b9a2a0c398b6ff3bd71dbcf3436ecc6bb15 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 15:55:29 +0200
Subject: [PATCH 13/25] add image tag vaildation
---
.../component_handlers/kafka_connect/model.py | 6 +++++
kpops/component_handlers/kubernetes/utils.py | 27 +++++++++++++++++++
.../components/streams_bootstrap/__init__.py | 8 +++++-
.../kubernetes/test_utils.py | 20 ++++++++++++++
.../resources/resetter_values/defaults.yaml | 2 ++
tests/pipeline/test_generate.py | 8 ++++++
6 files changed, 70 insertions(+), 1 deletion(-)
create mode 100644 tests/component_handlers/kubernetes/test_utils.py
diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py
index 005d7c429..f09a0312c 100644
--- a/kpops/component_handlers/kafka_connect/model.py
+++ b/kpops/component_handlers/kafka_connect/model.py
@@ -13,6 +13,7 @@
from pydantic.json_schema import SkipJsonSchema
from typing_extensions import override
+from kpops.component_handlers.kubernetes.utils import validate_image_tag
from kpops.components.base_components.helm_app import HelmAppValues
from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr
from kpops.utils.pydantic import (
@@ -125,3 +126,8 @@ class KafkaConnectorResetterValues(HelmAppValues):
connector_type: Literal["source", "sink"]
config: KafkaConnectorResetterConfig
image_tag: str = Field(default="latest")
+
+ @pydantic.field_validator("image_tag", mode="before")
+ @classmethod
+ def validate_image_tag_field(cls, image_tag: Any) -> str:
+ return validate_image_tag(image_tag)
diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py
index 4f4599e2b..a9f185162 100644
--- a/kpops/component_handlers/kubernetes/utils.py
+++ b/kpops/component_handlers/kubernetes/utils.py
@@ -1,5 +1,8 @@
import hashlib
import logging
+import re
+
+from kpops.api.exception import ValidationError
log = logging.getLogger("K8sUtils")
@@ -26,3 +29,27 @@ def trim(max_len: int, name: str, suffix: str) -> str:
)
return new_name
return name
+
+
+def validate_image_tag(image_tag: str) -> str:
+ """Validate an image tag.
+
+ Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-).
+ It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}
+
+ :param image_tag: Docker image tag to be validated.
+ :return: The validated image tag.
+ """
+ if isinstance(image_tag, str) and is_valid_image_tag(image_tag):
+ return image_tag
+ msg = (
+ "Image tag is not valid. "
+ "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). "
+ "It can be up to 128 characters long."
+ )
+ raise ValidationError(msg)
+
+
+def is_valid_image_tag(image_tag: str) -> bool:
+ pattern = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$"
+ return bool(re.match(pattern, image_tag))
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 223fee9e6..f026a77dc 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -1,11 +1,12 @@
import logging
from abc import ABC
-from typing import Self
+from typing import Any, Self
import pydantic
from pydantic import Field
from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig
+from kpops.component_handlers.kubernetes.utils import validate_image_tag
from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
@@ -26,6 +27,11 @@ class StreamsBootstrapValues(HelmAppValues):
image_tag: str = Field(default="latest")
+ @pydantic.field_validator("image_tag", mode="before")
+ @classmethod
+ def validate_image_tag_field(cls, image_tag: Any) -> str:
+ return validate_image_tag(image_tag)
+
class StreamsBootstrap(HelmApp, ABC):
"""Base for components with a streams-bootstrap Helm chart.
diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py
new file mode 100644
index 000000000..0c1bdc3f1
--- /dev/null
+++ b/tests/component_handlers/kubernetes/test_utils.py
@@ -0,0 +1,20 @@
+import pytest
+
+from kpops.api.exception import ValidationError
+from kpops.component_handlers.kubernetes.utils import validate_image_tag
+
+
+def test_is_valid_image_tag():
+ assert validate_image_tag("1.2.3") == "1.2.3"
+ assert validate_image_tag("123") == "123"
+ assert validate_image_tag("1_2_3") == "1_2_3"
+ assert validate_image_tag("1-2-3") == "1-2-3"
+ assert validate_image_tag("latest") == "latest"
+ assert (
+ validate_image_tag(
+ "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07"
+ )
+ == "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07"
+ )
+ with pytest.raises(ValidationError):
+ assert validate_image_tag("la!est") is False
diff --git a/tests/pipeline/resources/resetter_values/defaults.yaml b/tests/pipeline/resources/resetter_values/defaults.yaml
index 550c9c729..950ed4969 100644
--- a/tests/pipeline/resources/resetter_values/defaults.yaml
+++ b/tests/pipeline/resources/resetter_values/defaults.yaml
@@ -9,3 +9,5 @@ helm-app:
kafka-sink-connector:
app:
"connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector"
+ resetter_values:
+ imageTag: override-default-image-tag
diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py
index 915ae17d7..71d8946e1 100644
--- a/tests/pipeline/test_generate.py
+++ b/tests/pipeline/test_generate.py
@@ -840,6 +840,10 @@ def test_substitution_in_inflated_component(self):
enriched_pipeline[1]["_resetter"]["app"]["label"]
== "inflated-connector-name"
)
+ assert (
+ enriched_pipeline[1]["_resetter"]["app"]["imageTag"]
+ == "override-default-image-tag"
+ )
def test_substitution_in_resetter(self):
pipeline = kpops.generate(
@@ -857,3 +861,7 @@ def test_substitution_in_resetter(self):
assert enriched_pipeline[0]["name"] == "es-sink-connector"
assert enriched_pipeline[0]["_resetter"]["name"] == "es-sink-connector"
assert enriched_pipeline[0]["_resetter"]["app"]["label"] == "es-sink-connector"
+ assert (
+ enriched_pipeline[1]["_resetter"]["app"]["imageTag"]
+ == "override-default-image-tag"
+ )
From 3b4830358f2ecb16df407100832bf057f40eeaea Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Wed, 26 Jun 2024 16:00:04 +0200
Subject: [PATCH 14/25] Update files
---
tests/pipeline/test_generate.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py
index 71d8946e1..ef68aadf9 100644
--- a/tests/pipeline/test_generate.py
+++ b/tests/pipeline/test_generate.py
@@ -862,6 +862,6 @@ def test_substitution_in_resetter(self):
assert enriched_pipeline[0]["_resetter"]["name"] == "es-sink-connector"
assert enriched_pipeline[0]["_resetter"]["app"]["label"] == "es-sink-connector"
assert (
- enriched_pipeline[1]["_resetter"]["app"]["imageTag"]
+ enriched_pipeline[0]["_resetter"]["app"]["imageTag"]
== "override-default-image-tag"
)
From 9ffdce69ddbc993bed083948b602f0396b63ad3f Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 12:34:32 +0200
Subject: [PATCH 15/25] Update files
---
docs/docs/schema/defaults.json | 3 +++
docs/docs/schema/pipeline.json | 2 ++
kpops/components/streams_bootstrap/__init__.py | 11 +++++------
3 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index 69ca8ffa7..2c08e0089 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -922,6 +922,7 @@
"properties": {
"imageTag": {
"default": "latest",
+ "description": "Docker image tag of the Kafka Streams app.",
"title": "Imagetag",
"type": "string"
},
@@ -1280,6 +1281,7 @@
},
"imageTag": {
"default": "latest",
+ "description": "Docker image tag of the Kafka Streams app.",
"title": "Imagetag",
"type": "string"
},
@@ -1430,6 +1432,7 @@
"properties": {
"imageTag": {
"default": "latest",
+ "description": "Docker image tag of the Kafka Streams app.",
"title": "Imagetag",
"type": "string"
},
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index 0d98c4e7e..ba40f0292 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -590,6 +590,7 @@
"properties": {
"imageTag": {
"default": "latest",
+ "description": "Docker image tag of the Kafka Streams app.",
"title": "Imagetag",
"type": "string"
},
@@ -948,6 +949,7 @@
},
"imageTag": {
"default": "latest",
+ "description": "Docker image tag of the Kafka Streams app.",
"title": "Imagetag",
"type": "string"
},
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index f026a77dc..a28a82b59 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -25,7 +25,9 @@ class StreamsBootstrapValues(HelmAppValues):
:param image_tag: Docker image tag of the Kafka Streams app.
"""
- image_tag: str = Field(default="latest")
+ image_tag: str = Field(
+ default="latest", description=describe_attr("image_tag", __doc__)
+ )
@pydantic.field_validator("image_tag", mode="before")
@classmethod
@@ -58,11 +60,8 @@ class StreamsBootstrap(HelmApp, ABC):
@pydantic.model_validator(mode="after")
def warning_for_latest_image_tag(self) -> Self:
- if self.app.image_tag == "latest" and self.is_substituted():
+ if self.validate_ and self.app.image_tag == "latest":
log.warning(
- f"The imageTag for component '{self.name}' is not set and defaults to 'latest'. Please, consider providing a stable imageTag."
+ f"The imageTag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable imageTag."
)
return self
-
- def is_substituted(self):
- return "$" not in self.name
From b494f051298cb828bd700c402dd12036043d9e8d Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 12:41:48 +0200
Subject: [PATCH 16/25] Update files
---
kpops/components/streams_bootstrap/__init__.py | 2 +-
tests/components/test_streams_bootstrap.py | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index a28a82b59..da24e629d 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -62,6 +62,6 @@ class StreamsBootstrap(HelmApp, ABC):
def warning_for_latest_image_tag(self) -> Self:
if self.validate_ and self.app.image_tag == "latest":
log.warning(
- f"The imageTag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable imageTag."
+ f"The image tag for component '{self.name}' is set or defaulted to 'latest'. Please, consider providing a stable image tag."
)
return self
diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py
index a82fca8a9..3a153fbde 100644
--- a/tests/components/test_streams_bootstrap.py
+++ b/tests/components/test_streams_bootstrap.py
@@ -48,6 +48,7 @@ def test_default_configs(self, config: KpopsConfig, handlers: ComponentHandlers)
)
assert streams_bootstrap.version == "2.9.0"
assert streams_bootstrap.namespace == "test-namespace"
+ assert streams_bootstrap.app.image_tag == "latest"
@pytest.mark.asyncio()
async def test_should_deploy_streams_bootstrap_app(
@@ -63,6 +64,7 @@ async def test_should_deploy_streams_bootstrap_app(
**{
"namespace": "test-namespace",
"app": {
+ "imageTag": "1.0.0",
"streams": {
"outputTopic": "test",
"brokers": "fake-broker:9092",
@@ -94,6 +96,7 @@ async def test_should_deploy_streams_bootstrap_app(
"test-namespace",
{
"nameOverride": "${pipeline.name}-example-name",
+ "imageTag": "1.0.0",
"streams": {
"brokers": "fake-broker:9092",
"outputTopic": "test",
From 82b5e809dc3a57e70371fcbef6d264800a2466f8 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 12:44:54 +0200
Subject: [PATCH 17/25] Update files
---
kpops/components/streams_bootstrap/__init__.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index da24e629d..c11f7557e 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -1,7 +1,11 @@
import logging
from abc import ABC
-from typing import Any, Self
+from typing import Any
+try:
+ from typing import Self # pyright: ignore[reportAttributeAccessIssue]
+except ImportError:
+ from typing_extensions import Self
import pydantic
from pydantic import Field
From 947a22b7e7a608166d6f69805c065040ef2c63b0 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 12:45:20 +0200
Subject: [PATCH 18/25] Update files
---
kpops/components/streams_bootstrap/__init__.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index c11f7557e..24d951c89 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -2,10 +2,6 @@
from abc import ABC
from typing import Any
-try:
- from typing import Self # pyright: ignore[reportAttributeAccessIssue]
-except ImportError:
- from typing_extensions import Self
import pydantic
from pydantic import Field
@@ -14,6 +10,12 @@
from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
+try:
+ from typing import Self # pyright: ignore[reportAttributeAccessIssue]
+except ImportError:
+ from typing_extensions import Self
+
+
STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig(
repository_name="bakdata-streams-bootstrap",
url="https://bakdata.github.io/streams-bootstrap/",
From 6e51a5fe5adcaa53a2c41c9ddf434aee68fed5da Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 15:18:18 +0200
Subject: [PATCH 19/25] Update files
---
docs/docs/schema/defaults.json | 6 +--
docs/docs/schema/pipeline.json | 4 +-
kpops/api/exception.py | 5 ++
.../component_handlers/kafka_connect/model.py | 3 +-
kpops/component_handlers/kubernetes/utils.py | 25 +++++-----
.../components/streams_bootstrap/__init__.py | 18 ++++---
.../kubernetes/test_utils.py | 47 +++++++++++++------
7 files changed, 68 insertions(+), 40 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index 2c08e0089..c66ef3f38 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -922,7 +922,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Docker image tag of the Kafka Streams app.",
+ "description": "Docker image tag of the streams-bootstrap app.",
"title": "Imagetag",
"type": "string"
},
@@ -1281,7 +1281,7 @@
},
"imageTag": {
"default": "latest",
- "description": "Docker image tag of the Kafka Streams app.",
+ "description": "Docker image tag of the streams-bootstrap app.",
"title": "Imagetag",
"type": "string"
},
@@ -1432,7 +1432,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Docker image tag of the Kafka Streams app.",
+ "description": "Docker image tag of the streams-bootstrap app.",
"title": "Imagetag",
"type": "string"
},
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index ba40f0292..8e5f17354 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -590,7 +590,7 @@
"properties": {
"imageTag": {
"default": "latest",
- "description": "Docker image tag of the Kafka Streams app.",
+ "description": "Docker image tag of the streams-bootstrap app.",
"title": "Imagetag",
"type": "string"
},
@@ -949,7 +949,7 @@
},
"imageTag": {
"default": "latest",
- "description": "Docker image tag of the Kafka Streams app.",
+ "description": "Docker image tag of the streams-bootstrap app.",
"title": "Imagetag",
"type": "string"
},
diff --git a/kpops/api/exception.py b/kpops/api/exception.py
index 65094fd29..593bdcd5c 100644
--- a/kpops/api/exception.py
+++ b/kpops/api/exception.py
@@ -11,3 +11,8 @@ class ParsingException(Exception):
class ClassNotFoundError(Exception):
"""Similar to builtin `ModuleNotFoundError`; class doesn't exist inside module."""
+
+
+class InvalidImageTagError(ValidationError):
+ def __init__(self, *args):
+ super().__init__(*args)
diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py
index f09a0312c..1af6b51b5 100644
--- a/kpops/component_handlers/kafka_connect/model.py
+++ b/kpops/component_handlers/kafka_connect/model.py
@@ -130,4 +130,5 @@ class KafkaConnectorResetterValues(HelmAppValues):
@pydantic.field_validator("image_tag", mode="before")
@classmethod
def validate_image_tag_field(cls, image_tag: Any) -> str:
- return validate_image_tag(image_tag)
+ validate_image_tag(image_tag)
+ return image_tag
diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py
index a9f185162..1ea5daf67 100644
--- a/kpops/component_handlers/kubernetes/utils.py
+++ b/kpops/component_handlers/kubernetes/utils.py
@@ -1,11 +1,14 @@
import hashlib
import logging
import re
+from typing import Any
from kpops.api.exception import ValidationError
log = logging.getLogger("K8sUtils")
+IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$"
+
def trim(max_len: int, name: str, suffix: str) -> str:
"""Shortens long K8s identifiers.
@@ -31,25 +34,23 @@ def trim(max_len: int, name: str, suffix: str) -> str:
return name
-def validate_image_tag(image_tag: str) -> str:
+def validate_image_tag(image_tag: Any) -> None:
"""Validate an image tag.
Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-).
It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}
:param image_tag: Docker image tag to be validated.
- :return: The validated image tag.
"""
- if isinstance(image_tag, str) and is_valid_image_tag(image_tag):
- return image_tag
- msg = (
- "Image tag is not valid. "
- "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). "
- "It can be up to 128 characters long."
- )
- raise ValidationError(msg)
+ if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)):
+ msg = (
+ "Image tag is not valid. "
+ "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). "
+ "It can be up to 128 characters long."
+ )
+ raise ValidationError(msg)
def is_valid_image_tag(image_tag: str) -> bool:
- pattern = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$"
- return bool(re.match(pattern, image_tag))
+ """Check if the image tag is valid according to the specified regex pattern."""
+ return bool(re.match(IMAGE_TAG_PATTERN, image_tag))
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 24d951c89..3bb4ba59e 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -1,6 +1,8 @@
+from __future__ import annotations
+
import logging
from abc import ABC
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pydantic
from pydantic import Field
@@ -10,10 +12,11 @@
from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
-try:
- from typing import Self # pyright: ignore[reportAttributeAccessIssue]
-except ImportError:
- from typing_extensions import Self
+if TYPE_CHECKING:
+ try:
+ from typing import Self # pyright: ignore[reportAttributeAccessIssue]
+ except ImportError:
+ from typing_extensions import Self
STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig(
@@ -28,7 +31,7 @@
class StreamsBootstrapValues(HelmAppValues):
"""Base value class for all streams bootstrap related components.
- :param image_tag: Docker image tag of the Kafka Streams app.
+ :param image_tag: Docker image tag of the streams-bootstrap app.
"""
image_tag: str = Field(
@@ -38,7 +41,8 @@ class StreamsBootstrapValues(HelmAppValues):
@pydantic.field_validator("image_tag", mode="before")
@classmethod
def validate_image_tag_field(cls, image_tag: Any) -> str:
- return validate_image_tag(image_tag)
+ validate_image_tag(image_tag)
+ return image_tag
class StreamsBootstrap(HelmApp, ABC):
diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py
index 0c1bdc3f1..49885016b 100644
--- a/tests/component_handlers/kubernetes/test_utils.py
+++ b/tests/component_handlers/kubernetes/test_utils.py
@@ -1,20 +1,37 @@
import pytest
from kpops.api.exception import ValidationError
-from kpops.component_handlers.kubernetes.utils import validate_image_tag
+from kpops.component_handlers.kubernetes.utils import (
+ validate_image_tag,
+)
+MAX_LENGTH = 128
-def test_is_valid_image_tag():
- assert validate_image_tag("1.2.3") == "1.2.3"
- assert validate_image_tag("123") == "123"
- assert validate_image_tag("1_2_3") == "1_2_3"
- assert validate_image_tag("1-2-3") == "1-2-3"
- assert validate_image_tag("latest") == "latest"
- assert (
- validate_image_tag(
- "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07"
- )
- == "1ff6c18fbef2045af6b9c16bf034cc421a29027b800e4f9b68ae9b1cb3e9ae07"
- )
- with pytest.raises(ValidationError):
- assert validate_image_tag("la!est") is False
+
+def test_validate_image_tag_valid():
+ valid_tags = [
+ "valid-tag",
+ "VALID_TAG",
+ "valid.tag",
+ "valid-tag_123",
+ "v" * MAX_LENGTH,
+ ]
+ for tag in valid_tags:
+ validate_image_tag(tag)
+
+
+def test_if_image_tag_is_invalid():
+ invalid_tags = [
+ "invalid tag!",
+ "",
+ " " * (MAX_LENGTH + 1),
+ "a" * (MAX_LENGTH + 1),
+ "@invalid",
+ None,
+ 123,
+ {},
+ [],
+ ]
+ for tag in invalid_tags:
+ with pytest.raises(ValidationError):
+ validate_image_tag(tag)
From be5e1b009dfca26008f3018b954f57a5d9f26658 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 16:08:57 +0200
Subject: [PATCH 20/25] Remove image tag from connect
---
kpops/api/exception.py | 5 -----
.../component_handlers/kafka_connect/model.py | 9 ---------
kpops/component_handlers/kubernetes/utils.py | 20 -------------------
.../components/streams_bootstrap/__init__.py | 12 +++++++++--
.../kubernetes/test_utils.py | 16 +++------------
.../test_generate/atm-fraud/pipeline.yaml | 1 -
.../test_generate/word-count/pipeline.yaml | 1 -
.../test_inflate_pipeline/pipeline.yaml | 1 -
.../pipeline.yaml | 1 -
.../test_read_from_component/pipeline.yaml | 2 --
.../test_with_env_defaults/pipeline.yaml | 1 -
11 files changed, 13 insertions(+), 56 deletions(-)
diff --git a/kpops/api/exception.py b/kpops/api/exception.py
index 593bdcd5c..65094fd29 100644
--- a/kpops/api/exception.py
+++ b/kpops/api/exception.py
@@ -11,8 +11,3 @@ class ParsingException(Exception):
class ClassNotFoundError(Exception):
"""Similar to builtin `ModuleNotFoundError`; class doesn't exist inside module."""
-
-
-class InvalidImageTagError(ValidationError):
- def __init__(self, *args):
- super().__init__(*args)
diff --git a/kpops/component_handlers/kafka_connect/model.py b/kpops/component_handlers/kafka_connect/model.py
index 1af6b51b5..840df06c3 100644
--- a/kpops/component_handlers/kafka_connect/model.py
+++ b/kpops/component_handlers/kafka_connect/model.py
@@ -5,7 +5,6 @@
from pydantic import (
BaseModel,
ConfigDict,
- Field,
SerializationInfo,
field_validator,
model_serializer,
@@ -13,7 +12,6 @@
from pydantic.json_schema import SkipJsonSchema
from typing_extensions import override
-from kpops.component_handlers.kubernetes.utils import validate_image_tag
from kpops.components.base_components.helm_app import HelmAppValues
from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr
from kpops.utils.pydantic import (
@@ -125,10 +123,3 @@ class KafkaConnectorResetterConfig(CamelCaseConfigModel):
class KafkaConnectorResetterValues(HelmAppValues):
connector_type: Literal["source", "sink"]
config: KafkaConnectorResetterConfig
- image_tag: str = Field(default="latest")
-
- @pydantic.field_validator("image_tag", mode="before")
- @classmethod
- def validate_image_tag_field(cls, image_tag: Any) -> str:
- validate_image_tag(image_tag)
- return image_tag
diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py
index 1ea5daf67..5b17dc963 100644
--- a/kpops/component_handlers/kubernetes/utils.py
+++ b/kpops/component_handlers/kubernetes/utils.py
@@ -1,9 +1,6 @@
import hashlib
import logging
import re
-from typing import Any
-
-from kpops.api.exception import ValidationError
log = logging.getLogger("K8sUtils")
@@ -34,23 +31,6 @@ def trim(max_len: int, name: str, suffix: str) -> str:
return name
-def validate_image_tag(image_tag: Any) -> None:
- """Validate an image tag.
-
- Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-).
- It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}
-
- :param image_tag: Docker image tag to be validated.
- """
- if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)):
- msg = (
- "Image tag is not valid. "
- "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). "
- "It can be up to 128 characters long."
- )
- raise ValidationError(msg)
-
-
def is_valid_image_tag(image_tag: str) -> bool:
"""Check if the image tag is valid according to the specified regex pattern."""
return bool(re.match(IMAGE_TAG_PATTERN, image_tag))
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 3bb4ba59e..fff1e4f1b 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -8,7 +8,9 @@
from pydantic import Field
from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig
-from kpops.component_handlers.kubernetes.utils import validate_image_tag
+from kpops.component_handlers.kubernetes.utils import (
+ is_valid_image_tag,
+)
from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
@@ -41,7 +43,13 @@ class StreamsBootstrapValues(HelmAppValues):
@pydantic.field_validator("image_tag", mode="before")
@classmethod
def validate_image_tag_field(cls, image_tag: Any) -> str:
- validate_image_tag(image_tag)
+ if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)):
+ msg = (
+ "Image tag is not valid. "
+ "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). "
+ "It can be up to 128 characters long."
+ )
+ raise pydantic.ValidationError(msg)
return image_tag
diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py
index 49885016b..2648c9a16 100644
--- a/tests/component_handlers/kubernetes/test_utils.py
+++ b/tests/component_handlers/kubernetes/test_utils.py
@@ -1,9 +1,4 @@
-import pytest
-
-from kpops.api.exception import ValidationError
-from kpops.component_handlers.kubernetes.utils import (
- validate_image_tag,
-)
+from kpops.component_handlers.kubernetes.utils import is_valid_image_tag
MAX_LENGTH = 128
@@ -17,7 +12,7 @@ def test_validate_image_tag_valid():
"v" * MAX_LENGTH,
]
for tag in valid_tags:
- validate_image_tag(tag)
+ assert is_valid_image_tag(tag) is True
def test_if_image_tag_is_invalid():
@@ -27,11 +22,6 @@ def test_if_image_tag_is_invalid():
" " * (MAX_LENGTH + 1),
"a" * (MAX_LENGTH + 1),
"@invalid",
- None,
- 123,
- {},
- [],
]
for tag in invalid_tags:
- with pytest.raises(ValidationError):
- validate_image_tag(tag)
+ assert is_valid_image_tag(tag) is False
diff --git a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
index 8559e1534..528da26bb 100644
--- a/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_example/test_generate/atm-fraud/pipeline.yaml
@@ -393,7 +393,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: atm-fraud-postgresql-connector
connectorType: sink
- imageTag: latest
name: postgresql-connector
namespace: ${NAMESPACE}
prefix: atm-fraud-
diff --git a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
index a10c5b5a3..7e7d16fe8 100644
--- a/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_example/test_generate/word-count/pipeline.yaml
@@ -142,7 +142,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: word-count-redis-sink-connector
connectorType: sink
- imageTag: latest
name: redis-sink-connector
namespace: ${NAMESPACE}
prefix: word-count-
diff --git a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
index 5ead506c6..1040139c3 100644
--- a/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_inflate_pipeline/pipeline.yaml
@@ -255,7 +255,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-pipeline-with-inflate-should-inflate-inflated-sink-connector
connectorType: sink
- imageTag: latest
name: should-inflate-inflated-sink-connector
namespace: example-namespace
prefix: resources-pipeline-with-inflate-
diff --git a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
index 180b80ec0..e1dd399cd 100644
--- a/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_kafka_connect_sink_weave_from_topics/pipeline.yaml
@@ -73,7 +73,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-kafka-connect-sink-es-sink-connector
connectorType: sink
- imageTag: latest
name: es-sink-connector
namespace: example-namespace
prefix: resources-kafka-connect-sink-
diff --git a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
index 63fb42b8e..ff1dd9e2b 100644
--- a/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_read_from_component/pipeline.yaml
@@ -172,7 +172,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-read-from-component-inflate-step-inflated-sink-connector
connectorType: sink
- imageTag: latest
name: inflate-step-inflated-sink-connector
namespace: example-namespace
prefix: resources-read-from-component-
@@ -367,7 +366,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-read-from-component-inflate-step-without-prefix-inflated-sink-connector
connectorType: sink
- imageTag: latest
name: inflate-step-without-prefix-inflated-sink-connector
namespace: example-namespace
prefix: resources-read-from-component-
diff --git a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
index edb1d4623..a37ad0f3d 100644
--- a/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
+++ b/tests/pipeline/snapshots/test_generate/test_with_env_defaults/pipeline.yaml
@@ -73,7 +73,6 @@
brokers: http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092
connector: resources-pipeline-with-env-defaults-es-sink-connector
connectorType: sink
- imageTag: latest
name: es-sink-connector
namespace: development-namespace
prefix: resources-pipeline-with-env-defaults-
From e1fcdc7d27214998722ec330c63b89599173d671 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 16:11:49 +0200
Subject: [PATCH 21/25] Update files
---
kpops/component_handlers/kubernetes/utils.py | 6 ++++-
.../kubernetes/test_utils.py | 26 ++++++++++++-------
2 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py
index 5b17dc963..c50b0b617 100644
--- a/kpops/component_handlers/kubernetes/utils.py
+++ b/kpops/component_handlers/kubernetes/utils.py
@@ -32,5 +32,9 @@ def trim(max_len: int, name: str, suffix: str) -> str:
def is_valid_image_tag(image_tag: str) -> bool:
- """Check if the image tag is valid according to the specified regex pattern."""
+ """Check if the image tag is valid according to the specified regex pattern.
+
+ Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-).
+ It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}
+ """
return bool(re.match(IMAGE_TAG_PATTERN, image_tag))
diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py
index 2648c9a16..c440cf83b 100644
--- a/tests/component_handlers/kubernetes/test_utils.py
+++ b/tests/component_handlers/kubernetes/test_utils.py
@@ -1,27 +1,33 @@
+import pytest
+
from kpops.component_handlers.kubernetes.utils import is_valid_image_tag
MAX_LENGTH = 128
-def test_validate_image_tag_valid():
- valid_tags = [
+@pytest.mark.parametrize(
+ "tag",
+ [
"valid-tag",
"VALID_TAG",
"valid.tag",
"valid-tag_123",
"v" * MAX_LENGTH,
- ]
- for tag in valid_tags:
- assert is_valid_image_tag(tag) is True
+ ],
+)
+def test_validate_image_tag_valid(tag):
+ assert is_valid_image_tag(tag) is True
-def test_if_image_tag_is_invalid():
- invalid_tags = [
+@pytest.mark.parametrize(
+ "tag",
+ [
"invalid tag!",
"",
" " * (MAX_LENGTH + 1),
"a" * (MAX_LENGTH + 1),
"@invalid",
- ]
- for tag in invalid_tags:
- assert is_valid_image_tag(tag) is False
+ ],
+)
+def test_if_image_tag_is_invalid(tag):
+ assert is_valid_image_tag(tag) is False
From 47d72443d0b843848986978facff1134c3ac0040 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 16:13:13 +0200
Subject: [PATCH 22/25] add type to test
---
tests/component_handlers/kubernetes/test_utils.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py
index c440cf83b..982a070e3 100644
--- a/tests/component_handlers/kubernetes/test_utils.py
+++ b/tests/component_handlers/kubernetes/test_utils.py
@@ -15,7 +15,7 @@
"v" * MAX_LENGTH,
],
)
-def test_validate_image_tag_valid(tag):
+def test_validate_image_tag_valid(tag: str):
assert is_valid_image_tag(tag) is True
@@ -29,5 +29,5 @@ def test_validate_image_tag_valid(tag):
"@invalid",
],
)
-def test_if_image_tag_is_invalid(tag):
+def test_if_image_tag_is_invalid(tag: str):
assert is_valid_image_tag(tag) is False
From 1c3698eabb6f9a8f871d799d7efb2ed11583bac9 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Thu, 4 Jul 2024 16:27:56 +0200
Subject: [PATCH 23/25] Update files
---
docs/docs/schema/defaults.json | 3 ++
docs/docs/schema/pipeline.json | 2 ++
kpops/component_handlers/kubernetes/utils.py | 11 +------
.../components/streams_bootstrap/__init__.py | 21 +++---------
.../kubernetes/test_utils.py | 33 -------------------
5 files changed, 11 insertions(+), 59 deletions(-)
delete mode 100644 tests/component_handlers/kubernetes/test_utils.py
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index c66ef3f38..d5f479201 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -923,6 +923,7 @@
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
+ "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
@@ -1282,6 +1283,7 @@
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
+ "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
@@ -1433,6 +1435,7 @@
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
+ "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
diff --git a/docs/docs/schema/pipeline.json b/docs/docs/schema/pipeline.json
index 8e5f17354..d8c953c82 100644
--- a/docs/docs/schema/pipeline.json
+++ b/docs/docs/schema/pipeline.json
@@ -591,6 +591,7 @@
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
+ "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
@@ -950,6 +951,7 @@
"imageTag": {
"default": "latest",
"description": "Docker image tag of the streams-bootstrap app.",
+ "pattern": "^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$",
"title": "Imagetag",
"type": "string"
},
diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py
index c50b0b617..598e1a706 100644
--- a/kpops/component_handlers/kubernetes/utils.py
+++ b/kpops/component_handlers/kubernetes/utils.py
@@ -1,9 +1,9 @@
import hashlib
import logging
-import re
log = logging.getLogger("K8sUtils")
+# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names
IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$"
@@ -29,12 +29,3 @@ def trim(max_len: int, name: str, suffix: str) -> str:
)
return new_name
return name
-
-
-def is_valid_image_tag(image_tag: str) -> bool:
- """Check if the image tag is valid according to the specified regex pattern.
-
- Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-).
- It can be up to 128 characters long and must follow the regex pattern: [a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}
- """
- return bool(re.match(IMAGE_TAG_PATTERN, image_tag))
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index fff1e4f1b..7ac93c3f8 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -2,14 +2,14 @@
import logging
from abc import ABC
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
import pydantic
from pydantic import Field
from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig
from kpops.component_handlers.kubernetes.utils import (
- is_valid_image_tag,
+ IMAGE_TAG_PATTERN,
)
from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
@@ -20,7 +20,6 @@
except ImportError:
from typing_extensions import Self
-
STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig(
repository_name="bakdata-streams-bootstrap",
url="https://bakdata.github.io/streams-bootstrap/",
@@ -37,21 +36,11 @@ class StreamsBootstrapValues(HelmAppValues):
"""
image_tag: str = Field(
- default="latest", description=describe_attr("image_tag", __doc__)
+ default="latest",
+ pattern=IMAGE_TAG_PATTERN,
+ description=describe_attr("image_tag", __doc__),
)
- @pydantic.field_validator("image_tag", mode="before")
- @classmethod
- def validate_image_tag_field(cls, image_tag: Any) -> str:
- if not (isinstance(image_tag, str) and is_valid_image_tag(image_tag)):
- msg = (
- "Image tag is not valid. "
- "Image tags consist of lowercase and uppercase letters, digits, underscores (_), periods (.), and dashes (-). "
- "It can be up to 128 characters long."
- )
- raise pydantic.ValidationError(msg)
- return image_tag
-
class StreamsBootstrap(HelmApp, ABC):
"""Base for components with a streams-bootstrap Helm chart.
diff --git a/tests/component_handlers/kubernetes/test_utils.py b/tests/component_handlers/kubernetes/test_utils.py
deleted file mode 100644
index 982a070e3..000000000
--- a/tests/component_handlers/kubernetes/test_utils.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import pytest
-
-from kpops.component_handlers.kubernetes.utils import is_valid_image_tag
-
-MAX_LENGTH = 128
-
-
-@pytest.mark.parametrize(
- "tag",
- [
- "valid-tag",
- "VALID_TAG",
- "valid.tag",
- "valid-tag_123",
- "v" * MAX_LENGTH,
- ],
-)
-def test_validate_image_tag_valid(tag: str):
- assert is_valid_image_tag(tag) is True
-
-
-@pytest.mark.parametrize(
- "tag",
- [
- "invalid tag!",
- "",
- " " * (MAX_LENGTH + 1),
- "a" * (MAX_LENGTH + 1),
- "@invalid",
- ],
-)
-def test_if_image_tag_is_invalid(tag: str):
- assert is_valid_image_tag(tag) is False
From 4cd941be00f998e615a095ab9e45922e31a43232 Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Mon, 8 Jul 2024 13:36:43 +0200
Subject: [PATCH 24/25] Update test and pattern
---
kpops/component_handlers/kubernetes/utils.py | 3 ---
.../components/streams_bootstrap/__init__.py | 6 ++---
tests/components/test_streams_bootstrap.py | 22 ++++++++++++++++++-
3 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/kpops/component_handlers/kubernetes/utils.py b/kpops/component_handlers/kubernetes/utils.py
index 598e1a706..4f4599e2b 100644
--- a/kpops/component_handlers/kubernetes/utils.py
+++ b/kpops/component_handlers/kubernetes/utils.py
@@ -3,9 +3,6 @@
log = logging.getLogger("K8sUtils")
-# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names
-IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$"
-
def trim(max_len: int, name: str, suffix: str) -> str:
"""Shortens long K8s identifiers.
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 7ac93c3f8..88777a157 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -8,9 +8,6 @@
from pydantic import Field
from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig
-from kpops.component_handlers.kubernetes.utils import (
- IMAGE_TAG_PATTERN,
-)
from kpops.components.base_components.helm_app import HelmApp, HelmAppValues
from kpops.utils.docstring import describe_attr
@@ -28,6 +25,9 @@
log = logging.getLogger("StreamsBootstrap")
+# Source of the pattern: https://kubernetes.io/docs/concepts/containers/images/#image-names
+IMAGE_TAG_PATTERN = r"^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$"
+
class StreamsBootstrapValues(HelmAppValues):
"""Base value class for all streams bootstrap related components.
diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py
index 3a153fbde..3ece4612d 100644
--- a/tests/components/test_streams_bootstrap.py
+++ b/tests/components/test_streams_bootstrap.py
@@ -1,6 +1,8 @@
+import re
from unittest.mock import MagicMock
import pytest
+from pydantic import ValidationError
from pytest_mock import MockerFixture
from kpops.component_handlers import ComponentHandlers
@@ -10,7 +12,7 @@
HelmUpgradeInstallFlags,
)
from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name
-from kpops.components.streams_bootstrap import StreamsBootstrap
+from kpops.components.streams_bootstrap import StreamsBootstrap, StreamsBootstrapValues
from kpops.config import KpopsConfig
from tests.components import PIPELINE_BASE_DIR
@@ -104,3 +106,21 @@ async def test_should_deploy_streams_bootstrap_app(
},
HelmUpgradeInstallFlags(version="1.2.3"),
)
+
+ @pytest.mark.asyncio()
+ async def test_should_raise_validation_error_for_invalid_image_tag(
+ self,
+ config: KpopsConfig,
+ handlers: ComponentHandlers,
+ ):
+ with pytest.raises(
+ ValidationError,
+ match=re.escape(
+ "1 validation error for StreamsBootstrapValues\nimageTag\n String should match pattern '^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$'"
+ ),
+ ):
+ StreamsBootstrapValues(
+ **{
+ "imageTag": "invalid image tag!",
+ }
+ )
From bb78dcec5d910920226f974a86b664445a85c68a Mon Sep 17 00:00:00 2001
From: Ramin Gharib <ramin.gharib@bakdata.com>
Date: Mon, 8 Jul 2024 13:38:31 +0200
Subject: [PATCH 25/25] Update files
---
docs/docs/schema/defaults.json | 2 +-
kpops/components/streams_bootstrap/__init__.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/docs/schema/defaults.json b/docs/docs/schema/defaults.json
index d5f479201..77ceed301 100644
--- a/docs/docs/schema/defaults.json
+++ b/docs/docs/schema/defaults.json
@@ -1345,7 +1345,7 @@
"$ref": "#/$defs/StreamsBootstrapValues"
}
],
- "description": "Streams bootstrap app values"
+ "description": "streams-bootstrap app values"
},
"from": {
"anyOf": [
diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py
index 88777a157..c6c329b85 100644
--- a/kpops/components/streams_bootstrap/__init__.py
+++ b/kpops/components/streams_bootstrap/__init__.py
@@ -45,7 +45,7 @@ class StreamsBootstrapValues(HelmAppValues):
class StreamsBootstrap(HelmApp, ABC):
"""Base for components with a streams-bootstrap Helm chart.
- :param app: Streams bootstrap app values
+ :param app: streams-bootstrap app values
:param repo_config: Configuration of the Helm chart repo to be used for
deploying the component, defaults to streams-bootstrap Helm repo
:param version: Helm chart version, defaults to "2.9.0"