From fb59dec055be1e104ae41b55310b38cd8e0b9445 Mon Sep 17 00:00:00 2001 From: "Simon Wu(Infra SH)" Date: Wed, 27 Sep 2023 16:06:37 +0800 Subject: [PATCH] feat(ipv6): add support for IPv6 env --- api/v1beta1/rabbitmqcluster_types.go | 2 + api/v1beta1/zz_generated.deepcopy.go | 1 - .../bases/rabbitmq.com_rabbitmqclusters.yaml | 5 +- controllers/rabbitmqcluster_controller.go | 1 + controllers/reconcile_operator_defaults.go | 12 ++++- docs/api/rabbitmq.com.ref.asciidoc | 1 + internal/resource/statefulset.go | 50 +++++++++++++++++++ internal/status/zz_generated.deepcopy.go | 1 - main.go | 2 + 9 files changed, 70 insertions(+), 5 deletions(-) diff --git a/api/v1beta1/rabbitmqcluster_types.go b/api/v1beta1/rabbitmqcluster_types.go index 980c668ba..dee3902d5 100644 --- a/api/v1beta1/rabbitmqcluster_types.go +++ b/api/v1beta1/rabbitmqcluster_types.go @@ -98,6 +98,8 @@ type RabbitmqClusterSpec struct { // Secret backend configuration for the RabbitmqCluster. // Enables to fetch default user credentials and certificates from K8s external secret stores. SecretBackend SecretBackend `json:"secretBackend,omitempty"` + // IPFamily represents the IP Family (IPv4 or IPv6), default IPv4 + IPFamily string `json:"ipFamily,omitempty"` } // SecretBackend configures a single secret backend. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 1e15e99b3..cd46ec3a1 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* RabbitMQ Cluster Operator diff --git a/config/crd/bases/rabbitmq.com_rabbitmqclusters.yaml b/config/crd/bases/rabbitmq.com_rabbitmqclusters.yaml index e2e76d8d6..86692c605 100644 --- a/config/crd/bases/rabbitmq.com_rabbitmqclusters.yaml +++ b/config/crd/bases/rabbitmq.com_rabbitmqclusters.yaml @@ -10,7 +10,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.12.1 + controller-gen.kubebuilder.io/version: v0.13.0 name: rabbitmqclusters.rabbitmq.com spec: group: rabbitmq.com @@ -544,6 +544,9 @@ spec: type: object x-kubernetes-map-type: atomic type: array + ipFamily: + description: IPFamily represents the IP Family (IPv4 or IPv6), default IPv4 + type: string override: properties: service: diff --git a/controllers/rabbitmqcluster_controller.go b/controllers/rabbitmqcluster_controller.go index f8853c99f..957aeeace 100644 --- a/controllers/rabbitmqcluster_controller.go +++ b/controllers/rabbitmqcluster_controller.go @@ -68,6 +68,7 @@ type RabbitmqClusterReconciler struct { DefaultUserUpdaterImage string DefaultImagePullSecrets string ControlRabbitmqImage bool + DefaultIPFamily string } // the rbac rule requires an empty row at the end to render diff --git a/controllers/reconcile_operator_defaults.go b/controllers/reconcile_operator_defaults.go index 620b58d1c..ddf7757dc 100644 --- a/controllers/reconcile_operator_defaults.go +++ b/controllers/reconcile_operator_defaults.go @@ -3,12 +3,13 @@ package controllers import ( "context" "fmt" + "strings" + "time" + rabbitmqv1beta1 "github.com/rabbitmq/cluster-operator/v2/api/v1beta1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" - "strings" - "time" ) // reconcileOperatorDefaults updates current rabbitmqCluster with operator defaults from the Reconciler @@ -41,6 +42,13 @@ func (r *RabbitmqClusterReconciler) reconcileOperatorDefaults(ctx context.Contex return requeue, err } } + + if rabbitmqCluster.Spec.IPFamily == "" { + rabbitmqCluster.Spec.IPFamily = r.DefaultIPFamily + if requeue, err := r.updateRabbitmqCluster(ctx, rabbitmqCluster, "ipFamily"); err != nil { + return requeue, err + } + } return 0, nil } diff --git a/docs/api/rabbitmq.com.ref.asciidoc b/docs/api/rabbitmq.com.ref.asciidoc index a44afb481..cc7bd20d0 100644 --- a/docs/api/rabbitmq.com.ref.asciidoc +++ b/docs/api/rabbitmq.com.ref.asciidoc @@ -314,6 +314,7 @@ Spec is the desired state of the RabbitmqCluster Custom Resource. | *`terminationGracePeriodSeconds`* __integer__ | TerminationGracePeriodSeconds is the timeout that each rabbitmqcluster pod will have to terminate gracefully. It defaults to 604800 seconds ( a week long) to ensure that the container preStop lifecycle hook can finish running. For more information, see: https://github.com/rabbitmq/cluster-operator/blob/main/docs/design/20200520-graceful-pod-termination.md | *`delayStartSeconds`* __integer__ | DelayStartSeconds is the time the init container (`setup-container`) will sleep before terminating. This effectively delays the time between starting the Pod and starting the `rabbitmq` container. RabbitMQ relies on up-to-date DNS entries early during peer discovery. The purpose of this artificial delay is to ensure that DNS entries are up-to-date when booting RabbitMQ. For more information, see https://github.com/kubernetes/kubernetes/issues/92559 If your Kubernetes DNS backend is configured with a low DNS cache value or publishes not ready addresses promptly, you can decrase this value or set it to 0. | *`secretBackend`* __xref:{anchor_prefix}-jackfan.us.kg-rabbitmq-cluster-operator-v2-api-v1beta1-secretbackend[$$SecretBackend$$]__ | Secret backend configuration for the RabbitmqCluster. Enables to fetch default user credentials and certificates from K8s external secret stores. +| *`ipFamily`* __string__ | IPFamily represents the IP Family (IPv4 or IPv6), default IPv4 |=== diff --git a/internal/resource/statefulset.go b/internal/resource/statefulset.go index d058889cf..e8cabc35a 100644 --- a/internal/resource/statefulset.go +++ b/internal/resource/statefulset.go @@ -41,6 +41,10 @@ const ( DeletionMarker string = "skipPreStopChecks" ) +var ( + ipFamily string = "IPv4" +) + type StatefulSetBuilder struct { *RabbitmqResourceBuilder } @@ -86,6 +90,8 @@ func (builder *StatefulSetBuilder) Build() (client.Object, error) { } + ipFamily = builder.Instance.Spec.IPFamily + return sts, nil } @@ -423,6 +429,15 @@ func (builder *StatefulSetBuilder) podTemplateSpec(previousPodAnnotations map[st }, } + if ipFamily == "IPv6" { + volumes = append(volumes, corev1.Volume{ + Name: "ipv6-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }) + } + if !builder.Instance.VaultDefaultUserSecretEnabled() && !builder.Instance.ExternalSecretEnabled() { appendDefaultUserSecretVolumeProjection(volumes, builder.Instance, "") } else if builder.Instance.ExternalSecretEnabled() { @@ -478,6 +493,13 @@ func (builder *StatefulSetBuilder) podTemplateSpec(previousPodAnnotations map[st }, } + if ipFamily == "IPv6" { + rabbitmqContainerVolumeMounts = append(rabbitmqContainerVolumeMounts, corev1.VolumeMount{ + Name: "ipv6-conf", + MountPath: "/ipv6", + }) + } + if !builder.Instance.VaultDefaultUserSecretEnabled() { rabbitmqContainerVolumeMounts = append(rabbitmqContainerVolumeMounts, corev1.VolumeMount{ Name: "rabbitmq-confd", MountPath: "/etc/rabbitmq/conf.d/11-default_user.conf", SubPath: "default_user.conf", @@ -633,6 +655,19 @@ func (builder *StatefulSetBuilder) podTemplateSpec(previousPodAnnotations map[st }, }, } + + if ipFamily == "IPv6" { + podTemplateSpec.Spec.Containers[0].Env = append(podTemplateSpec.Spec.Containers[0].Env, []corev1.EnvVar{ + { + Name: "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS", + Value: "-kernel inetrc '/ipv6/erl_inetrc' -proto_dist inet6_tcp", + }, + { + Name: "RABBITMQ_CTL_ERL_ARGS", + Value: "-proto_dist inet6_tcp", + }}...) + } + if builder.Instance.VaultDefaultUserSecretEnabled() && builder.Instance.Spec.SecretBackend.Vault.DefaultUserUpdaterImage != nil && *builder.Instance.Spec.SecretBackend.Vault.DefaultUserUpdaterImage != "" { @@ -729,6 +764,7 @@ func setupContainer(instance *rabbitmqv1beta1.RabbitmqCluster) corev1.Container "&& chmod 600 /var/lib/rabbitmq/.erlang.cookie ; " + "cp /tmp/rabbitmq-plugins/enabled_plugins /operator/enabled_plugins ; " + "echo '[default]' > /var/lib/rabbitmq/.rabbitmqadmin.conf " + + getIPv6Cmd(ipFamily) + "&& sed -e 's/default_user/username/' -e 's/default_pass/password/' %s >> /var/lib/rabbitmq/.rabbitmqadmin.conf " + "&& chmod 600 /var/lib/rabbitmq/.rabbitmqadmin.conf ; " + "sleep " + strconv.Itoa(int(pointer.Int32Deref(instance.Spec.DelayStartSeconds, 30))), @@ -771,6 +807,13 @@ func setupContainer(instance *rabbitmqv1beta1.RabbitmqCluster) corev1.Container }, } + if ipFamily == "IPv6" { + setupContainer.VolumeMounts = append(setupContainer.VolumeMounts, corev1.VolumeMount{ + Name: "ipv6-conf", + MountPath: "/ipv6", + }) + } + if instance.VaultDefaultUserSecretEnabled() { // Vault annotation automatically mounts the volume setupContainer.Command[2] = fmt.Sprintf(setupContainer.Command[2], "/etc/rabbitmq/conf.d/11-default_user.conf") @@ -1097,3 +1140,10 @@ func containerRabbitmq(containers []corev1.Container) corev1.Container { } return corev1.Container{} } + +func getIPv6Cmd(ipFamily string) string { + if ipFamily == "IPv6" { + return "&& echo '{inet6, true}.' > /ipv6/erl_inetrc" + } + return "" +} diff --git a/internal/status/zz_generated.deepcopy.go b/internal/status/zz_generated.deepcopy.go index 34b5ba51d..87ca68bc8 100644 --- a/internal/status/zz_generated.deepcopy.go +++ b/internal/status/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* RabbitMQ Cluster Operator diff --git a/main.go b/main.go index 24d1be22b..92b46fc0f 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,7 @@ func main() { controlRabbitmqImage = false defaultUserUpdaterImage = "rabbitmqoperator/default-user-credential-updater:1.0.2" defaultImagePullSecrets = "" + defaultIPFamily = "IPv4" ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":9782", "The address the metric endpoint binds to.") @@ -159,6 +160,7 @@ func main() { DefaultUserUpdaterImage: defaultUserUpdaterImage, DefaultImagePullSecrets: defaultImagePullSecrets, ControlRabbitmqImage: controlRabbitmqImage, + DefaultIPFamily: defaultIPFamily, }).SetupWithManager(mgr) if err != nil { log.Error(err, "unable to create controller", controllerName)