Drop support for metadata records (#403)

* Remove creation of meta data records; changed logic to delete DNSEntries; cleanup of meta data records

* command line options and updated README

* drop obsolete tests

* discontinue section

* fix lint issues

* address feedback from review

* fix check for update recordset

* add loadtest tool

* filter entries by kind in name

* fix deletion of entries after controller restart

charts/external-dns-management/templates/deployment.yaml

@@ -324,6 +324,9 @@ spec:
         {{- if .Values.configuration.compoundLockStatusCheckPeriod }}
         - --compound.lock-status-check-period={{ .Values.configuration.compoundLockStatusCheckPeriod }}
         {{- end }}
+        {{- if .Values.configuration.compoundMaxMetadataRecordDeletionsPerReconciliation }}
+        - --compound.max-metadata-record-deletions-per-reconciliation={{ .Values.configuration.compoundMaxMetadataRecordDeletionsPerReconciliation }}
+        {{- end }}
         {{- if .Values.configuration.compoundNetlifyDnsAdvancedBatchSize }}
         - --compound.netlify-dns.advanced.batch-size={{ .Values.configuration.compoundNetlifyDnsAdvancedBatchSize }}
         {{- end }}
@@ -420,9 +423,6 @@ spec:
         {{- if .Values.configuration.compoundSetup }}
         - --compound.setup={{ .Values.configuration.compoundSetup }}
         {{- end }}
-        {{- if .Values.configuration.compoundStatisticPoolSize }}
-        - --compound.statistic.pool.size={{ .Values.configuration.compoundStatisticPoolSize }}
-        {{- end }}
         {{- if .Values.configuration.compoundTtl }}
         - --compound.ttl={{ .Values.configuration.compoundTtl }}
         {{- end }}
@@ -801,6 +801,9 @@ spec:
         {{- if .Values.configuration.maintainer }}
         - --maintainer={{ .Values.configuration.maintainer }}
         {{- end }}
+        {{- if .Values.configuration.maxMetadataRecordDeletionsPerReconciliation }}
+        - --max-metadata-record-deletions-per-reconciliation={{ .Values.configuration.maxMetadataRecordDeletionsPerReconciliation }}
+        {{- end }}
         {{- if .Values.configuration.namespace }}
         - --namespace={{ .Values.configuration.namespace }}
         {{- end }}
@@ -975,9 +978,6 @@ spec:
         {{- if .Values.configuration.setup }}
         - --setup={{ .Values.configuration.setup }}
         {{- end }}
-        {{- if .Values.configuration.statisticPoolSize }}
-        - --statistic.pool.size={{ .Values.configuration.statisticPoolSize }}
-        {{- end }}
         {{- if .Values.configuration.target }}
         - --target={{ .Values.configuration.target }}
         {{- end }}

charts/external-dns-management/values.yaml

@@ -132,6 +132,7 @@ configuration:
   # compoundInfobloxDnsRatelimiterEnabled:
   # compoundInfobloxDnsRatelimiterQps:
   # compoundLockStatusCheckPeriod:
+  # compoundMaxMetadataRecordDeletionsPerReconciliation:
   # compoundNetlifyDnsAdvancedBatchSize:
   # compoundNetlifyDnsAdvancedMaxRetries:
   # compoundNetlifyDnsRatelimiterBurst:
@@ -164,7 +165,6 @@ configuration:
   # compoundRfc2136RatelimiterQps:
   # compoundSecretsPoolSize: 2
   # compoundSetup: 10
-  # compoundStatisticPoolSize:
   # compoundTtl: 120
   # compoundZonepoliciesPoolSize:
   # config:
@@ -291,6 +291,7 @@ configuration:
   # lockStatusCheckPeriod:
   # logLevel: info
   # maintainer:
+  # maxMetadataRecordDeletionsPerReconciliation:
   # namespace: default
   # namespaceLocalAccessOnly: false
   # netlifyDnsAdvancedBatchSize:
@@ -350,7 +351,6 @@ configuration:
   # serviceDNSTargetsPoolSize: 2
   # servicesPoolSize:
   # setup: 10
-  # statisticPoolSize:
   # target: ""
   # targetCreatorLabelName: ""
   # targetCreatorLabelValue: ""

examples/60-owner.yaml

@@ -1,3 +1,5 @@
+# Please note that starting with release `v0.23` DNSOwner resources are only used to clean up metadata DNS records.
+# The owner identifiers are not used for any other purpose anymore.
 apiVersion: dns.gardener.cloud/v1alpha1
 kind: DNSOwner
 metadata:

hack/tools/loadtest/main.go

@@ -0,0 +1,122 @@
+// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//
+// This small tool can be used to create a given number of DNS entries for load tests:
+//
+// Usage:
+//	go run main.go
+//
+//  Command line options:
+//     -base-domain string
+//     		base domain for the entries (mandatory)
+//     -count int
+//     		number of entries to create (default 10)
+//     -kubeconfig string
+//     		absolute path to the kubeconfig file (defaults to the env variable `KUBECONFIG`)
+//     -label string
+//     		label value for label 'loadtest' to set on the entries (default "true")
+//
+// You may use `kubectl delete dnsentry -l loadtest=<label-value>` to delete them all at once.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	dnsv1alpha1 "github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1"
+	dnsmanclient "github.com/gardener/external-dns-management/pkg/dnsman2/client"
+	"github.com/gardener/gardener/pkg/controllerutils"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var (
+	kubeconfig string
+	baseDomain string
+	labelValue string
+	count      int
+)
+
+func main() {
+	flag.StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "absolute path to the kubeconfig file")
+	flag.IntVar(&count, "count", 10, "number of entries to create")
+	flag.StringVar(&baseDomain, "base-domain", "", "base domain for the entries")
+	flag.StringVar(&labelValue, "label", "true", "label value for label 'loadtest' to set on the entries")
+	flag.Parse()
+
+	if baseDomain == "" {
+		fmt.Fprintf(os.Stderr, "-base-domain is required\n")
+		os.Exit(1)
+	}
+
+	c, err := createClient()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create client: %w\n", err)
+		os.Exit(1)
+	}
+
+	ctx := context.Background()
+
+	fmt.Fprintf(os.Stdout, "Creating %d entries - please wait\n", count)
+
+	if err := createEntries(ctx, c, count, "e%05d", baseDomain, "loadtest", labelValue); err != nil {
+		fmt.Fprintf(os.Stderr, "failed to create entries: %w\n", err)
+		os.Exit(2)
+	}
+
+	fmt.Fprintf(os.Stdout, "Done - all %d entries created\n", count)
+}
+
+func createEntries(ctx context.Context, c client.Client, count int, nameTemplate, baseDomain, labelKey, labelValue string) error {
+	for i := 0; i < count; i++ {
+		entry := &dnsv1alpha1.DNSEntry{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "default",
+				Name:      fmt.Sprintf(nameTemplate, i),
+			},
+		}
+		if _, err := controllerutils.CreateOrGetAndMergePatch(ctx, c, entry, func() error {
+			entry.Labels = map[string]string{
+				labelKey: labelValue,
+			}
+			entry.Spec = dnsv1alpha1.DNSEntrySpec{
+				DNSName: fmt.Sprintf("%s.%s", fmt.Sprintf(nameTemplate, i), baseDomain),
+				Targets: []string{fmt.Sprintf("2.%d.%d.%d", i>>16, (i&0xff00)>>8, i&0xff)},
+				TTL:     ptr.To[int64](120),
+			}
+			return nil
+		}); err != nil {
+			return fmt.Errorf("failed to create/update entry %s: %w", entry.Name, err)
+		}
+		if i > 0 && i%100 == 0 {
+			fmt.Fprintf(os.Stdout, "%d/%d entries created...\n", i, count)
+		}
+	}
+
+	return nil
+}
+
+func createClient() (client.Client, error) {
+	if kubeconfig == "" {
+		return nil, fmt.Errorf("-kubeconfig or KUBECONFIG env var is required")
+	}
+
+	cfg, err := clientcmd.LoadFromFile(kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+	clientConfig := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{})
+	restConfig, err := clientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return client.New(restConfig, client.Options{Scheme: dnsmanclient.ClusterScheme})
+}

pkg/controller/provider/alicloud/handler.go

@@ -111,10 +111,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return state, nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := raw.ExecuteRequests(logger, &h.config, h.access, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/aws/execution.go

@@ -72,8 +72,8 @@ func buildResourceRecordSet(ctx context.Context, name dns.DNSSetName, policy *dn
 }
 
 func (this *Execution) addChange(ctx context.Context, action route53types.ChangeAction, req *provider.ChangeRequest, dnsset *dns.DNSSet) error {
-	name, rset := dns.MapToProvider(req.Type, dnsset, this.zone.Domain())
-	name = name.Align()
+	name := dnsset.Name.Align()
+	rset := dnsset.Sets[req.Type]
 	if len(rset.Records) == 0 {
 		return nil
 	}

pkg/controller/provider/aws/handler.go

@@ -204,10 +204,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return provider.NewDNSZoneState(dnssets), nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	ctx := context.Background()
 	err := h.executeRequests(ctx, logger, zone, state, reqs)

pkg/controller/provider/azure-private/execution.go

@@ -56,12 +56,12 @@ func (exec *Execution) buildRecordSet(req *provider.ChangeRequest) (buildStatus,
 		return bs_invalidRoutingPolicy, "", nil
 	}
 
-	setName, rset := dns.MapToProvider(req.Type, dnsset, exec.zoneName)
-	name, ok := utils.DropZoneName(setName.DNSName, exec.zoneName)
+	name, ok := utils.DropZoneName(dnsset.Name.DNSName, exec.zoneName)
 	if !ok {
 		return bs_invalidName, "", &armprivatedns.RecordSet{Name: &name}
 	}
 
+	rset := dnsset.Sets[req.Type]
 	if len(rset.Records) == 0 {
 		return bs_empty, "", nil
 	}

pkg/controller/provider/azure-private/handler.go

@@ -184,10 +184,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return provider.NewDNSZoneState(dnssets), nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := h.executeRequests(logger, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/azure/execution.go

@@ -56,12 +56,12 @@ func (exec *Execution) buildRecordSet(req *provider.ChangeRequest) (buildStatus,
 		return bs_invalidRoutingPolicy, "", nil
 	}
 
-	setName, rset := dns.MapToProvider(req.Type, dnsset, exec.zoneName)
-	name, ok := utils.DropZoneName(setName.DNSName, exec.zoneName)
+	name, ok := utils.DropZoneName(dnsset.Name.DNSName, exec.zoneName)
 	if !ok {
 		return bs_invalidName, "", &armdns.RecordSet{Name: &name}
 	}
 
+	rset := dnsset.Sets[req.Type]
 	if len(rset.Records) == 0 {
 		return bs_empty, "", nil
 	}

pkg/controller/provider/azure/handler.go

@@ -184,10 +184,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return provider.NewDNSZoneState(dnssets), nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := h.executeRequests(logger, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/cloudflare/handler.go

@@ -109,10 +109,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return state, nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := raw.ExecuteRequests(logger, &h.config, h.access, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/google/execution.go

@@ -51,11 +51,13 @@ func (this *Execution) addChange(req *provider.ChangeRequest) {
 	var err error
 
 	if req.Addition != nil {
-		setName, newset = dns.MapToProvider(req.Type, req.Addition, this.zone.Domain())
+		setName = req.Addition.Name
+		newset = req.Addition.Sets[req.Type]
 		policy, err = extractRoutingPolicy(req.Addition)
 	}
 	if req.Deletion != nil {
-		setName, oldset = dns.MapToProvider(req.Type, req.Deletion, this.zone.Domain())
+		setName = req.Deletion.Name
+		oldset = req.Deletion.Sets[req.Type]
 		if req.Addition == nil {
 			policy, err = extractRoutingPolicy(req.Deletion)
 		}

pkg/controller/provider/google/handler.go

@@ -186,10 +186,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return provider.NewDNSZoneState(dnssets), nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := h.executeRequests(logger, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/mock/handler.go

@@ -109,10 +109,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return h.mock.CloneZoneState(zone)
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := h.executeRequests(logger, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/netlify/handler.go

@@ -127,10 +127,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return state, nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := raw.ExecuteRequests(logger, &h.config, h.access, zone, state, reqs)
 	h.cache.ApplyRequests(logger, err, zone, reqs)

pkg/controller/provider/openstack/execution.go

@@ -52,8 +52,8 @@ func (exec *Execution) buildRecordSet(req *provider.ChangeRequest) (buildStatus,
 		return bsInvalidRoutingPolicy, nil
 	}
 
-	name, rset := dns.MapToProvider(req.Type, dnsset, exec.zone.Domain())
-
+	name := dnsset.Name
+	rset := dnsset.Sets[req.Type]
 	if len(rset.Records) == 0 {
 		return bsEmpty, nil
 	}

pkg/controller/provider/openstack/handler.go

@@ -194,10 +194,6 @@ func (h *Handler) getZoneState(zone provider.DNSHostedZone, _ provider.ZoneCache
 	return provider.NewDNSZoneState(dnssets), nil
 }
 
-func (h *Handler) ReportZoneStateConflict(zone provider.DNSHostedZone, err error) bool {
-	return h.cache.ReportZoneStateConflict(zone, err)
-}
-
 // ExecuteRequests applies a given change request to a given hosted zone.
 func (h *Handler) ExecuteRequests(logger logger.LogContext, zone provider.DNSHostedZone, state provider.DNSZoneState, reqs []*provider.ChangeRequest) error {
 	err := h.executeRequests(logger, zone, state, reqs)