tests: workaround docker bug affecting kongintegration flakiness

Makefile

@@ -501,7 +501,6 @@ _test.kongintegration: gotestsum go-junit-report
 		-race \
 		-parallel $(NCPU) \
 		-coverpkg=$(PKG_LIST) \
-		-run=$(TEST_CASE) \
 		-coverprofile=coverage.kongintegration.out \
 		./test/kongintegration | \
 	$(GOJUNIT) -iocopy -out $(JUNIT_REPORT) -parser gotest

internal/manager/health_check_test.go

@@ -11,9 +11,10 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/phayes/freeport"
 	"github.com/stretchr/testify/require"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+
+	"github.com/kong/kubernetes-ingress-controller/v2/test/helpers"
 )
 
 func TestHealthCheckServer(t *testing.T) {
@@ -80,8 +81,7 @@ func TestHealthCheckServer_Start(t *testing.T) {
 	h.setHealthzCheck(healthz.Ping)
 
 	// Get free local port.
-	port, err := freeport.GetFreePort()
-	require.NoError(t, err)
+	port := helpers.GetFreePort(t)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	addr := fmt.Sprintf("localhost:%d", port)

test/e2e/utils_test.go

@@ -21,7 +21,6 @@ import (
 	"github.com/kong/go-kong/kong"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/gke"
 	"github.com/kong/kubernetes-testing-framework/pkg/environments"
-	"github.com/phayes/freeport"
 	"github.com/sethvargo/go-password/password"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -30,6 +29,7 @@ import (
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
+	"github.com/kong/kubernetes-ingress-controller/v2/test/helpers"
 	"github.com/kong/kubernetes-ingress-controller/v2/test/internal/testenv"
 )
 
@@ -353,8 +353,7 @@ func getKongProxyNodePortIP(ctx context.Context, t *testing.T, env environments.
 func startPortForwarder(ctx context.Context, t *testing.T, env environments.Environment, namespace, name, targetPort string) int {
 	t.Helper()
 
-	localPort, err := freeport.GetFreePort()
-	require.NoError(t, err)
+	localPort := helpers.GetFreePort(t)
 
 	kubeconfig := getTemporaryKubeconfig(t, env)
 	cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfig, "port-forward", "-n", namespace, name, fmt.Sprintf("%d:%s", localPort, targetPort))

test/envtest/ingress_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/kong/deck/file"
 	"github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/generators"
-	"github.com/phayes/freeport"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -25,6 +24,7 @@ import (
 
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/util/builder"
 	"github.com/kong/kubernetes-ingress-controller/v2/test"
+	"github.com/kong/kubernetes-ingress-controller/v2/test/helpers"
 )
 
 func TestIngressWorksWithServiceBackendsSpecifyingOnlyPortNames(t *testing.T) {
@@ -45,8 +45,7 @@ func TestIngressWorksWithServiceBackendsSpecifyingOnlyPortNames(t *testing.T) {
 	ingressClassName := "kongenvtest"
 	deployIngressClass(ctx, t, ingressClassName, ctrlClient)
 
-	diagPort, err := freeport.GetFreePort()
-	require.NoError(t, err)
+	diagPort := helpers.GetFreePort(t)
 	ns := CreateNamespace(ctx, t, ctrlClient)
 	RunManager(ctx, t, envcfg,
 		AdminAPIOptFns(),

test/envtest/manager_debugendpoints_test.go

@@ -9,10 +9,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/phayes/freeport"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"k8s.io/client-go/kubernetes/scheme"
+
+	"github.com/kong/kubernetes-ingress-controller/v2/test/helpers"
 )
 
 func TestDebugEndpoints(t *testing.T) {
@@ -26,9 +26,8 @@ func TestDebugEndpoints(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	ports, err := freeport.GetFreePorts(2)
-	require.NoError(t, err)
-	diagPort, healthPort := ports[0], ports[1]
+	diagPort := helpers.GetFreePort(t)
+	healthPort := helpers.GetFreePort(t)
 	envcfg := Setup(t, scheme.Scheme)
 	RunManager(ctx, t, envcfg,
 		AdminAPIOptFns(),

test/envtest/run.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/go-logr/zapr"
-	"github.com/phayes/freeport"
 	"github.com/samber/lo"
 	"github.com/samber/mo"
 	"github.com/stretchr/testify/assert"
@@ -23,6 +22,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/manager"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/featuregates"
 	"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
+	"github.com/kong/kubernetes-ingress-controller/v2/test/helpers"
 	"github.com/kong/kubernetes-ingress-controller/v2/test/mocks"
 )
 
@@ -57,8 +57,7 @@ func ConfigForEnvConfig(t *testing.T, envcfg *rest.Config, opts ...mocks.AdminAP
 	cfg.ProxySyncSeconds = 0.1
 	cfg.InitCacheSyncDuration = 0
 
-	p, err := freeport.GetFreePort()
-	require.NoError(t, err)
+	p := helpers.GetFreePort(t)
 	cfg.MetricsAddr = fmt.Sprintf("localhost:%d", p)
 
 	// And other settings which are irrelevant here.

test/helpers/ports.go

@@ -0,0 +1,49 @@
+package helpers
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/phayes/freeport"
+)
+
+// GetFreePort asks the kernel for a free open port that is ready to use.
+// On top of that, it also makes sure that the port hasn't been used in the current test run yet to reduce
+// chances of a race condition in parallel tests.
+func GetFreePort(t *testing.T) int {
+	var (
+		freePort    int
+		retriesLeft = 100
+	)
+	for {
+		// Get a random free port, but do not use it yet...
+		var err error
+		freePort, err = freeport.GetFreePort()
+		if err != nil {
+			continue
+		}
+
+		// ... First, check if the port has been used in this test run already to reduce chances of a race condition.
+		_, wasUsed := usedPorts.LoadOrStore(freePort, true)
+
+		// The port hasn't been used in this test run - we can use it. It was stored in usedPorts, so it will not be
+		// used again during this test run.
+		if !wasUsed {
+			break
+		}
+
+		// Otherwise, the port was used in this test run. We need to get another one.
+		freePort = 0
+		retriesLeft--
+		if retriesLeft == 0 {
+			break
+		}
+	}
+	if freePort == 0 {
+		t.Fatal("no ports available")
+	}
+	return freePort
+}
+
+// userPorts keeps track of ports that were used in the current test run.
+var usedPorts sync.Map

test/kongintegration/containers/httpbin.go

@@ -25,7 +25,7 @@ func NewHTTPBin(ctx context.Context, t *testing.T) HTTPBin {
 	require.NoError(t, err)
 	req := testcontainers.ContainerRequest{
 		Image:        test.HTTPBinImage,
-		ExposedPorts: []string{port.Port()},
+		ExposedPorts: []string{MappedLocalPort(t, port)},
 		WaitingFor:   wait.ForListeningPort(port),
 	}
 	httpBinC, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{

test/kongintegration/containers/kong.go

@@ -43,8 +43,11 @@ type Kong struct {
 // It sets up a cleanup function that will terminate the container when the test finishes.
 func NewKong(ctx context.Context, t *testing.T, opts ...KongOpt) Kong {
 	req := testcontainers.ContainerRequest{
-		Image:        kongImageUnderTest(),
-		ExposedPorts: []string{kongAdminPort, kongProxyPort},
+		Image: kongImageUnderTest(),
+		ExposedPorts: []string{
+			MappedLocalPort(t, kongAdminPort),
+			MappedLocalPort(t, kongProxyPort),
+		},
 		Env: map[string]string{
 			"KONG_DATABASE":      "off",
 			"KONG_ADMIN_LISTEN":  fmt.Sprintf("0.0.0.0:%s", kongAdminPort),
@@ -68,12 +71,13 @@ func NewKong(ctx context.Context, t *testing.T, opts ...KongOpt) Kong {
 	kong := Kong{
 		container: kongC,
 	}
-	adminURL, err := url.Parse(kong.AdminURL(ctx, t))
-	require.NoError(t, err)
 	t.Cleanup(func() {
 		assert.NoError(t, kongC.Terminate(ctx))
 	})
 
+	adminURL, err := url.Parse(kong.AdminURL(ctx, t))
+	require.NoError(t, err)
+
 	const (
 		tickTime = 100 * time.Millisecond
 		waitTime = time.Minute

test/kongintegration/containers/ports.go

@@ -0,0 +1,21 @@
+package containers
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/docker/go-connections/nat"
+
+	"github.com/kong/kubernetes-ingress-controller/v2/test/helpers"
+)
+
+// MappedLocalPort returns a port mapping for a container port that can be used to access the
+// container from the host. The returned string is in the format expected by testcontainers.ContainerRequest ExposedPorts.
+//
+// This is a workaround for a bug in docker that causes IPv4 and IPv6 port mappings to overlap with
+// different containers on the same host. See: https://github.com/moby/moby/issues/42442.
+func MappedLocalPort(t *testing.T, containerPort nat.Port) string {
+	// Return the port mapping in the format expected by testcontainers.ContainerRequest ExposedPorts:
+	// <host>:<host-port>:<container-port>.
+	return fmt.Sprintf("0.0.0.0:%d:%s", helpers.GetFreePort(t), containerPort.Port())
+}

test/kongintegration/parser_golden_tests_outputs_test.go

@@ -6,12 +6,12 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/kong/deck/file"
 	"github.com/kong/go-kong/kong"
 	"github.com/samber/lo"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"sigs.k8s.io/yaml"
 
@@ -24,6 +24,11 @@ import (
 func TestParsersGoldenTestsOutputs(t *testing.T) {
 	t.Parallel()
 
+	const (
+		timeout = 5 * time.Second
+		tick    = 100 * time.Millisecond
+	)
+
 	ctx := context.Background()
 
 	const goldenTestsOutputsGlob = "../../internal/dataplane/parser/testdata/golden/*/*_golden.yaml"
@@ -54,10 +59,22 @@ func TestParsersGoldenTestsOutputs(t *testing.T) {
 		err = yaml.Unmarshal(goldenTestOutput, content)
 		require.NoError(t, err)
 
-		err, resourceErrors, parseErr := sut.Update(ctx, sendconfig.ContentWithHash{Content: content})
-		assert.NoError(t, err)
-		assert.Empty(t, resourceErrors)
-		assert.NoError(t, parseErr)
+		require.Eventually(t, func() bool {
+			err, resourceErrors, parseErr := sut.Update(ctx, sendconfig.ContentWithHash{Content: content})
+			if err != nil {
+				t.Logf("error: %v", err)
+				return false
+			}
+			if len(resourceErrors) > 0 {
+				t.Logf("resource errors: %v", resourceErrors)
+				return false
+			}
+			if parseErr != nil {
+				t.Logf("parse error: %v", parseErr)
+				return false
+			}
+			return true
+		}, timeout, tick)
 	}
 
 	t.Run("expressions router", func(t *testing.T) {