Sonobuoy e2e nigthly tests

.github/workflows/nightly-test.yaml

@@ -3,6 +3,7 @@ name: Nightly Latest/Edge Tests
 on:
   schedule:
     - cron: "0 0 * * *" # Runs every midnight
+  pull_request: # todo remove before merge
 
 permissions:
   contents: read
@@ -42,6 +43,7 @@ jobs:
           TEST_SUBSTRATE: lxd
           TEST_LXD_IMAGE: ${{ matrix.os }}
           TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
+          TEST_CNCF_E2E: true
           # Test the latest (up to) 6 releases for the flavour
           # TODO(ben): upgrade nightly to run all flavours
           TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic"
@@ -64,3 +66,15 @@ jobs:
         if: ${{ failure() && github.event_name == 'pull_request' }}
         uses: mxschmitt/action-tmate@v3
         timeout-minutes: 10
+      - name: Extract CNCF conformance report
+        if: failure()
+        working-directory: tests/integration
+        run: |
+         tar -xf sonobuoy_e2e.tar.gz --one-top-level
+         echo "os_dash=${{ matrix.os }}" | sed 's/:/-/g' >> $GITHUB_ENV
+      - name: Upload CNCF conformance report artifact
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+         name: report_sonobuoy_e2e_${{ env.os_dash }}_${{ matrix.arch }}
+         path: tests/integration/sonobuoy_e2e

tests/integration/tests/test_cncf_conformace.py

@@ -0,0 +1,67 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+import logging
+import os
+from typing import List
+
+import pytest
+from test_util import config, harness, util
+
+LOG = logging.getLogger(__name__)
+
+
+@pytest.mark.skipif(
+    os.getenv("TEST_CNCF_E2E") in ["false", None],
+    reason="Test is long and should be run nightly",
+)
+@pytest.mark.node_count(2)
+def test_cncf_conformance(instances: List[harness.Instance]):
+    cluster_node = cluster_setup(instances)
+    install_sonobuoy(cluster_node)
+
+    cluster_node.exec(
+        ["./sonobuoy"],
+    )
+    cluster_node.exec(
+        ["./sonobuoy", "run", "--plugin", "e2e", "--wait"],
+    )
+    cluster_node.exec(
+        ["./sonobuoy", "retrieve", "-f", "sonobuoy_e2e.tar.gz"],
+    )
+    cluster_node.exec(
+        ["tar", "-xf", "sonobuoy_e2e.tar.gz", "--one-top-level"],
+    )
+    resp = cluster_node.exec(
+        ["./sonobuoy", "results", "sonobuoy_e2e.tar.gz"],
+        capture_output=True,
+    )
+    LOG.info(resp.stdout.decode())
+    cluster_node.pull_file("/root/sonobuoy_e2e.tar.gz", "sonobuoy_e2e.tar.gz")
+    assert "Failed: 0" in resp.stdout.decode()
+
+
+def cluster_setup(instances: List[harness.Instance]) -> harness.Instance:
+    cluster_node = instances[0]
+    joining_node = instances[1]
+
+    join_token = util.get_join_token(cluster_node, joining_node)
+    util.join_cluster(joining_node, join_token)
+
+    util.wait_until_k8s_ready(cluster_node, instances)
+
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 2, "node should have joined cluster"
+    assert "control-plane" in util.get_local_node_status(cluster_node)
+    assert "control-plane" in util.get_local_node_status(joining_node)
+
+    config = cluster_node.exec(["k8s", "config"], capture_output=True)
+    cluster_node.exec(["dd", "of=/root/.kube/config"], input=config.stdout)
+
+    return cluster_node
+
+
+def install_sonobuoy(instance: harness.Instance):
+    instance.exec(["curl", "-L", config.SONOBUOY_TAR_GZ, "-o", "sonobuoy.tar.gz"])
+    instance.exec(["tar", "xvzf", "sonobuoy.tar.gz"])
+    instance.exec(["./sonobuoy", "version"])

tests/integration/tests/test_util/config.py

@@ -2,6 +2,7 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import os
+import platform
 from pathlib import Path
 
 DIR = Path(__file__).absolute().parent
@@ -124,3 +125,18 @@
 STRICT_INTERFACE_CHANNELS = (
     os.environ.get("TEST_STRICT_INTERFACE_CHANNELS", "").strip().split()
 )
+
+# ARCH_MAP is a map changing python architecture naming to common one
+ARCH_MAP = {"aarch64": "arm64", "x86_64": "amd64"}
+
+# ARCHITECTURE is an architecture of current machine
+ARCHITECTURE = os.getenv("TEST_ARCHITECTURE") or ARCH_MAP.get(platform.machine())
+
+# SONOBUOY_VERSION is version of sonobuoy to use
+SONOBUOY_VERSION = os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.2"
+
+# SONOBUOY_TAR_GZ is a full path of sonobuoy to download
+SONOBUOY_TAR_GZ = (
+    os.getenv("TEST_SONOBUOY_TAR_GZ")
+    or f"https://github.com/vmware-tanzu/sonobuoy/releases/download/{SONOBUOY_VERSION}/sonobuoy_{SONOBUOY_VERSION[1: ]}_linux_{ARCHITECTURE}.tar.gz"  # noqa
+)

tests/integration/tox.ini

@@ -1,7 +1,7 @@
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-env_list = format, lint, integration
+env_list = format, lint, integration, e2e
 
 [testenv]
 set_env =