Introduce separate test stages (#98)

This is for a few reasons: - It allows making only unit tests count towards the measured test coverage. - Gives some visibility into how the separate kinds of tests behave. - Parallelizes the three slow suites, not in an optimal way, but this might have a small positive effect on total test time.
Aiven-Open · Nov 20, 2023 · 50442a1 · 50442a1
1 parent 21bd6b0
commit 50442a1
Show file tree

Hide file tree

Showing 578 changed files with 2,547 additions and 6 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -28,18 +28,15 @@ jobs:
     name: Check packaging metadata
     uses: less-action/reusables/.github/workflows/python-test-build.yaml@0afb53ddb81137deb9d41e7d911eb1755bb451e3
 
-  test:
-    name: Test
+  unit-test:
+    name: Unit tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.11", "3.12"]
     steps:
       - uses: actions/checkout@v3
 
-      - name: Start Kafka backend
-        run: docker compose --file=container/compose.yml up -d
-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -51,7 +48,7 @@ jobs:
         run: pip install --upgrade -e '.[test]'
 
       - name: Run test suite
-        run: python -X dev -m coverage run -m pytest
+        run: python -X dev -m coverage run -m pytest -m "not java and not roundtrip and not integration"
 
       - name: Collect coverage
         run: |
@@ -65,6 +62,75 @@ jobs:
           fail_ci_if_error: true
           name: codecov-py${{ matrix.python-version }}
 
+  roundtrip-test:
+    name: Roundtrip tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: setup.cfg
+
+      - name: Install requirements
+        run: pip install --upgrade -e '.[test]'
+
+      - name: Run test suite
+        run: python -X dev -m pytest -m roundtrip
+
+  java-test:
+    name: Java tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: setup.cfg
+
+      - name: Install requirements
+        run: pip install --upgrade -e '.[test]'
+
+      - name: Run Java tests
+        run: python -X dev -m pytest -m java
+
+  integration-test:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Start Kafka backend
+        run: docker compose --file=container/compose.yml up -d
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: setup.cfg
+
+      - name: Install requirements
+        run: pip install --upgrade -e '.[test]'
+
+      - name: Run integration tests
+        run: python -X dev -m pytest -m integration
+
   check-generate-schema:
     name: Check schema
     runs-on: ubuntu-latest

diff --git a/codegen/generate_tests.py b/codegen/generate_tests.py
@@ -52,6 +52,7 @@ def get_entities() -> Iterator[tuple[type, str]]:
 from tests.conftest import setup_buffer, JavaTester
 from kio.serial import entity_reader
 from typing import Final
+import pytest
 """
 import_code = """\
 from {entity_module} import {entity_type}
@@ -62,6 +63,7 @@ def get_entities() -> Iterator[tuple[type, str]]:
 read_{entity_snake_case}: Final = entity_reader({entity_type})
 
 
+@pytest.mark.roundtrip
 @given(from_type({entity_type}))
 @settings(max_examples=1)
 def test_{entity_snake_case}_roundtrip(instance: {entity_type}) -> None:
@@ -74,6 +76,7 @@ def test_{entity_snake_case}_roundtrip(instance: {entity_type}) -> None:
 """
 
 test_code_java = """\
+@pytest.mark.java
 @given(instance=from_type({entity_type}))
 def test_{entity_snake_case}_java(instance: {entity_type}, java_tester: JavaTester) -> None:
     java_tester.test(instance)

diff --git a/pyproject.toml b/pyproject.toml
@@ -78,6 +78,11 @@ filterwarnings = [
   'ignore: ast\.[A-z]+ is deprecated:DeprecationWarning',
   'ignore: Attribute s is deprecated and will be removed:DeprecationWarning',
 ]
+markers = [
+  "integration: marks tests interacting with a running Apache Kafka® instance",
+  "roundtrip: marks roundtrip Hypothesis serialization tests",
+  "java: marks tests validating serialization against a Java process",
+]
 
 
 [tool.ruff]

diff --git a/tests/generated/test_add_offsets_to_txn_v0_request.py b/tests/generated/test_add_offsets_to_txn_v0_request.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_request: Final = entity_reader(AddOffsetsToTxnRequest)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnRequest))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest) -> None:
@@ -26,6 +28,7 @@ def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest)
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnRequest))
 def test_add_offsets_to_txn_request_java(
     instance: AddOffsetsToTxnRequest, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v0_response.py b/tests/generated/test_add_offsets_to_txn_v0_response.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_response: Final = entity_reader(AddOffsetsToTxnResponse)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnResponse))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_response_roundtrip(
@@ -28,6 +30,7 @@ def test_add_offsets_to_txn_response_roundtrip(
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnResponse))
 def test_add_offsets_to_txn_response_java(
     instance: AddOffsetsToTxnResponse, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v1_request.py b/tests/generated/test_add_offsets_to_txn_v1_request.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_request: Final = entity_reader(AddOffsetsToTxnRequest)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnRequest))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest) -> None:
@@ -26,6 +28,7 @@ def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest)
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnRequest))
 def test_add_offsets_to_txn_request_java(
     instance: AddOffsetsToTxnRequest, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v1_response.py b/tests/generated/test_add_offsets_to_txn_v1_response.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_response: Final = entity_reader(AddOffsetsToTxnResponse)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnResponse))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_response_roundtrip(
@@ -28,6 +30,7 @@ def test_add_offsets_to_txn_response_roundtrip(
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnResponse))
 def test_add_offsets_to_txn_response_java(
     instance: AddOffsetsToTxnResponse, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v2_request.py b/tests/generated/test_add_offsets_to_txn_v2_request.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_request: Final = entity_reader(AddOffsetsToTxnRequest)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnRequest))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest) -> None:
@@ -26,6 +28,7 @@ def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest)
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnRequest))
 def test_add_offsets_to_txn_request_java(
     instance: AddOffsetsToTxnRequest, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v2_response.py b/tests/generated/test_add_offsets_to_txn_v2_response.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_response: Final = entity_reader(AddOffsetsToTxnResponse)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnResponse))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_response_roundtrip(
@@ -28,6 +30,7 @@ def test_add_offsets_to_txn_response_roundtrip(
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnResponse))
 def test_add_offsets_to_txn_response_java(
     instance: AddOffsetsToTxnResponse, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v3_request.py b/tests/generated/test_add_offsets_to_txn_v3_request.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_request: Final = entity_reader(AddOffsetsToTxnRequest)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnRequest))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest) -> None:
@@ -26,6 +28,7 @@ def test_add_offsets_to_txn_request_roundtrip(instance: AddOffsetsToTxnRequest)
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnRequest))
 def test_add_offsets_to_txn_request_java(
     instance: AddOffsetsToTxnRequest, java_tester: JavaTester

diff --git a/tests/generated/test_add_offsets_to_txn_v3_response.py b/tests/generated/test_add_offsets_to_txn_v3_response.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -15,6 +16,7 @@
 read_add_offsets_to_txn_response: Final = entity_reader(AddOffsetsToTxnResponse)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddOffsetsToTxnResponse))
 @settings(max_examples=1)
 def test_add_offsets_to_txn_response_roundtrip(
@@ -28,6 +30,7 @@ def test_add_offsets_to_txn_response_roundtrip(
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddOffsetsToTxnResponse))
 def test_add_offsets_to_txn_response_java(
     instance: AddOffsetsToTxnResponse, java_tester: JavaTester

diff --git a/tests/generated/test_add_partitions_to_txn_v0_request.py b/tests/generated/test_add_partitions_to_txn_v0_request.py
@@ -2,6 +2,7 @@
 
 from typing import Final
 
+import pytest
 from hypothesis import given
 from hypothesis import settings
 from hypothesis.strategies import from_type
@@ -16,6 +17,7 @@
 read_add_partitions_to_txn_topic: Final = entity_reader(AddPartitionsToTxnTopic)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddPartitionsToTxnTopic))
 @settings(max_examples=1)
 def test_add_partitions_to_txn_topic_roundtrip(
@@ -32,6 +34,7 @@ def test_add_partitions_to_txn_topic_roundtrip(
 read_add_partitions_to_txn_request: Final = entity_reader(AddPartitionsToTxnRequest)
 
 
+@pytest.mark.roundtrip
 @given(from_type(AddPartitionsToTxnRequest))
 @settings(max_examples=1)
 def test_add_partitions_to_txn_request_roundtrip(
@@ -45,6 +48,7 @@ def test_add_partitions_to_txn_request_roundtrip(
     assert instance == result
 
 
+@pytest.mark.java
 @given(instance=from_type(AddPartitionsToTxnRequest))
 def test_add_partitions_to_txn_request_java(
     instance: AddPartitionsToTxnRequest, java_tester: JavaTester