diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c847389..765c363 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Breaking changes: Improvements and new features: +- Add the class :py:class:`~pytest_container.container.MultiStageContainer` as a + replacement of :py:class:`~pytest_container.build.MultiStageBuild` to handle + container images built from a :file:`Containerfile` with multiple stages + - Add the function :py:func:`~pytest_container.container.ContainerData.read_container_logs` to get access to the logs of the running container @@ -71,8 +75,9 @@ Internal changes: Breaking changes: - add the parameter ``container_runtime`` to - :py:func:`~pytest_container.container.ContainerBaseABC.prepare_container` and - :py:func:`~pytest_container.build.MultiStageBuild.prepare_build`. + ``ContainerBaseABC.prepare_container`` (now called + :py:func:`~pytest_container.container.ContainerPrepareABC.prepare_container`) + and :py:func:`~pytest_container.build.MultiStageBuild.prepare_build`. - deprecate the function ``pytest_container.container_from_pytest_param``, please use @@ -225,7 +230,8 @@ Improvements and new features: parametrize this test run. - Add support to add tags to container images via - :py:attr:`~pytest_container.container.DerivedContainer.add_build_tags`. + ``DerivedContainer.add_build_tags`` (is now called + :py:attr:`~pytest_container.container._ContainerForBuild.add_build_tags`) - Lock container preparation so that only a single process is pulling & building a container image. diff --git a/pytest_container/__init__.py b/pytest_container/__init__.py index 1f660ed..997259e 100644 --- a/pytest_container/__init__.py +++ b/pytest_container/__init__.py @@ -10,6 +10,7 @@ "container_from_pytest_param", "container_to_pytest_param", "DerivedContainer", + "MultiStageContainer", "add_extra_run_and_build_args_options", "add_logging_level_options", "auto_container_parametrize", @@ -31,6 +32,7 @@ from .container import container_from_pytest_param from .container import container_to_pytest_param from .container import DerivedContainer +from .container import MultiStageContainer from .helpers import add_extra_run_and_build_args_options from .helpers import add_logging_level_options from .helpers import auto_container_parametrize diff --git a/pytest_container/container.py b/pytest_container/container.py index 910f055..32641ce 100644 --- a/pytest_container/container.py +++ b/pytest_container/container.py @@ -25,6 +25,7 @@ from os.path import isabs from os.path import join from pathlib import Path +from string import Template from subprocess import call from subprocess import check_output from types import TracebackType @@ -613,7 +614,7 @@ def filelock_filename(self) -> str: if isinstance(value, list): all_elements.append("".join([str(elem) for elem in value])) elif isinstance(value, dict): - all_elements.append("".join(value.values())) + all_elements.append("".join(str(v) for v in value.values())) else: all_elements.append(str(value)) @@ -624,7 +625,7 @@ def filelock_filename(self) -> str: return f"{sha3_256((''.join(all_elements)).encode()).hexdigest()}.lock" -class ContainerBaseABC(ABC): +class ContainerPrepareABC(ABC): """Abstract base class defining the methods that must be implemented by the classes fed to the ``*container*`` fixtures. @@ -639,6 +640,8 @@ def prepare_container( ) -> None: """Prepares the container so that it can be launched.""" + +class ContainerBaseABC(ContainerPrepareABC): @abstractmethod def get_base(self) -> "Union[Container, DerivedContainer]": """Returns the Base of this Container Image. If the container has no @@ -795,20 +798,12 @@ def _run_container_build( @dataclass(unsafe_hash=True) -class DerivedContainer(ContainerBase, ContainerBaseABC): - """Class for storing information about the Container Image under test, that - is build from a :file:`Containerfile`/:file:`Dockerfile` from a different - image (can be any image from a registry or an instance of - :py:class:`Container` or :py:class:`DerivedContainer`). +class _ContainerForBuild(ContainerBase): + """Intermediate class for adding properties to :py:class:`DerivedContainer` + and :py:class:`MultiStageContainer`. """ - base: Union[Container, "DerivedContainer", str] = "" - - #: The :file:`Containerfile` that is used to build this container derived - #: from :py:attr:`base`. - containerfile: str = "" - #: An optional image format when building images with :command:`buildah`. It #: is ignored when the container runtime is :command:`docker`. #: The ``oci`` image format is used by default. If the image format is @@ -822,6 +817,22 @@ class DerivedContainer(ContainerBase, ContainerBaseABC): #: has been built add_build_tags: List[str] = field(default_factory=list) + +@dataclass(unsafe_hash=True) +class DerivedContainer(_ContainerForBuild, ContainerBaseABC): + """Class for storing information about the Container Image under test, that + is build from a :file:`Containerfile`/:file:`Dockerfile` from a different + image (can be any image from a registry or an instance of + :py:class:`Container` or :py:class:`DerivedContainer`). + + """ + + base: Union[Container, "DerivedContainer", str] = "" + + #: The :file:`Containerfile` that is used to build this container derived + #: from :py:attr:`base`. + containerfile: str = "" + def __post_init__(self) -> None: super().__post_init__() if not self.base: @@ -895,8 +906,88 @@ def prepare_container( assert self._build_tag == internal_build_tag +@dataclass +class MultiStageContainer(_ContainerForBuild, ContainerPrepareABC): + """Class representing a container built from a :file:`Containerfile` + containing multiple stages. The :py:attr:`MultiStageContainer.containerfile` + is templated using the builtin :py:class:`string.Template`, where container + image IDs are inserted from the containers in + :py:attr:`MultiStageContainer.containers` after these have been + built/pulled. + + """ + #: :file:`Containerfile` to built the container. If any stages require + #: images that are defined using a :py:class:`DerivedContainer` or a + #: :py:class:`Container`, then insert their ids as a template name and + #: provide that name and the class instance as the key & value into + #: :py:attr:`containers`. + containerfile: str = "" + #: Dictionary of container stages that are used to build the final + #: image. The keys are the template names used in :py:attr:`containerfile` + #: and will be replaced with the container image ids of the respective values. + containers: Dict[str, Union[Container, DerivedContainer, str]] = field( + default_factory=dict + ) + + #: Optional stage of the multistage container build that should be built. + #: The last stage is built by default. + target_stage: str = "" + + def prepare_container( + self, + container_runtime: OciRuntimeBase, + rootdir: Path, + extra_build_args: Optional[List[str]], + ) -> None: + """Builds all intermediate containers and then builds the final + container up to :py:attr:`target_stage` or to the last stage. + + """ + + template_kwargs: Dict[str, str] = {} + + for name, ctr in self.containers.items(): + if isinstance(ctr, str): + warnings.warn( + UserWarning( + "Putting container URLs or scratch into the containers " + "dictionary is not required, just enter them into the " + "containerfile directly." + ) + ) + + if ctr == "scratch": + template_kwargs[name] = ctr + else: + c = Container(url=ctr) + c.prepare_container( + container_runtime, rootdir, extra_build_args + ) + template_kwargs[name] = c._build_tag + else: + ctr.prepare_container( + container_runtime, rootdir, extra_build_args + ) + template_kwargs[name] = ctr._build_tag + + ctrfile = Template(self.containerfile).substitute(**template_kwargs) + + build_args = tuple(*extra_build_args) if extra_build_args else () + if self.target_stage: + build_args += ("--target", self.target_stage) + + self.container_id, internal_tag = _run_container_build( + container_runtime, + rootdir, + ctrfile, + None, + build_args, + self.image_format, + self.add_build_tags, + ) + assert self._build_tag == internal_tag @dataclass(frozen=True) @@ -915,7 +1006,7 @@ class ContainerData: #: the testinfra connection to the running container connection: Any #: the container data class that has been used in this test - container: Union[Container, DerivedContainer] + container: Union[Container, DerivedContainer, MultiStageContainer] #: any ports that are exposed by this container forwarded_ports: List[PortForwarding] @@ -962,50 +1053,65 @@ def container_to_pytest_param( def container_and_marks_from_pytest_param( ctr_or_param: Container, ) -> Tuple[Container, Literal[None]]: - ... + ... # pragma: no cover @overload def container_and_marks_from_pytest_param( ctr_or_param: DerivedContainer, ) -> Tuple[DerivedContainer, Literal[None]]: - ... + ... # pragma: no cover + + +@overload +def container_and_marks_from_pytest_param( + ctr_or_param: MultiStageContainer, +) -> Tuple[MultiStageContainer, Literal[None]]: + ... # pragma: no cover @overload def container_and_marks_from_pytest_param( ctr_or_param: _pytest.mark.ParameterSet, ) -> Tuple[ - Union[Container, DerivedContainer], + Union[Container, DerivedContainer, MultiStageContainer], Optional[Collection[Union[_pytest.mark.MarkDecorator, _pytest.mark.Mark]]], ]: - ... + ... # pragma: no cover def container_and_marks_from_pytest_param( ctr_or_param: Union[ - _pytest.mark.ParameterSet, Container, DerivedContainer + _pytest.mark.ParameterSet, + Container, + DerivedContainer, + MultiStageContainer, ], ) -> Tuple[ - Union[Container, DerivedContainer], + Union[Container, DerivedContainer, MultiStageContainer], Optional[Collection[Union[_pytest.mark.MarkDecorator, _pytest.mark.Mark]]], ]: - """Extracts the :py:class:`~pytest_container.container.Container` or - :py:class:`~pytest_container.container.DerivedContainer` and the + """Extracts the :py:class:`~pytest_container.container.Container`, + :py:class:`~pytest_container.container.DerivedContainer` or + :py:class:`~pytest_container.container.MultiStageContainer` and the corresponding marks from a `pytest.param `_ and returns both. If ``param`` is either a :py:class:`~pytest_container.container.Container` - or a :py:class:`~pytest_container.container.DerivedContainer`, then param is - returned directly and the second return value is ``None``. + or a :py:class:`~pytest_container.container.DerivedContainer` or a + :py:class:`~pytest_container.container.MultiStageContainer`, then ``param`` + is returned directly and the second return value is ``None``. """ - if isinstance(ctr_or_param, (Container, DerivedContainer)): + if isinstance( + ctr_or_param, (Container, DerivedContainer, MultiStageContainer) + ): return ctr_or_param, None if len(ctr_or_param.values) > 0 and isinstance( - ctr_or_param.values[0], (Container, DerivedContainer) + ctr_or_param.values[0], + (Container, DerivedContainer, MultiStageContainer), ): return ctr_or_param.values[0], ctr_or_param.marks @@ -1049,7 +1155,7 @@ class ContainerLauncher: """ #: The container that will be launched - container: Union[Container, DerivedContainer] + container: Union[Container, DerivedContainer, MultiStageContainer] #: The container runtime via which the container will be launched container_runtime: OciRuntimeBase diff --git a/pytest_container/pod.py b/pytest_container/pod.py index 57b9b68..bc13d6d 100644 --- a/pytest_container/pod.py +++ b/pytest_container/pod.py @@ -18,6 +18,7 @@ from pytest_container.container import create_host_port_port_forward from pytest_container.container import DerivedContainer from pytest_container.container import lock_host_port_search +from pytest_container.container import MultiStageContainer from pytest_container.inspect import PortForwarding from pytest_container.logging import _logger from pytest_container.runtime import get_selected_runtime @@ -35,7 +36,7 @@ class Pod: """ #: containers belonging to the pod - containers: List[Union[DerivedContainer, Container]] + containers: List[Union[MultiStageContainer, DerivedContainer, Container]] #: ports exposed by the pod forwarded_ports: List[PortForwarding] = field(default_factory=list) diff --git a/source/api.rst b/source/api.rst index 9019673..e01331e 100644 --- a/source/api.rst +++ b/source/api.rst @@ -8,6 +8,8 @@ The container module .. automodule:: pytest_container.container :members: :undoc-members: + :private-members: + :show-inheritance: The pod module diff --git a/source/fixtures.rst b/source/fixtures.rst index 60a2d39..b726ae9 100644 --- a/source/fixtures.rst +++ b/source/fixtures.rst @@ -45,5 +45,5 @@ directive is only supported for docker images. While this is the default with :command:`docker`, :command:`buildah` will by default build images in the ``OCIv1`` format which does **not** support ``HEALTHCHECK``. To ensure that your created container includes the ``HEALTHCHECK``, set the attribute -:py:attr:`~pytest_container.container.DerivedContainer.image_format` to +:py:attr:`~pytest_container.container._ContainerForBuild.image_format` to :py:attr:`~pytest_container.container.ImageFormat.DOCKER`. diff --git a/source/usage.rst b/source/usage.rst index 651b919..21d1d31 100644 --- a/source/usage.rst +++ b/source/usage.rst @@ -1,6 +1,80 @@ Usage Tips ========== + +Running multi-stage builds +-------------------------- + +``pytest_container`` has builtin support for testing containers that are built +as part of multi-stage builds. More specifically, it supports using containers +defined via the :py:class:`~pytest_container.container.Container` and +:py:class:`~pytest_container.container.DerivedContainer` classes. This is +achieved by providing a :file:`Containerfile` and inserting template strings +instead of the specific container URIs or IDs, e.g.: + +.. code-block:: python + + # the "base" of the final stage + FINAL_BASE = DerivedContainer(...) + + MULTI_STAGE_CTR = MultiStageContainer( + containers={ + "final": FINAL_BASE, + "base": "registry.opensuse.org/opensuse/leap:latest", + }, + containerfile="""FROM $base as base + WORKDIR /src/ + RUN # build something here + + FROM $final as deploy + COPY --from=base /src/binary /usr/bin/binary + """, + ) + + +Such a defined container can be used in a similar fashion to the +:py:class:`~pytest_container.container.Container` and +:py:class:`~pytest_container.container.DerivedContainer` classes in tests as +follows: + +.. code-block:: python + + @pytest.mark.parametrize("container", [MULTI_STAGE_CTR], indirect=True) + def test_multistage_binary_in_final_stage(container: ContainerData): + binary = container.connection.file("/usr/bin/binary") + assert binary.exists and binary.is_executable + + +It is also possible to define which stage of a multi-stage container should be +built. By default the last stage is built. To built a different stage, set +:py:attr:`~pytest_container.container.MultiStageContainer.target_stage` to the +stage to build to (this is equivalent to the ``--target`` setting supplied to +:command:`buildah bud` or :command:`docker build`): + +.. code-block:: python + + MULTI_STAGE_CTR_W_STAGE = MultiStageContainer( + containers={ + "final": FINAL_BASE, + "base": "registry.opensuse.org/opensuse/leap:latest", + }, + containerfile="""FROM $base as base + WORKDIR /src/ + RUN # build something here + + FROM $final as deploy + COPY --from=base /src/binary /usr/bin/binary + """, + target_stage="base" + ) + + @pytest.mark.parametrize("container", [MULTI_STAGE_CTR_W_STAGE], indirect=True) + def test_multistage_binary_in_first_stage(container: ContainerData): + binary = container.connection.file("/src/binary") + assert binary.exists and binary.is_executable + + + Adding global build, run or pod create arguments ------------------------------------------------ @@ -8,7 +82,7 @@ Sometimes it is necessary to customize the build, run or pod create parameters of the container runtime globally, e.g. to use the host's network with docker via ``--network=host``. -The :py:meth:`~pytest_container.container.ContainerBaseABC.prepare_container` +The :py:meth:`~pytest_container.container.ContainerPrepareABC.prepare_container` and :py:meth:`~pytest_container.container.ContainerBase.get_launch_cmd` methods support passing such additional arguments/flags, but this is rather cumbersome to use in practice. The ``*container*`` and ``pod*`` fixtures will therefore diff --git a/tests/test_container_build.py b/tests/test_container_build.py index 1c7a4f1..acfb974 100644 --- a/tests/test_container_build.py +++ b/tests/test_container_build.py @@ -1,18 +1,14 @@ # pylint: disable=missing-function-docstring,missing-module-docstring -from pathlib import Path from typing import Union import pytest from pytest import Config from pytest_container import Container from pytest_container import DerivedContainer -from pytest_container import get_extra_build_args -from pytest_container.build import MultiStageBuild from pytest_container.container import ContainerData from pytest_container.container import ContainerLauncher from pytest_container.container import EntrypointSelection from pytest_container.inspect import PortForwarding -from pytest_container.runtime import LOCALHOST from pytest_container.runtime import OciRuntimeBase from .images import LEAP @@ -63,27 +59,6 @@ CONTAINER_IMAGES = [LEAP, LEAP_WITH_MAN, LEAP_WITH_MAN_AND_LUA] -MULTI_STAGE_BUILD = MultiStageBuild( - containers={ - "builder": LEAP_WITH_MAN, - "runner1": LEAP, - "runner2": "docker.io/alpine", - }, - containerfile_template=r"""FROM $builder as builder -WORKDIR /src -RUN echo $$'#!/bin/sh \n\ -echo "foobar"' > test.sh && chmod +x test.sh - -FROM $runner1 as runner1 -WORKDIR /bin -COPY --from=builder /src/test.sh . -ENTRYPOINT ["/bin/test.sh"] - -FROM $runner2 as runner2 -WORKDIR /bin -COPY --from=builder /src/test.sh . -""", -) # This container would just stop if we would launch it with -d and use the # default entrypoint. If we set the entrypoint to bash, then it should stay up. @@ -254,67 +229,6 @@ def test_derived_container_respects_launch_args( assert int(container.connection.check_output("id -u").strip()) == 0 -def test_multistage_containerfile() -> None: - assert "FROM docker.io/alpine" in MULTI_STAGE_BUILD.containerfile - - -def test_multistage_build( - tmp_path: Path, pytestconfig: Config, container_runtime: OciRuntimeBase -): - MULTI_STAGE_BUILD.build( - tmp_path, - pytestconfig.rootpath, - container_runtime, - extra_build_args=get_extra_build_args(pytestconfig), - ) - - -def test_multistage_build_target( - tmp_path: Path, pytestconfig: Config, container_runtime: OciRuntimeBase -): - first_target = MULTI_STAGE_BUILD.build( - tmp_path, - pytestconfig.rootpath, - container_runtime, - "runner1", - extra_build_args=get_extra_build_args(pytestconfig), - ) - assert ( - LOCALHOST.check_output( - f"{container_runtime.runner_binary} run --rm {first_target}", - ).strip() - == "foobar" - ) - - second_target = MULTI_STAGE_BUILD.build( - tmp_path, - pytestconfig, - container_runtime, - "runner2", - extra_build_args=get_extra_build_args(pytestconfig), - ) - - assert first_target != second_target - assert ( - LOCALHOST.check_output( - f"{container_runtime.runner_binary} run --rm {second_target} /bin/test.sh", - ).strip() - == "foobar" - ) - - for (distro, target) in ( - ("Leap", first_target), - ("Alpine", second_target), - ): - assert ( - distro - in LOCALHOST.check_output( - f"{container_runtime.runner_binary} run --rm --entrypoint= {target} " - "cat /etc/os-release", - ).strip() - ) - - LEAP_THAT_ECHOES_STUFF = DerivedContainer( base=LEAP, containerfile="""CMD ["echo", "foobar"]""" ) diff --git a/tests/test_multistage_container.py b/tests/test_multistage_container.py new file mode 100644 index 0000000..82c53a6 --- /dev/null +++ b/tests/test_multistage_container.py @@ -0,0 +1,170 @@ +# pylint: disable=missing-function-docstring,missing-module-docstring +from pathlib import Path + +import pytest +from pytest_container import get_extra_build_args +from pytest_container import MultiStageBuild +from pytest_container import MultiStageContainer +from pytest_container import OciRuntimeBase +from pytest_container.container import ContainerData +from pytest_container.container import EntrypointSelection +from pytest_container.runtime import LOCALHOST + +from tests.images import LEAP +from tests.images import LEAP_URL +from tests.images import LEAP_WITH_MAN + + +_MULTISTAGE_CTR_FILE = r"""FROM $builder as builder +WORKDIR /src +RUN echo $$'#!/bin/sh \n\ +echo "foobar"' > test.sh && chmod +x test.sh + +FROM $runner1 as runner1 +WORKDIR /bin +COPY --from=builder /src/test.sh . +ENTRYPOINT ["/bin/test.sh"] + +FROM $runner2 as runner2 +WORKDIR /bin +COPY --from=builder /src/test.sh . +""" + +_MULTISTAGE_CTRS = { + "builder": LEAP_WITH_MAN, + "runner1": LEAP, + "runner2": "docker.io/alpine", +} + +MULTI_STAGE_BUILD = MultiStageBuild( + containers=_MULTISTAGE_CTRS, + containerfile_template=_MULTISTAGE_CTR_FILE, +) + +MULTI_STAGE_CTR = MultiStageContainer( + containers=_MULTISTAGE_CTRS, + containerfile=_MULTISTAGE_CTR_FILE, +) + +MULTI_STAGE_CTR_STAGE_1 = MultiStageContainer( + containers=_MULTISTAGE_CTRS, + containerfile=_MULTISTAGE_CTR_FILE, + target_stage="runner1", + entry_point=EntrypointSelection.BASH, +) + +MULTI_STAGE_CTR_STAGE_2 = MultiStageContainer( + containers=_MULTISTAGE_CTRS, + containerfile=_MULTISTAGE_CTR_FILE, + target_stage="runner2", +) + + +def test_multistage_containerfile() -> None: + assert "FROM docker.io/alpine" in MULTI_STAGE_BUILD.containerfile + + +def test_multistage_build( + tmp_path: Path, + pytestconfig: pytest.Config, + container_runtime: OciRuntimeBase, +): + MULTI_STAGE_BUILD.build( + tmp_path, + pytestconfig.rootpath, + container_runtime, + extra_build_args=get_extra_build_args(pytestconfig), + ) + + +def test_multistage_build_target( + tmp_path: Path, + pytestconfig: pytest.Config, + container_runtime: OciRuntimeBase, +): + first_target = MULTI_STAGE_BUILD.build( + tmp_path, + pytestconfig.rootpath, + container_runtime, + "runner1", + extra_build_args=get_extra_build_args(pytestconfig), + ) + assert ( + LOCALHOST.check_output( + f"{container_runtime.runner_binary} run --rm {first_target}", + ).strip() + == "foobar" + ) + + second_target = MULTI_STAGE_BUILD.build( + tmp_path, + pytestconfig, + container_runtime, + "runner2", + extra_build_args=get_extra_build_args(pytestconfig), + ) + + assert first_target != second_target + assert ( + LOCALHOST.check_output( + f"{container_runtime.runner_binary} run --rm {second_target} /bin/test.sh", + ).strip() + == "foobar" + ) + + for (distro, target) in ( + ("Leap", first_target), + ("Alpine", second_target), + ): + assert ( + distro + in LOCALHOST.check_output( + f"{container_runtime.runner_binary} run --rm --entrypoint= {target} " + "cat /etc/os-release", + ).strip() + ) + + +@pytest.mark.parametrize("container", [MULTI_STAGE_CTR], indirect=True) +def test_multistage_container_without_stage(container: ContainerData) -> None: + assert container.connection.file("/bin/test.sh").exists + assert ( + "Alpine" in container.connection.file("/etc/os-release").content_string + ) + + +@pytest.mark.parametrize("container", [MULTI_STAGE_CTR_STAGE_2], indirect=True) +def test_multistage_container_with_runner2_stage( + container: ContainerData, +) -> None: + assert container.connection.file("/bin/test.sh").exists + assert ( + "Alpine" in container.connection.file("/etc/os-release").content_string + ) + + +@pytest.mark.parametrize("container", [MULTI_STAGE_CTR_STAGE_1], indirect=True) +def test_multistage_container_with_runner1_stage( + container: ContainerData, +) -> None: + assert container.connection.file("/bin/test.sh").exists + assert ( + "Leap" in container.connection.file("/etc/os-release").content_string + ) + + +@pytest.mark.parametrize( + "container", + [ + MultiStageContainer( + containerfile="""FROM $nothing as nothing +FROM $builder as builder +RUN zypper -n in busybox +""", + containers={"nothing": "scratch", "builder": LEAP_URL}, + ) + ], + indirect=True, +) +def test_multistage_does_not_pull_scratch(container: ContainerData) -> None: + container.connection.check_output("true")