make environment/dependencies/initial_context to accept callable

doc/en/download/Driver.rst

@@ -17,3 +17,26 @@ test_plan.py
 suites.py
 +++++++++
 .. literalinclude:: ../../../examples/Driver/Dependency/suites.py
+
+.. _example_env_buider:
+
+Environment Builder
+===================
+
+Required files:
+  - :download:`test_plan.py <../../../examples/Driver/EnvBuilder/test_plan.py>`
+  - :download:`suites.py <../../../examples/Driver/EnvBuilder/suites.py>`
+  - :download:`env_builder.py <../../../examples/Driver/EnvBuilder/env_builder.py>`
+
+test_plan.py
+++++++++++++
+.. literalinclude:: ../../../examples/Driver/EnvBuilder/test_plan.py
+
+suites.py
++++++++++
+.. literalinclude:: ../../../examples/Driver/EnvBuilder/suites.py
+
+env_builder.py
+++++++++++++++
+.. literalinclude:: ../../../examples/Driver/EnvBuilder/env_builder.py
+

examples/Driver/Dependency/test_plan.py

@@ -51,9 +51,9 @@ def main(plan):
         port=context("server_2", "{{port}}"),
     )
 
-    # If driver A depends on driver B to start, we will put driver A in the key
+    # If driver A is a dependency for driver B to start, we put driver A in the key
     # of dependencies dictionary and driver B as its corresponding value, so
-    # driver A is before driver B visually.
+    # visually driver A appears before driver B.
 
     # Here server_1 and server_2 will be started simutaneously to reduce the
     # overall test running time while not violating the dependencies.

examples/Driver/EnvBuilder/env_builder.py

@@ -0,0 +1,64 @@
+"""
+This demonstrates one possible way of implementing of EnvBuilder class.
+"""
+import functools
+
+from testplan.common.utils.context import context
+from testplan.testing.multitest.driver.tcp import TCPClient, TCPServer
+
+
+class EnvBuilder:
+    def __init__(self, env_name: str, drivers: list):
+        """
+        :param env_name: name of this env builder
+        :param drivers: list of drivers to be created
+        """
+        self.env_name = env_name
+        self.drivers = drivers
+        self.client_auto_connect = False if len(self.drivers) == 3 else True
+        self._client1 = None
+        self._client2 = None
+        self._server1 = None
+
+    def build_env(self):
+        return [getattr(self, driver_name) for driver_name in self.drivers]
+
+    def init_ctx(self):
+        return {"env_name": self.env_name}
+
+    def build_deps(self):
+        if len(self.drivers) == 2:
+            return {self.server1: self.client1}
+        elif len(self.drivers) == 3:
+            return {self.server1: (self.client1, self.client2)}
+
+    @property
+    # @functools.cached_property
+    def client1(self):
+        if not self._client1:
+            self._client1 = TCPClient(
+                name="client1",
+                host=context("server1", "{{host}}"),
+                port=context("server1", "{{port}}"),
+                connect_at_start=self.client_auto_connect,
+            )
+        return self._client1
+
+    @property
+    # @functools.cached_property
+    def client2(self):
+        if not self._client2:
+            self._client2 = TCPClient(
+                name="client2",
+                host=context("server1", "{{host}}"),
+                port=context("server1", "{{port}}"),
+                connect_at_start=self.client_auto_connect,
+            )
+        return self._client2
+
+    @property
+    # @functools.cached_property
+    def server1(self):
+        if not self._server1:
+            self._server1 = TCPServer(name="server1")
+        return self._server1

examples/Driver/EnvBuilder/suites.py

@@ -0,0 +1,48 @@
+from testplan.testing.multitest import testsuite, testcase
+
+
+@testsuite
+class TestOneClient:
+    def setup(self, env, result):
+        result.log(f"Testing with [{env.env_name}] env")
+        env.server1.accept_connection()
+
+    @testcase
+    def test_send_and_receive_msg(self, env, result):
+        env.client1.send_text("hi server")
+        result.equal("hi server", env.server1.receive_text())
+
+        env.server1.send_text("hi client")
+        result.equal("hi client", env.client1.receive_text())
+
+
+@testsuite
+class TestTwoClients:
+    def setup(self, env, result):
+        result.log(f"Testing with [{env.env_name}] env")
+
+        env.client1.connect()
+        self.conn1 = env.server1.accept_connection()
+
+        env.client2.connect()
+        self.conn2 = env.server1.accept_connection()
+
+    @testcase
+    def test_send_and_receive_msg(self, env, result):
+        env.client1.send_text("hi server from client1")
+        env.client2.send_text("hi server from client2")
+
+        result.equal(
+            "hi server from client1",
+            env.server1.receive_text(conn_idx=self.conn1),
+        )
+        result.equal(
+            "hi server from client2",
+            env.server1.receive_text(conn_idx=self.conn2),
+        )
+
+        env.server1.send_text("hi client1", conn_idx=self.conn1)
+        env.server1.send_text("hi client2", conn_idx=self.conn2)
+
+        result.equal("hi client1", env.client1.receive_text())
+        result.equal("hi client2", env.client2.receive_text())

examples/Driver/EnvBuilder/test_plan.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+This example demonstrates usage of callable object to construct environment,
+intial_context and dependencies for a multitest at runtime.
+"""
+
+import sys
+
+from testplan import test_plan
+from testplan.testing.multitest import MultiTest
+from env_builder import EnvBuilder
+from suites import TestOneClient, TestTwoClients
+
+
+@test_plan(name="EnvBuilderExample")
+def main(plan):
+
+    env_builder1 = EnvBuilder("One Client", ["client1", "server1"])
+    env_builder2 = EnvBuilder("Two Clients", ["client1", "client2", "server1"])
+    plan.add(
+        MultiTest(
+            name="TestOneClient",
+            suites=[TestOneClient()],
+            environment=env_builder1.build_env,
+            dependencies=env_builder1.build_deps,
+            initial_context=env_builder1.init_ctx,
+        )
+    )
+
+    plan.add(
+        MultiTest(
+            name="TestTwoClients",
+            suites=[TestTwoClients()],
+            environment=env_builder2.build_env,
+            dependencies=env_builder2.build_deps,
+            initial_context=env_builder2.init_ctx,
+        )
+    )
+
+
+if __name__ == "__main__":
+    res = main()
+    print("Exiting code: {}".format(res.exit_code))
+    sys.exit(res.exit_code)

testplan/common/entity/base.py

@@ -473,7 +473,6 @@ def get_options(cls):
         """
         return {
             ConfigOption("runpath"): Or(None, str, callable),
-            ConfigOption("initial_context", default={}): dict,
             ConfigOption("path_cleanup", default=False): bool,
             ConfigOption("status_wait_timeout", default=600): int,
             ConfigOption("abort_wait_timeout", default=300): int,
@@ -490,8 +489,6 @@ class Entity(logger.Loggable):
 
     :param runpath: Path to be used for temp/output files by entity.
     :type runpath: ``str`` or ``NoneType`` callable that returns ``str``
-    :param initial_context: Initial key: value pair context information.
-    :type initial_context: ``dict``
     :param path_cleanup: Remove previous runpath created dirs/files.
     :type path_cleanup: ``bool``
     :param status_wait_timeout: Timeout for wait status events.
@@ -964,15 +961,14 @@ def _run_batch_steps(self):
         start_threads, start_procs = self._get_start_info()
 
         self._add_step(self.setup)
-        self.pre_resource_steps()
-        self._add_step(self.resources.start)
-
-        self.pre_main_steps()
-        self.main_batch_steps()
-        self.post_main_steps()
-
-        self._add_step(self.resources.stop, is_reversed=True)
-        self.post_resource_steps()
+        self.add_pre_resource_steps()
+        self.add_start_resource_steps()
+
+        self.add_pre_main_steps()
+        self.add_main_batch_steps()
+        self.add_post_main_steps()
+        self.add_stop_resource_steps()
+        self.add_post_resource_steps()
         self._add_step(self.teardown)
 
         self._run()
@@ -1055,31 +1051,43 @@ def _execute_step(self, step, *args, **kwargs):
             self.result.step_results[step.__name__] = res
             self.status.update_metadata(**{str(step): res})
 
-    def pre_resource_steps(self):
+    def add_pre_resource_steps(self):
         """
         Runnable steps to run before environment started.
         """
         pass
 
-    def pre_main_steps(self):
+    def add_start_resource_steps(self):
+        """
+        Runnable steps to start environment
+        """
+        self._add_step(self.resources.start)
+
+    def add_pre_main_steps(self):
         """
         Runnable steps to run after environment started.
         """
         pass
 
-    def main_batch_steps(self):
+    def add_main_batch_steps(self):
         """
         Runnable steps to be executed while environment is running.
         """
         pass
 
-    def post_main_steps(self):
+    def add_post_main_steps(self):
         """
         Runnable steps to run before environment stopped.
         """
         pass
 
-    def post_resource_steps(self):
+    def add_stop_resource_steps(self):
+        """
+        Runnable steps to stop environment
+        """
+        self._add_step(self.resources.stop, is_reversed=True)
+
+    def add_post_resource_steps(self):
         """
         Runnable steps to run after environment stopped.
         """

testplan/common/report/base.py

@@ -345,6 +345,12 @@ def get_by_uid(self, uid):
         """
         return self.entries[self._index[uid]]
 
+    def has_uid(self, uid):
+        """
+        Has a child report of `uid`
+        """
+        return uid in self._index
+
     def __getitem__(self, uid):
         """Shortcut to `get_by_uid()` method via [] operator."""
         return self.get_by_uid(uid)

testplan/report/testing/base.py

@@ -186,6 +186,8 @@ class ReportCategories:
     QUNIT = "qunit"
     JUNIT = "junit"
     ERROR = "error"
+    # use for before/after_start/stop, setup, teardown, etc
+    SYNTHESIZED = "synthesized"
 
 
 class ExceptionLogger(ExceptionLoggerBase):
@@ -318,7 +320,9 @@ def runtime_status(self):
     def runtime_status(self, new_status):
         """Set the runtime_status of all child entries."""
         for entry in self:
-            entry.runtime_status = new_status
+            # TODO: use suite_related flag for now, use synthesized instead
+            if not getattr(entry, "suite_related", False):
+                entry.runtime_status = new_status
         self._runtime_status = new_status
 
     def set_runtime_status_filtered(
@@ -382,10 +386,14 @@ def counter(self):
         """
         counter = Counter({Status.PASSED: 0, Status.FAILED: 0, "total": 0})
 
+        # exclude rerun and synthesized entries from counter
         for child in self:
             if child.category == ReportCategories.ERROR:
                 counter.update({Status.ERROR: 1, "total": 1})
-            elif child.category == ReportCategories.TASK_RERUN:
+            elif child.category in (
+                ReportCategories.TASK_RERUN,
+                ReportCategories.SYNTHESIZED,
+            ):
                 pass
             else:
                 counter.update(child.counter)
@@ -662,6 +670,7 @@ def __init__(
         fix_spec_path=None,
         env_status=None,
         strict_order=False,
+        suite_related=False,
         **kwargs,
     ):
         super(TestGroupReport, self).__init__(name=name, **kwargs)
@@ -692,6 +701,10 @@ def __init__(
 
         self.covered_lines: Optional[dict] = None
 
+        # TODO: replace with synthesized flag
+        # this is for special handling in interactive mode, e.g no run button
+        self.suite_related = suite_related
+
     def __str__(self):
         return (
             f'{self.__class__.__name__}(name="{self.name}", category="{self.category}",'
@@ -839,6 +852,7 @@ def __init__(
         suite_related=False,
         status_override=None,
         status_reason=None,
+        category=ReportCategories.TESTCASE,
         **kwargs,
     ):
         super(TestCaseReport, self).__init__(name=name, **kwargs)
@@ -854,7 +868,7 @@ def __init__(
         # testcase is default to passed (e.g no assertion)
         self._status = Status.UNKNOWN
         self._runtime_status = RuntimeStatus.READY
-        self.category = ReportCategories.TESTCASE
+        self.category = category
         self.status_reason = status_reason
 
         self.covered_lines: Optional[dict] = None