Merge pull request #178 from diegoferigo/refactor/pytest

Improve testing suite
robotology-legacy · Apr 20, 2020 · c42ceec · c42ceec
2 parents 424d295 + c82f660
commit c42ceec
Show file tree

Hide file tree

Showing 62 changed files with 130 additions and 46 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -65,15 +65,27 @@ jobs:
 
       - name: Python Tests [ScenarI/O]
         run: |
-          cd tests/scenario
-          pytest -v
+          cd tests
+          pytest -m "scenario"
 
       - name: Python Tests with Valgrind [ScenarI/O]
         if: failure()
         run: |
           pip3 install colour-valgrind
-          cd tests/scenario
-          valgrind --log-file=/tmp/valgrind.log pytest -v -s || colour-valgrind -t /tmp/valgrind.log
+          cd tests
+          valgrind --log-file=/tmp/valgrind.log pytest -s -m "scenario" || colour-valgrind -t /tmp/valgrind.log
+
+      - name: Python Tests [gym_ignition]
+        run: |
+          cd tests
+          pytest -m "gym_ignition"
+
+      - name: Python Tests with Valgrind [gym_ignition]
+        if: failure()
+        run: |
+          pip3 install colour-valgrind
+          cd tests
+          valgrind --log-file=/tmp/valgrind.log pytest -s -m "gym_ignition" || colour-valgrind -t /tmp/valgrind.log
 
   user:
     name: User installation
@@ -112,12 +124,24 @@ jobs:
 
       - name: Python Tests [ScenarI/O]
         run: |
-          cd tests/scenario
-          pytest -v
+          cd tests
+          pytest -m "scenario"
 
       - name: Python Tests with Valgrind [ScenarI/O]
         if: failure()
         run: |
           pip3 install colour-valgrind
-          cd tests/scenario
-          valgrind --log-file=/tmp/valgrind.log pytest -v -s || colour-valgrind -t /tmp/valgrind.log
+          cd tests
+          valgrind --log-file=/tmp/valgrind.log pytest -s -m "scenario" || colour-valgrind -t /tmp/valgrind.log
+
+      - name: Python Tests [gym_ignition]
+        run: |
+          cd tests
+          pytest -m "gym_ignition"
+
+      - name: Python Tests with Valgrind [gym_ignition]
+        if: failure()
+        run: |
+          pip3 install colour-valgrind
+          cd tests
+          valgrind --log-file=/tmp/valgrind.log pytest -s -m "gym_ignition" || colour-valgrind -t /tmp/valgrind.log
diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
@@ -52,10 +52,10 @@ jobs:
           cd dist
           pip install *.tar.gz
 
-      - name: Python Tests [ScenarI/O]
+      - name: Python Tests
         run: |
-          cd tests/scenario
-          pytest -v
+          cd tests
+          pytest
 
   # ========================
   # TEST BDIST_WHEEL PACKAGE
@@ -114,8 +114,8 @@ jobs:
       - name: Install bdist
         run: docker exec -i -w /dist test sh -c 'pip install *.whl'
 
-      - name: Python Tests [ScenarI/O]
-        run: docker exec -i -w /tests/scenario test sh -c 'pytest -v'
+      - name: Python Tests
+        run: docker exec -i -w /tests/scenario test sh -c 'pytest'
 
   # =============
   # PUSH PACKAGES

diff --git a/gym_ignition/__init__.py → python/gym_ignition/__init__.py b/gym_ignition/__init__.py → python/gym_ignition/__init__.py
diff --git a/gym_ignition/base/__init__.py → python/gym_ignition/base/__init__.py b/gym_ignition/base/__init__.py → python/gym_ignition/base/__init__.py
diff --git a/gym_ignition/base/runtime.py → python/gym_ignition/base/runtime.py b/gym_ignition/base/runtime.py → python/gym_ignition/base/runtime.py
diff --git a/gym_ignition/base/task.py → python/gym_ignition/base/task.py b/gym_ignition/base/task.py → python/gym_ignition/base/task.py
diff --git a/gym_ignition/controllers/__init__.py → python/gym_ignition/controllers/__init__.py b/gym_ignition/controllers/__init__.py → python/gym_ignition/controllers/__init__.py
diff --git a/gym_ignition/controllers/gazebo/__init__.py → ...m_ignition/controllers/gazebo/__init__.py b/gym_ignition/controllers/gazebo/__init__.py → ...m_ignition/controllers/gazebo/__init__.py
diff --git a/...lers/gazebo/computed_torque_fixed_base.py → ...lers/gazebo/computed_torque_fixed_base.py b/...lers/gazebo/computed_torque_fixed_base.py → ...lers/gazebo/computed_torque_fixed_base.py
diff --git a/gym_ignition/experimental/gympp_env.py → ...on/gym_ignition/experimental/gympp_env.py b/gym_ignition/experimental/gympp_env.py → ...on/gym_ignition/experimental/gympp_env.py
diff --git a/gym_ignition/gympp/__init__.py → python/gym_ignition/gympp/__init__.py b/gym_ignition/gympp/__init__.py → python/gym_ignition/gympp/__init__.py
diff --git a/gym_ignition/gympp/cartpole.py → python/gym_ignition/gympp/cartpole.py b/gym_ignition/gympp/cartpole.py → python/gym_ignition/gympp/cartpole.py
diff --git a/gym_ignition/runtimes/__init__.py → python/gym_ignition/runtimes/__init__.py b/gym_ignition/runtimes/__init__.py → python/gym_ignition/runtimes/__init__.py
diff --git a/gym_ignition/runtimes/gazebo_runtime.py → ...n/gym_ignition/runtimes/gazebo_runtime.py b/gym_ignition/runtimes/gazebo_runtime.py → ...n/gym_ignition/runtimes/gazebo_runtime.py
diff --git a/gym_ignition/runtimes/realtime_runtime.py → ...gym_ignition/runtimes/realtime_runtime.py b/gym_ignition/runtimes/realtime_runtime.py → ...gym_ignition/runtimes/realtime_runtime.py
diff --git a/gym_ignition/scenario/__init__.py → python/gym_ignition/scenario/__init__.py b/gym_ignition/scenario/__init__.py → python/gym_ignition/scenario/__init__.py
diff --git a/gym_ignition/scenario/model_with_file.py → .../gym_ignition/scenario/model_with_file.py b/gym_ignition/scenario/model_with_file.py → .../gym_ignition/scenario/model_with_file.py
diff --git a/gym_ignition/scenario/model_wrapper.py → ...on/gym_ignition/scenario/model_wrapper.py b/gym_ignition/scenario/model_wrapper.py → ...on/gym_ignition/scenario/model_wrapper.py
diff --git a/gym_ignition/utils/__init__.py → python/gym_ignition/utils/__init__.py b/gym_ignition/utils/__init__.py → python/gym_ignition/utils/__init__.py
diff --git a/gym_ignition/utils/gazebo_env_vars.py → python/gym_ignition/utils/gazebo_env_vars.py b/gym_ignition/utils/gazebo_env_vars.py → python/gym_ignition/utils/gazebo_env_vars.py
diff --git a/gym_ignition/utils/inverse_kinematics_nlp.py → ..._ignition/utils/inverse_kinematics_nlp.py b/gym_ignition/utils/inverse_kinematics_nlp.py → ..._ignition/utils/inverse_kinematics_nlp.py
diff --git a/gym_ignition/utils/logger.py → python/gym_ignition/utils/logger.py b/gym_ignition/utils/logger.py → python/gym_ignition/utils/logger.py
diff --git a/gym_ignition/utils/misc.py → python/gym_ignition/utils/misc.py b/gym_ignition/utils/misc.py → python/gym_ignition/utils/misc.py
diff --git a/gym_ignition/utils/resource_finder.py → python/gym_ignition/utils/resource_finder.py b/gym_ignition/utils/resource_finder.py → python/gym_ignition/utils/resource_finder.py
diff --git a/gym_ignition/utils/scenario.py → python/gym_ignition/utils/scenario.py b/gym_ignition/utils/scenario.py → python/gym_ignition/utils/scenario.py
diff --git a/gym_ignition/utils/typing.py → python/gym_ignition/utils/typing.py b/gym_ignition/utils/typing.py → python/gym_ignition/utils/typing.py
diff --git a/gym_ignition_environments/models/__init__.py → ..._ignition_environments/models/__init__.py b/gym_ignition_environments/models/__init__.py → ..._ignition_environments/models/__init__.py
diff --git a/gym_ignition_environments/models/cartpole.py → ..._ignition_environments/models/cartpole.py b/gym_ignition_environments/models/cartpole.py → ..._ignition_environments/models/cartpole.py
diff --git a/gym_ignition_environments/models/pendulum.py → ..._ignition_environments/models/pendulum.py b/gym_ignition_environments/models/pendulum.py → ..._ignition_environments/models/pendulum.py
diff --git a/gym_ignition_environments/tasks/__init__.py → ...m_ignition_environments/tasks/__init__.py b/gym_ignition_environments/tasks/__init__.py → ...m_ignition_environments/tasks/__init__.py
diff --git a/...ts/tasks/cartpole_continuous_balancing.py → ...ts/tasks/cartpole_continuous_balancing.py b/...ts/tasks/cartpole_continuous_balancing.py → ...ts/tasks/cartpole_continuous_balancing.py
diff --git a/...ents/tasks/cartpole_continuous_swingup.py → ...ents/tasks/cartpole_continuous_swingup.py b/...ents/tasks/cartpole_continuous_swingup.py → ...ents/tasks/cartpole_continuous_swingup.py
diff --git a/...ents/tasks/cartpole_discrete_balancing.py → ...ents/tasks/cartpole_discrete_balancing.py b/...ents/tasks/cartpole_discrete_balancing.py → ...ents/tasks/cartpole_discrete_balancing.py
diff --git a/...on_environments/tasks/pendulum_swingup.py → ...on_environments/tasks/pendulum_swingup.py b/...on_environments/tasks/pendulum_swingup.py → ...on_environments/tasks/pendulum_swingup.py
diff --git a/setup.py b/setup.py
@@ -148,7 +148,8 @@ def build_extension(self, ext):
         'numpy',
         'gym_ignition_models',
     ],
-    packages=find_packages(),
+    packages=find_packages("python"),
+    package_dir={'': "python"},
     ext_modules=[CMakeExtension(name='InstallAllTargets', cmake_configuration='PyPI')],
     cmdclass={
         'build_ext': BuildExtension,

diff --git a/tests/python/__init__.py → tests/.python/__init__.py b/tests/python/__init__.py → tests/.python/__init__.py
diff --git a/tests/python/test_bindings.py → tests/.python/test_bindings.py b/tests/python/test_bindings.py → tests/.python/test_bindings.py
diff --git a/tests/python/test_contacts.py → tests/.python/test_contacts.py b/tests/python/test_contacts.py → tests/.python/test_contacts.py
diff --git a/...python/test_environments_gazebowrapper.py → ...python/test_environments_gazebowrapper.py b/...python/test_environments_gazebowrapper.py → ...python/test_environments_gazebowrapper.py
diff --git a/tests/python/test_environments_gympp.py → tests/.python/test_environments_gympp.py b/tests/python/test_environments_gympp.py → tests/.python/test_environments_gympp.py
diff --git a/tests/python/test_externalforce.py → tests/.python/test_externalforce.py b/tests/python/test_externalforce.py → tests/.python/test_externalforce.py
diff --git a/tests/python/test_inverse_kinematics.py → tests/.python/test_inverse_kinematics.py b/tests/python/test_inverse_kinematics.py → tests/.python/test_inverse_kinematics.py
diff --git a/tests/python/test_joint_force.py → tests/.python/test_joint_force.py b/tests/python/test_joint_force.py → tests/.python/test_joint_force.py
diff --git a/.../python/test_pendulum_wrt_ground_truth.py → ....python/test_pendulum_wrt_ground_truth.py b/.../python/test_pendulum_wrt_ground_truth.py → ....python/test_pendulum_wrt_ground_truth.py
diff --git a/tests/python/test_rates.py → tests/.python/test_rates.py b/tests/python/test_rates.py → tests/.python/test_rates.py
diff --git a/tests/python/test_robot_base.py → tests/.python/test_robot_base.py b/tests/python/test_robot_base.py → tests/.python/test_robot_base.py
diff --git a/tests/python/test_robot_controller.py → tests/.python/test_robot_controller.py b/tests/python/test_robot_controller.py → tests/.python/test_robot_controller.py
diff --git a/tests/python/test_runtimes.py → tests/.python/test_runtimes.py b/tests/python/test_runtimes.py → tests/.python/test_runtimes.py
diff --git a/tests/python/utils.py → tests/.python/utils.py b/tests/python/utils.py → tests/.python/utils.py
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (C) 2019 Istituto Italiano di Tecnologia (IIT). All rights reserved.
+# This software may be modified and distributed under the terms of the
+# GNU Lesser General Public License v2.1 or any later version.
+
+from . import common
diff --git a/tests/common/__init__.py b/tests/common/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (C) 2020 Istituto Italiano di Tecnologia (IIT). All rights reserved.
+# This software may be modified and distributed under the terms of the
+# GNU Lesser General Public License v2.1 or any later version.
+
+from . import utils
diff --git a/tests/scenario/utils.py → tests/common/utils.py b/tests/scenario/utils.py → tests/common/utils.py
@@ -3,8 +3,8 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
-import tempfile
 from typing import Tuple
+from gym_ignition.utils import misc
 from gym_ignition import scenario_bindings as bindings
 
 
@@ -15,6 +15,29 @@ def id_gazebo_fn(val: Tuple[float, float, float]):
 
 @pytest.fixture(scope="function")
 def gazebo_fixture(request):
+    """
+    Return an instance of the GazeboSimulator ensuring that it
+    is closed even in case of failure.
+
+    Example:
+
+        @pytest.mark.parametrize("gazebo", [(0.001, 1.0, 1)], indirect=True)
+        def test_foo(gazebo: bindings.GazeboSimulator):
+
+            assert gazebo.initialize()
+            ...
+
+        @pytest.mark.parametrize("gazebo, name",
+                                 [((0.001, 1.0, 1), "name_1"),
+                                  ((0.001, 1.0, 1), "name_2"),
+                                  ((0.001, 1.0, 1), "name_3")],
+                                  indirect="gazebo")
+        def test_foo(gazebo: bindings.GazeboSimulator, name: str):
+
+            model_name = name
+            assert gazebo.initialize()
+            ...
+    """
 
     step_size, rtf, iterations = request.param
     gazebo = bindings.GazeboSimulator(step_size, rtf, iterations)
@@ -35,11 +58,7 @@ def get_multi_world_sdf_file() -> str:
         </world>
     </sdf>"""
 
-    handle, multi_world_sdf = tempfile.mkstemp()
-
-    with open(handle, 'w') as f:
-        f.write(multi_world_sdf_string)
-
+    multi_world_sdf = misc.string_to_file(multi_world_sdf_string)
     return multi_world_sdf
 
 
@@ -76,20 +95,13 @@ def get_cube_urdf_string() -> str:
 
 def get_cube_urdf() -> str:
 
-    handle, model_file = tempfile.mkstemp()
-    with open(handle, 'w') as f:
-        f.write(get_cube_urdf_string())
-
+    model_file = misc.string_to_file(get_cube_urdf_string())
     return model_file
 
 
 def get_empty_world_sdf() -> str:
 
     world_sdf_string = bindings.getEmptyWorld()
 
-    handle, world_file = tempfile.mkstemp()
-
-    with open(handle, 'w') as f:
-        f.write(world_sdf_string)
-
+    world_file = misc.string_to_file(world_sdf_string)
     return world_file
diff --git a/tests/pytest.ini b/tests/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+addopts = -rsxX -v
+markers =
+    gympp: Select the tests in the 'gympp' folder
+    scenario: Select the tests in the 'scenario' folder
+    gym_ignition: Select the tests in the 'gym_ignition' folder
diff --git a/tests/test_gym_ignition/__init__.py b/tests/test_gym_ignition/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2019 Istituto Italiano di Tecnologia (IIT). All rights reserved.
+# This software may be modified and distributed under the terms of the
+# GNU Lesser General Public License v2.1 or any later version.
diff --git a/tests/test_gym_ignition/test_reproducibility.py b/tests/test_gym_ignition/test_reproducibility.py
@@ -0,0 +1,12 @@
+# Copyright (C) 2020 Istituto Italiano di Tecnologia (IIT). All rights reserved.
+# This software may be modified and distributed under the terms of the
+# GNU Lesser General Public License v2.1 or any later version.
+
+import pytest
+pytestmark = pytest.mark.gym_ignition
+
+
+@pytest.mark.xfail
+def test_reproducibility():
+
+    assert False
diff --git a/tests/scenario/__init__.py → tests/test_scenario/__init__.py b/tests/scenario/__init__.py → tests/test_scenario/__init__.py
diff --git a/tests/scenario/test_custom_controllers.py → .../test_scenario/test_custom_controllers.py b/tests/scenario/test_custom_controllers.py → .../test_scenario/test_custom_controllers.py
@@ -3,10 +3,12 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
+pytestmark = pytest.mark.scenario
+
 import numpy as np
-from . import utils
 import gym_ignition_models
-from .utils import gazebo_fixture as gazebo
+from ..common import utils
+from ..common.utils import gazebo_fixture as gazebo
 from gym_ignition import scenario_bindings as bindings
 from gym_ignition.controllers.gazebo import computed_torque_fixed_base as context
 
@@ -27,7 +29,8 @@ def test_computed_torque_fixed_base(gazebo: bindings.GazeboSimulator):
     world = gazebo.getWorld()
 
     # Insert the physics
-    world.insertWorldPlugin("libPhysicsSystem.so", "scenario::plugins::gazebo::Physics")
+    assert world.insertWorldPlugin("libPhysicsSystem.so",
+                                   "scenario::plugins::gazebo::Physics")
 
     # Get the panda urdf
     panda_urdf = gym_ignition_models.get_model_file("panda")

diff --git a/tests/scenario/test_gazebo_simulator.py → tests/test_scenario/test_gazebo_simulator.py b/tests/scenario/test_gazebo_simulator.py → tests/test_scenario/test_gazebo_simulator.py
@@ -3,8 +3,10 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
-from . import utils
-from .utils import gazebo_fixture as gazebo
+pytestmark = pytest.mark.scenario
+
+from ..common import utils
+from ..common.utils import gazebo_fixture as gazebo
 from gym_ignition import scenario_bindings as bindings
 
 # Set the verbosity

diff --git a/tests/scenario/test_ignition_fuel.py → tests/test_scenario/test_ignition_fuel.py b/tests/scenario/test_ignition_fuel.py → tests/test_scenario/test_ignition_fuel.py
@@ -3,8 +3,10 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
-from . import utils
-from .utils import gazebo_fixture as gazebo
+pytestmark = pytest.mark.scenario
+
+from ..common import utils
+from ..common.utils import gazebo_fixture as gazebo
 from gym_ignition import scenario_bindings as bindings
 
 # Set the verbosity

diff --git a/tests/scenario/test_model.py → tests/test_scenario/test_model.py b/tests/scenario/test_model.py → tests/test_scenario/test_model.py
@@ -3,9 +3,11 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
-from . import utils
+pytestmark = pytest.mark.scenario
+
+from ..common import utils
 import gym_ignition_models
-from .utils import gazebo_fixture as gazebo
+from ..common.utils import gazebo_fixture as gazebo
 from gym_ignition import scenario_bindings as bindings
 
 # Set the verbosity
@@ -18,7 +20,8 @@ def get_model(gazebo: bindings.GazeboSimulator,
     world = gazebo.getWorld()
     # TODO: assert world
 
-    world.insertWorldPlugin("libPhysicsSystem.so", "scenario::plugins::gazebo::Physics")
+    assert world.insertWorldPlugin("libPhysicsSystem.so",
+                                   "scenario::plugins::gazebo::Physics")
 
     model_urdf = gym_ignition_models.get_model_file(gym_ignition_model_name)
     assert world.insertModel(model_urdf,

diff --git a/tests/scenario/test_multi_world.py → tests/test_scenario/test_multi_world.py b/tests/scenario/test_multi_world.py → tests/test_scenario/test_multi_world.py
@@ -3,8 +3,10 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
-from . import utils
-from .utils import gazebo_fixture as gazebo
+pytestmark = pytest.mark.scenario
+
+from ..common import utils
+from ..common.utils import gazebo_fixture as gazebo
 from gym_ignition import scenario_bindings as bindings
 
 # Set the verbosity

diff --git a/tests/scenario/test_world.py → tests/test_scenario/test_world.py b/tests/scenario/test_world.py → tests/test_scenario/test_world.py
@@ -3,8 +3,10 @@
 # GNU Lesser General Public License v2.1 or any later version.
 
 import pytest
-from . import utils
-from .utils import gazebo_fixture as gazebo
+pytestmark = pytest.mark.scenario
+
+from ..common import utils
+from ..common.utils import gazebo_fixture as gazebo
 from gym_ignition import scenario_bindings as bindings
 
 # Set the verbosity
@@ -32,7 +34,7 @@ def test_load_default_world(gazebo: bindings.GazeboSimulator):
 def test_load_default_world_from_file(gazebo: bindings.GazeboSimulator):
 
     empty_world_sdf = utils.get_empty_world_sdf()
-    print(empty_world_sdf)
+
     assert gazebo.insertWorldFromSDF(empty_world_sdf)
 
     assert gazebo.initialize()
@@ -162,7 +164,8 @@ def test_world_physics_plugin(gazebo: bindings.GazeboSimulator):
     assert world.time() == 0
 
     # Insert the Physics system
-    world.insertWorldPlugin("libPhysicsSystem.so", "scenario::plugins::gazebo::Physics")
+    assert world.insertWorldPlugin("libPhysicsSystem.so",
+                                   "scenario::plugins::gazebo::Physics")
 
     # After the first step, the physics catches up with time
     gazebo.run()
@@ -191,7 +194,8 @@ def test_sim_time_starts_from_zero(gazebo: bindings.GazeboSimulator):
     dt = gazebo.stepSize()
 
     assert world.time() == 0
-    world.insertWorldPlugin("libPhysicsSystem.so", "scenario::plugins::gazebo::Physics")
+    assert world.insertWorldPlugin("libPhysicsSystem.so",
+                                   "scenario::plugins::gazebo::Physics")
     assert world.time() == 0
 
     gazebo.run(paused=True)