From e24b48e9500a28273e28e047936826d5aa5c8fa5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 10 Mar 2022 17:16:09 +0100 Subject: [PATCH 1/7] Move builtin definitions to a different file --- reframe/core/builtins.py | 137 ++++++++++++++++++++++++++++ reframe/core/meta.py | 178 +++++++++---------------------------- unittests/test_pipeline.py | 26 ++++++ 3 files changed, 207 insertions(+), 134 deletions(-) create mode 100644 reframe/core/builtins.py diff --git a/reframe/core/builtins.py b/reframe/core/builtins.py new file mode 100644 index 0000000000..b0e3809054 --- /dev/null +++ b/reframe/core/builtins.py @@ -0,0 +1,137 @@ +# Copyright 2016-2022 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +# +# Regression test class builtins +# + +import functools +import reframe.core.parameters as parameters +import reframe.core.variables as variables +import reframe.core.fixtures as fixtures +import reframe.core.hooks as hooks +import reframe.utility as utils +from reframe.core.deferrable import deferrable, _DeferredPerformanceExpression + + +__all__ = ['deferrable', 'deprecate', 'final', 'fixture', 'loggable', + 'loggable_as', 'parameter', 'performance_function', 'required', + 'require_deps', 'run_before', 'run_after', 'sanity_function', + 'variable'] + +parameter = parameters.TestParam +variable = variables.TestVar +required = variables.Undefined +deprecate = variables.TestVar.create_deprecated +fixture = fixtures.TestFixture + + +def final(fn): + '''Indicate that a function is final and cannot be overridden.''' + + fn._rfm_final = True + return fn + + +# Hook-related builtins + +def run_before(stage): + '''Decorator for attaching a test method to a given stage. + + See online docs for more information. + ''' + return hooks.attach_to('pre_' + stage) + + +def run_after(stage): + '''Decorator for attaching a test method to a given stage. + + See online docs for more information. + ''' + return hooks.attach_to('post_' + stage) + + +require_deps = hooks.require_deps + + +# Sanity and performance function builtins + +def sanity_function(fn): + '''Mark a function as the test's sanity function. + + Decorated functions must be unary and they will be converted into + deferred expressions. + ''' + + _def_fn = deferrable(fn) + setattr(_def_fn, '_rfm_sanity_fn', True) + return _def_fn + + +def performance_function(units, *, perf_key=None): + '''Decorate a function to extract a performance variable. + + The ``units`` argument indicates the units of the performance + variable to be extracted. + The ``perf_key`` optional arg will be used as the name of the + performance variable. If not provided, the function name will + be used as the performance variable name. + ''' + if not isinstance(units, str): + raise TypeError('performance units must be a string') + + if perf_key and not isinstance(perf_key, str): + raise TypeError("'perf_key' must be a string") + + def _deco_wrapper(func): + if not utils.is_trivially_callable(func, non_def_args=1): + raise TypeError( + f'performance function {func.__name__!r} has more ' + f'than one argument without a default value' + ) + + @functools.wraps(func) + def _perf_fn(*args, **kwargs): + return _DeferredPerformanceExpression( + func, units, *args, **kwargs + ) + + _perf_key = perf_key if perf_key else func.__name__ + setattr(_perf_fn, '_rfm_perf_key', _perf_key) + return _perf_fn + + return _deco_wrapper + + +def loggable_as(name): + '''Mark a property as loggable. + + :param name: An alternative name that will be used for logging + this property. If :obj:`None`, the name of the decorated + property will be used. + :raises ValueError: if the decorated function is not a property. + + .. versionadded:: 3.10.2 + + :meta private: + + ''' + def _loggable(fn): + if not hasattr(fn, 'fget'): + raise ValueError('decorated function does not ' + 'look like a property') + + # Mark property as loggable + # + # NOTE: Attributes cannot be set on property objects, so we + # set the attribute on one of its functions + prop_name = fn.fget.__name__ + fn.fget._rfm_loggable = (prop_name, name) + return fn + + return _loggable + + +loggable = loggable_as(None) diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 6c1393aceb..0d8d58afbd 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -11,6 +11,7 @@ import types import collections +import reframe.core.builtins as builtins import reframe.core.namespaces as namespaces import reframe.core.parameters as parameters import reframe.core.variables as variables @@ -18,13 +19,11 @@ import reframe.core.hooks as hooks import reframe.utility as utils -from reframe.core.exceptions import ReframeSyntaxError -from reframe.core.deferrable import deferrable, _DeferredPerformanceExpression +from reframe.core.exceptions import ReframeSyntaxError, ReframeFatalError from reframe.core.runtime import runtime class RegressionTestMeta(type): - class MetaNamespace(namespaces.LocalNamespace): '''Custom namespace to control the cls attribute assignment. @@ -210,6 +209,10 @@ def __setattr__(self, name, value): def __prepare__(metacls, name, bases, **kwargs): namespace = super().__prepare__(name, bases, **kwargs) + # + # Initialize the various class level helper data structures + # + # Keep reference to the bases inside the namespace namespace['_rfm_bases'] = [ b for b in bases if hasattr(b, '_rfm_var_space') @@ -218,30 +221,23 @@ def __prepare__(metacls, name, bases, **kwargs): # Regression test parameter space defined at the class level namespace['_rfm_local_param_space'] = namespaces.LocalNamespace() - # Directive to insert a regression test parameter directly in the - # class body as: `P0 = parameter([0,1,2,3])`. - namespace['parameter'] = parameters.TestParam - # Regression test var space defined at the class level namespace['_rfm_local_var_space'] = namespaces.LocalNamespace() - # Directives to add/modify a regression test variable - namespace['variable'] = variables.TestVar - namespace['required'] = variables.Undefined - namespace['deprecate'] = variables.TestVar.create_deprecated - # Regression test fixture space namespace['_rfm_local_fixture_space'] = namespaces.LocalNamespace() - # Directive to add a fixture - namespace['fixture'] = fixtures.TestFixture - # Utility decorators namespace['_rfm_ext_bound'] = set() - # Loggable attributes and properties + # Loggable properties namespace['_rfm_loggable_props'] = [] + namespace['_rfm_final_methods'] = set() + namespace['_rfm_hook_registry'] = hooks.HookRegistry() + namespace['_rfm_local_hook_registry'] = hooks.HookRegistry() + namespace['_rfm_perf_fns'] = namespaces.LocalNamespace() + def bind(fn, name=None): '''Directive to bind a free function to a class. @@ -268,134 +264,33 @@ def bind(fn, name=None): namespace['_rfm_ext_bound'].add(inst.__name__) return inst - def final(fn): - '''Indicate that a function is final and cannot be overridden.''' - - fn._rfm_final = True - return fn - - def loggable_as(name): - '''Mark a property loggable. - - :param name: An alternative name that will be used for logging - this property. If :obj:`None`, the name of the decorated - property will be used. - :raises ValueError: if the decorated function is not a property. - - .. versionadded:: 3.10.2 - - :meta private: - - ''' - def _loggable(fn): - if not hasattr(fn, 'fget'): - raise ValueError('decorated function does not ' - 'look like a property') - - prop_name = fn.fget.__name__ - namespace['_rfm_loggable_props'].append((prop_name, name)) - return fn - - return _loggable + # Register all builtins + for name in builtins.__all__: + namespace[name] = getattr(builtins, name) namespace['bind'] = bind - namespace['final'] = final - namespace['loggable'] = loggable_as(None) - namespace['loggable_as'] = loggable_as - namespace['_rfm_final_methods'] = set() - - # Hook-related functionality - def run_before(stage): - '''Decorator for attaching a test method to a given stage. - - See online docs for more information. - ''' - return hooks.attach_to('pre_' + stage) - - def run_after(stage): - '''Decorator for attaching a test method to a given stage. - - See online docs for more information. - ''' - return hooks.attach_to('post_' + stage) - - namespace['run_before'] = run_before - namespace['run_after'] = run_after - namespace['require_deps'] = hooks.require_deps - namespace['_rfm_hook_registry'] = hooks.HookRegistry() - namespace['_rfm_local_hook_registry'] = hooks.HookRegistry() - - # Machinery to add a sanity function - def sanity_function(fn): - '''Mark a function as the test's sanity function. - - Decorated functions must be unary and they will be converted into - deferred expressions. - ''' - - _def_fn = deferrable(fn) - setattr(_def_fn, '_rfm_sanity_fn', True) - return _def_fn - - namespace['sanity_function'] = sanity_function - namespace['deferrable'] = deferrable - - # Machinery to add performance functions - def performance_function(units, *, perf_key=None): - '''Decorate a function to extract a performance variable. - - The ``units`` argument indicates the units of the performance - variable to be extracted. - The ``perf_key`` optional arg will be used as the name of the - performance variable. If not provided, the function name will - be used as the performance variable name. - ''' - if not isinstance(units, str): - raise TypeError('performance units must be a string') - - if perf_key and not isinstance(perf_key, str): - raise TypeError("'perf_key' must be a string") - - def _deco_wrapper(func): - if not utils.is_trivially_callable(func, non_def_args=1): - raise TypeError( - f'performance function {func.__name__!r} has more ' - f'than one argument without a default value' - ) - - @functools.wraps(func) - def _perf_fn(*args, **kwargs): - return _DeferredPerformanceExpression( - func, units, *args, **kwargs - ) - - _perf_key = perf_key if perf_key else func.__name__ - setattr(_perf_fn, '_rfm_perf_key', _perf_key) - return _perf_fn - - return _deco_wrapper - - namespace['performance_function'] = performance_function - namespace['_rfm_perf_fns'] = namespaces.LocalNamespace() return metacls.MetaNamespace(namespace) def __new__(metacls, name, bases, namespace, **kwargs): - '''Remove directives from the class namespace. + '''Remove builtins from the class namespace. - It does not make sense to have some directives available after the - class was created or even at the instance level (e.g. doing + It does not make sense to have the builtins available after the class + was created or even at the instance level (e.g. doing ``self.parameter([1, 2, 3])`` does not make sense). So here, we - intercept those directives out of the namespace before the class is + intercept those builtins out of the namespace before the class is constructed. + ''' - directives = [ - 'parameter', 'variable', 'bind', 'run_before', 'run_after', - 'require_deps', 'required', 'deferrable', 'sanity_function', - 'final', 'performance_function', 'fixture' + # Collect the loggable properties + loggable_props = [] + namespace['_rfm_loggable_props'] = [ + v.fget._rfm_loggable for v in namespace.values() + if hasattr(v, 'fget') and hasattr(v.fget, '_rfm_loggable') ] - for b in directives: - namespace.pop(b) + + for n in builtins.__all__ + ['bind']: + namespace.pop(n) # Reset the external functions imported through the bind directive. for item in namespace.pop('_rfm_ext_bound'): @@ -913,7 +808,7 @@ def loggable_attrs(cls): return sorted(loggable_props + loggable_vars + loggable_params) -def make_test(name, bases, body, **kwargs): +def make_test(name, bases, body, methods=None, **kwargs): '''Define a new test class programmatically. Using this method is completely equivalent to using the :keyword:`class` @@ -948,13 +843,28 @@ class HelloTest(rfm.RunOnlyRegressionTest): created. :param body: A mapping of key/value pairs that will be inserted as class attributes in the newly created class. + :param methods: A list of functions to be added as methods to the class + that is being created. :param kwargs: Any keyword arguments to be passed to the :class:`RegressionTestMeta` metaclass. .. versionadded:: 3.10.0 + .. versionchanged:: 3.11.0 + Added the ``methods`` arguments. + ''' namespace = RegressionTestMeta.__prepare__(name, bases, **kwargs) + methods = methods or [] + + # Add methods to the body + for m in methods: + body[m.__name__] = m + namespace.update(body) + for k in list(namespace.keys()): + namespace.reset(k) + + # namespace.update(body) cls = RegressionTestMeta(name, bases, namespace, **kwargs) return cls diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index a62570c0d1..7ff1103089 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -9,6 +9,7 @@ import sys import reframe as rfm +import reframe.core.builtins as builtins import reframe.core.runtime as rt import reframe.utility.osext as osext import reframe.utility.sanity as sn @@ -1542,6 +1543,31 @@ def validate(self): _run(hello_cls(), *local_exec_ctx) +def test_make_test_with_builtins_inline(local_exec_ctx): + def set_message(obj): + obj.executable_opts = [obj.message] + + def validate(obj): + return sn.assert_found(obj.message, obj.stdout) + + hello_cls = make_test( + 'HelloTest', (rfm.RunOnlyRegressionTest,), + { + 'valid_systems': ['*'], + 'valid_prog_environs': ['*'], + 'executable': 'echo', + 'message': builtins.variable(str), + }, + methods=[ + builtins.run_before('run')(set_message), + builtins.sanity_function(validate) + ] + ) + hello_cls.setvar('message', 'hello') + assert hello_cls.__name__ == 'HelloTest' + _run(hello_cls(), *local_exec_ctx) + + def test_set_var_default(): class _X(rfm.RunOnlyRegressionTest): foo = variable(int, value=10) From b5ad3d192e9748ece5eb31026804cec1d87d05db Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 12 Mar 2022 15:07:27 +0100 Subject: [PATCH 2/7] Move builtin docs to the code --- docs/regression_test_api.rst | 597 +++-------------------------------- reframe/core/builtins.py | 108 +++++-- reframe/core/deferrable.py | 8 +- reframe/core/fixtures.py | 374 +++++++++++++++++++--- reframe/core/hooks.py | 23 +- reframe/core/parameters.py | 121 ++++++- reframe/core/variables.py | 145 ++++++++- 7 files changed, 742 insertions(+), 634 deletions(-) diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index 664670467e..f38c3a548c 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -28,545 +28,73 @@ Test Decorators .. autodecorator:: reframe.core.decorators.simple_test --------------- -Built-in types --------------- - -.. versionadded:: 3.4.2 - -ReFrame provides built-in types which facilitate the process of writing extensible regression tests (i.e. a test library). -These *builtins* are only available when used directly in the class body of classes derived from any of the :ref:`regression-bases`. -Through builtins, ReFrame internals are able to *pre-process* and validate the test input before the actual test creation takes place. -This provides the ReFrame internals with further control over the user's input, making the process of writing regression tests less error-prone. -In essence, these builtins exert control over the test creation, and they allow adding and/or modifying certain attributes of the regression test. - -.. note:: - The built-in types described below can only be used to declare class variables and must never be part of any container type. - Ignoring this restriction will result in undefined behavior. - - .. code:: - - class MyTest(rfm.RegressionMixin): - p0 = parameter([1, 2]) # Correct - p1 = [parameter([1, 2])] # Undefined behavior - - -.. py:function:: RegressionMixin.parameter(values=None, inherit_params=False, filter_params=None, fmt=None, loggable=False) - - Inserts or modifies a regression test parameter. - At the class level, these parameters are stored in a separate namespace referred to as the *parameter space*. - If a parameter with a matching name is already present in the parameter space of a parent class, the existing parameter values will be combined with those provided by this method following the inheritance behavior set by the arguments ``inherit_params`` and ``filter_params``. - Instead, if no parameter with a matching name exists in any of the parent parameter spaces, a new regression test parameter is created. - A regression test can be parameterized as follows: - - .. code:: python - - class Foo(rfm.RegressionTest): - variant = parameter(['A', 'B']) - # print(variant) # Error: a parameter may only be accessed from the class instance. - - @run_after('init') - def do_something(self): - if self.variant == 'A': - do_this() - else: - do_other() - - One of the most powerful features of these built-in functions is that they store their input information at the class level. - However, a parameter may only be accessed from the class instance and accessing it directly from the class body is disallowed. - With this approach, extending or specializing an existing parameterized regression test becomes straightforward, since the test attribute additions and modifications made through built-in functions in the parent class are automatically inherited by the child test. - For instance, continuing with the example above, one could override the :func:`do_something` hook in the :class:`Foo` regression test as follows: - - .. code:: python - - class Bar(Foo): - @run_after('init') - def do_something(self): - if self.variant == 'A': - override_this() - else: - override_other() - - Moreover, a derived class may extend, partially extend and/or modify the parameter values provided in the base class as shown below. - - .. code:: python - - class ExtendVariant(Bar): - # Extend the full set of inherited variant parameter values to ['A', 'B', 'C'] - variant = parameter(['C'], inherit_params=True) - - class PartiallyExtendVariant(Bar): - # Extend a subset of the inherited variant parameter values to ['A', 'D'] - variant = parameter(['D'], inherit_params=True, - filter_params=lambda x: x[:1]) - - class ModifyVariant(Bar): - # Modify the variant parameter values to ['AA', 'BA'] - variant = parameter(inherit_params=True, - filter_params=lambda x: map(lambda y: y+'A', x)) - - A parameter with no values is referred to as an *abstract parameter* (i.e. a parameter that is declared but not defined). - Therefore, classes with at least one abstract parameter are considered abstract classes. - - .. code:: python - - class AbstractA(Bar): - variant = parameter() - - class AbstractB(Bar): - variant = parameter(inherit_params=True, filter_params=lambda x: []) - - - :param values: An iterable containing the parameter values. - :param inherit_params: If :obj:`True`, the parameter values defined in any base class will be inherited. - In this case, the parameter values provided in the current class will extend the set of inherited parameter values. - If the parameter does not exist in any of the parent parameter spaces, this option has no effect. - :param filter_params: Function to filter/modify the inherited parameter values that may have been provided in any of the parent parameter spaces. - This function must accept a single iterable argument and return an iterable. - It will be called with the inherited parameter values and it must return the filtered set of parameter values. - This function will only have an effect if used with ``inherit_params=True``. - :param fmt: A formatting function that will be used to format the values of this parameter in the test's :attr:`~reframe.core.pipeline.RegressionTest.display_name`. - This function should take as argument the parameter value and return a string representation of the value. - If the returned value is not a string, it will be converted using the :py:func:`str` function. - :param loggable: Mark this parameter as loggable. - If :obj:`True`, this parameter will become a log record attribute under the name ``check_NAME``, where ``NAME`` is the name of the parameter. - - - .. versionadded:: 3.10.0 - The ``fmt`` argument is added. - - .. versionadded:: 3.11.0 - The ``loggable`` argument is added. - - -.. py:function:: RegressionMixin.variable(*types, value=None, field=None, **kwargs) - - Inserts a new regression test variable. - Declaring a test variable through the :func:`variable` built-in allows for a more robust test implementation than if the variables were just defined as regular test attributes (e.g. ``self.a = 10``). - Using variables declared through the :func:`variable` built-in guarantees that these regression test variables will not be redeclared by any child class, while also ensuring that any values that may be assigned to such variables comply with its original declaration. - In essence, declaring test variables with the :func:`variable` built-in removes any potential test errors that might be caused by accidentally overriding a class attribute. See the example below. - - - .. code:: python - - class Foo(rfm.RegressionTest): - my_var = variable(int, value=8) - not_a_var = my_var - 4 - - @run_after('init') - def access_vars(self): - print(self.my_var) # prints 8. - # self.my_var = 'override' # Error: my_var must be an int! - self.not_a_var = 'override' # However, this would work. Dangerous! - self.my_var = 10 # tests may also assign values the standard way - - Here, the argument ``value`` in the :func:`variable` built-in sets the default value for the variable. - This value may be accessed directly from the class body, as long as it was assigned before either in the same class body or in the class body of a parent class. - This behavior extends the standard Python data model, where a regular class attribute from a parent class is never available in the class body of a child class. - Hence, using the :func:`variable` built-in enables us to directly use or modify any variables that may have been declared upstream the class inheritance chain, without altering their original value at the parent class level. - - .. code:: python - - class Bar(Foo): - print(my_var) # prints 8 - # print(not_a_var) # This is standard Python and raises a NameError - - # Since my_var is available, we can also update its value: - my_var = 4 - - # Bar inherits the full declaration of my_var with the original type-checking. - # my_var = 'override' # Wrong type error again! - - @run_after('init') - def access_vars(self): - print(self.my_var) # prints 4 - print(self.not_a_var) # prints 4 - - - print(Foo.my_var) # prints 8 - print(Bar.my_var) # prints 4 - - - Here, :class:`Bar` inherits the variables from :class:`Foo` and can see that ``my_var`` has already been declared in the parent class. Therefore, the value of ``my_var`` is updated ensuring that the new value complies to the original variable declaration. - However, the value of ``my_var`` at :class:`Foo` remains unchanged. - - These examples above assumed that a default value can be provided to the variables in the bases tests, but that might not always be the case. - For example, when writing a test library, one might want to leave some variables undefined and force the user to set these when using the test. - As shown in the example below, imposing such requirement is as simple as not passing any ``value`` to the :func:`variable` built-in, which marks the given variable as *required*. - - .. code:: python - - # Test as written in the library - class EchoBaseTest(rfm.RunOnlyRegressionTest): - what = variable(str) - - valid_systems = ['*'] - valid_prog_environs = ['*'] - - @run_before('run') - def set_executable(self): - self.executable = f'echo {self.what}' - - @sanity_function - def assert_what(self): - return sn.assert_found(fr'{self.what}') - - - # Test as written by the user - @rfm.simple_test - class HelloTest(EchoBaseTest): - what = 'Hello' +.. _builtins: +-------- +Builtins +-------- - # A parameterized test with type-checking - @rfm.simple_test - class FoodTest(EchoBaseTest): - param = parameter(['Bacon', 'Eggs']) - - @run_after('init') - def set_vars_with_params(self): - self.what = self.param - - - Similarly to a variable with a value already assigned to it, the value of a required variable may be set either directly in the class body, on the :func:`__init__` method, or in any other hook before it is referenced. - Otherwise an error will be raised indicating that a required variable has not been set. - Conversely, a variable with a default value already assigned to it can be made required by assigning it the ``required`` keyword. - However, this ``required`` keyword is only available in the class body. - - .. code:: python - - class MyRequiredTest(HelloTest): - what = required - - - Running the above test will cause the :func:`set_exec_and_sanity` hook from :class:`EchoBaseTest` to throw an error indicating that the variable ``what`` has not been set. - - :param `*types`: the supported types for the variable. - :param value: the default value assigned to the variable. If no value is provided, the variable is set as ``required``. - :param field: the field validator to be used for this variable. - If no field argument is provided, it defaults to :attr:`reframe.core.fields.TypedField`. - The provided field validator by this argument must derive from :attr:`reframe.core.fields.Field`. - :param loggable: Mark this variable as loggable. - If :obj:`True`, this variable will become a log record attribute under the name ``check_NAME``, where ``NAME`` is the name of the variable. - :param `**kwargs`: *kwargs* to be forwarded to the constructor of the field validator. - - .. versionadded:: 3.10.2 - The ``loggable`` argument is added. - - -.. py:function:: RegressionMixin.fixture(cls, *, scope='test', action='fork', variants='all', variables=None) - - Declare a new fixture in the current regression test. - A fixture is a regression test that creates, prepares and/or manages a resource for another regression test. - Fixtures may contain other fixtures and so on, forming a directed acyclic graph. - A parent fixture (or a regular regression test) requires the resources managed by its child fixtures in order to run, and it may only access these fixture resources after its ``setup`` pipeline stage. - The execution of parent fixtures is postponed until all their respective children have completed execution. - However, the destruction of the resources managed by a fixture occurs in reverse order, only after all the parent fixtures have been destroyed. - This destruction of resources takes place during the ``cleanup`` pipeline stage of the regression test. - Fixtures must not define the members :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs`. - These variables are defined based on the values specified in the parent test, ensuring that the fixture runs with a suitable system partition and programming environment combination. - A fixture's :attr:`~reframe.core.pipeline.RegressionTest.name` attribute may be internally mangled depending on the arguments passed during the fixture declaration. - Hence, manually setting or modifying the :attr:`~reframe.core.pipeline.RegressionTest.name` attribute in the fixture class is disallowed, and breaking this restriction will result in undefined behavior. - - .. warning:: - The fixture name mangling is considered an internal framework mechanism and it may change in future versions without any notice. - Users must not express any logic in their tests that relies on a given fixture name mangling scheme. - - - By default, the resources managed by a fixture are private to the parent test. - However, it is possible to share these resources across different tests by passing the appropriate fixture ``scope`` argument. - The different scope levels are independent from each other and a fixture only executes once per scope, where all the tests that belong to that same scope may use the same resources managed by a given fixture instance. - The available scopes are: - - * **session**: This scope encloses all the tests and fixtures that run in the full ReFrame session. - This may include tests that use different system partition and programming environment combinations. - The fixture class must derive from :class:`~reframe.core.pipeline.RunOnlyRegressionTest` to avoid any implicit dependencies on the partition or the programming environment used. - * **partition**: This scope spans across a single system partition. - This may include different tests that run on the same partition but use different programming environments. - Fixtures with this scope must be independent of the programming environment, which restricts the fixture class to derive from :class:`~reframe.core.pipeline.RunOnlyRegressionTest`. - * **environment**: The extent of this scope covers a single combination of system partition and programming environment. - Since the fixture is guaranteed to have the same partition and programming environment as the parent test, the fixture class can be any derived class from :class:`~reframe.core.pipeline.RegressionTest`. - * **test**: This scope covers a single instance of the parent test, where the resources provided by the fixture are exclusive to each parent test instance. - The fixture class can be any derived class from :class:`~reframe.core.pipeline.RegressionTest`. - - Rather than specifying the scope at the fixture class definition, ReFrame fixtures set the scope level from the consumer side (i.e. when used by another test or fixture). - A test may declare multiple fixtures using the same class, where fixtures with different scopes are guaranteed to point to different instances of the fixture class. - On the other hand, when two or more fixtures use the same fixture class and have the same scope, these different fixtures will point to the same underlying resource if the fixtures refer to the same :ref:`variant` of the fixture class. - The example below illustrates the different fixture scope usages: - - .. code:: python - - class MyFixture(rfm.RunOnlyRegressionTest): - '''Manage some resource''' - my_var = variable(int, value=1) - ... - - - @rfm.simple_test - class TestA(rfm.RegressionTest): - valid_systems = ['p1', 'p2'] - valid_prog_environs = ['e1', 'e2'] - f1 = fixture(MyFixture, scope='session') # Shared throughout the full session - f2 = fixture(MyFixture, scope='partition') # Shared for each supported partition - f3 = fixture(MyFixture, scope='environment') # Shared for each supported part+environ - f4 = fixture(MyFixture, scope='test') # Private evaluation of MyFixture - ... - - - @rfm.simple_test - class TestB(rfm.RegressionTest): - valid_systems = ['p1'] - valid_prog_environs = ['e1'] - f1 = fixture(MyFixture, scope='test') # Another private instance of MyFixture - f2 = fixture(MyFixture, scope='environment') # Same as f3 in TestA for p1 + e1 - f3 = fixture(MyFixture, scope='session') # Same as f1 in TestA - ... - - @run_after('setup') - def access_fixture_resources(self): - '''Dummy pipeline hook to illustrate fixture resource access.''' - assert self.f1.my_var is not self.f2.my_var - assert self.f1.my_var is not self.f3.my_var - - - :class:`TestA` supports two different valid systems and another two valid programming environments. - Assuming that both environments are supported by each of the system partitions ``'p1'`` and ``'p2'``, this test will execute a total of four times. - This test uses the very simple :class:`MyFixture` fixture multiple times using different scopes, where fixture ``f1`` (session scope) will be shared across the four test instances, and fixture ``f4`` (test scope) will be executed once per test instance. - On the other hand, ``f2`` (partition scope) will run once per partition supported by test :class:`TestA`, and the multiple per-partition executions (i.e. for each programming environment) will share the same underlying resource for ``f2``. - Lastly, ``f3`` will run a total of four times, which is once per partition and environment combination. - This simple :class:`TestA` shows how multiple instances from the same test can share resources, but the real power behind fixtures is illustrated with :class:`TestB`, where this resource sharing is extended across different tests. - For simplicity, :class:`TestB` only supports a single partition ``'p1'`` and programming environment ``'e1'``, and similarly to :class:`TestA`, ``f1`` (test scope) causes a private evaluation of the fixture :class:`MyFixture`. - However, the resources managed by fixtures ``f2`` (environment scope) and ``f3`` (session scope) are shared with :class:`Test1`. - - Fixtures are treated by ReFrame as first-class ReFrame tests, which means that these classes can use the same built-in functionalities as in regular tests decorated with :func:`@rfm.simple_test`. - This includes the :func:`~reframe.core.pipeline.RegressionMixin.parameter` built-in, where fixtures may have more than one :ref:`variant`. - When this occurs, a parent test may select to either treat a parameterized fixture as a test parameter, or instead, to gather all the fixture variants from a single instance of the parent test. - In essence, fixtures implement `fork-join` model whose behavior may be controlled through the ``action`` argument. - This argument may be set to one of the following options: - - * **fork**: This option parameterizes the parent test as a function of the fixture variants. - The fixture handle will resolve to a single instance of the fixture. - * **join**: This option gathers all the variants from a fixture into a single instance of the parent test. - The fixture handle will point to a list containing all the fixture variants. - - A test may declare multiple fixtures with different ``action`` options, where the default ``action`` option is ``'fork'``. - The example below illustrates the behavior of these two different options. - - .. code:: python - - class ParamFix(rfm.RegressionTest): - '''Manage some resource''' - p = parameter(range(5)) # A simple test parameter - ... - - - @rfm.simple_test - class TestC(rfm.RegressionTest): - # Parameterize TestC for each ParamFix variant - f = fixture(ParamFix, action='fork') - ... - - @run_after('setup') - def access_fixture_resources(self): - print(self.f.p) # Prints the fixture's variant parameter value - - - @rfm.simple_test - class TestD(rfm.RegressionTest): - # Gather all fixture variants into a single test - f = fixture(ParamFix, action='join') - ... - - @run_after('setup') - def reduce_range(self): - '''Sum all the values of p for each fixture variant''' - res = functools.reduce(lambda x, y: x+y, (fix.p for fix in self.f)) - n = len(self.f)-1 - assert res == (n*n + n)/2 - - Here :class:`ParamFix` is a simple fixture class with a single parameter. - When the test :class:`TestC` uses this fixture with a ``'fork'`` action, the test is implicitly parameterized over each variant of :class:`ParamFix`. - Hence, when the :func:`access_fixture_resources` post-setup hook accesses the fixture ``f``, it only access a single instance of the :class:`ParamFix` fixture. - On the other hand, when this same fixture is used with a ``'join'`` action by :class:`TestD`, the test is not parameterized and all the :class:`ParamFix` instances are gathered into ``f`` as a list. - Thus, the post-setup pipeline hook :func:`reduce_range` can access all the fixture variants and compute a reduction of the different ``p`` values. - - When declaring a fixture, a parent test may select a subset of the fixture variants through the ``variants`` argument. - This variant selection can be done by either passing an iterable containing valid variant indices (see :ref:`test-variants` for further information on how the test variants are indexed), or instead, passing a mapping with the parameter name (of the fixture class) as keys and filtering functions as values. - These filtering functions are unary functions that return the value of a boolean expression on the values of the specified parameter, and they all must evaluate to :class:`True` for at least one of the fixture class variants. - See the example below for an illustration on how to filter-out fixture variants. - - .. code:: python - - class ComplexFixture(rfm.RegressionTest): - # A fixture with 400 different variants. - p0 = parameter(range(100)) - p1 = parameter(['a', 'b', 'c', 'd']) - ... - - @rfm.simple_test - class TestE(rfm.RegressionTest): - # Select the fixture variants with boolean conditions - foo = fixture(ComplexFixture, - variants={'p0': lambda x: x<10, 'p1': lambda x: x=='d'}) - - # Select the fixture variants by index - bar = fixture(ComplexFixture, variants=range(300,310)) - ... - - A parent test may also specify the value of different variables in the fixture class to be set before its instantiation. - Each variable must have been declared in the fixture class with the :func:`~reframe.core.pipeline.RegressionMixin.variable` built-in, otherwise it is silently ignored. - This variable specification is equivalent to deriving a new class from the fixture class, and setting these variable values in the class body of a newly derived class. - Therefore, when fixture declarations use the same fixture class and pass different values to the ``variables`` argument, the fixture class is interpreted as a different class for each of these fixture declarations. - See the example below. - - .. code:: python - - class Fixture(rfm.RegressionTest): - v = variable(int, value=1) - ... - - @rfm.simple_test - class TestF(rfm.RegressionTest): - foo = fixture(Fixture) - bar = fixture(Fixture, variables={'v':5}) - baz = fixture(Fixture, variables={'v':10}) - ... - - @run_after('setup') - def print_fixture_variables(self): - print(self.foo.v) # Prints 1 - print(self.bar.v) # Prints 5 - print(self.baz.v) # Prints 10 - - The test :class:`TestF` declares the fixtures ``foo``, ``bar`` and ``baz`` using the same :class:`Fixture` class. - If no variables were set in ``bar`` and ``baz``, this would result into the same fixture being declared multiple times in the same scope (implicitly set to ``'test'``), which would lead to a single instance of :class:`Fixture` being referred to by ``foo``, ``bar`` and ``baz``. - However, in this case ReFrame identifies that the declared fixtures pass different values to the ``variables`` argument in the fixture declaration, and executes these three fixtures separately. - - .. note:: - Mappings passed to the ``variables`` argument that define the same class variables in different order are interpreted as the same value. - The two fixture declarations below are equivalent, and both ``foo`` and ``bar`` will point to the same instance of the fixture class :class:`MyResource`. - - .. code:: python - - foo = fixture(MyResource, variables={'a':1, 'b':2}) - bar = fixture(MyResource, variables={'b':2, 'a':1}) - - - - :param cls: A class derived from :class:`~reframe.core.pipeline.RegressionTest` that manages a given resource. - The base from this class may be further restricted to other derived classes of :class:`~reframe.core.pipeline.RegressionTest` depending on the ``scope`` parameter. - :param scope: Sets the extent to which other regression tests may share the resources managed by a fixture. - The available scopes are, from more to less restrictive, ``'test'``, ``'environment'``, ``'partition'`` and ``'session'``. - By default a fixture's scope is set to ``'test'``, which makes the resource private to the test that uses the fixture. - This means that when multiple regression tests use the same fixture class with a ``'test'`` scope, the fixture will run once per regression test. - When the scope is set to ``'environment'``, the resources managed by the fixture are shared across all the tests that use the fixture and run on the same system partition and use the same programming environment. - When the scope is set to ``'partition'``, the resources managed by the fixture are shared instead across all the tests that use the fixture and run on the same system partition. - Lastly, when the scope is set to ``'session'``, the resources managed by the fixture are shared across the full ReFrame session. - Fixtures with either ``'partition'`` or ``'session'`` scopes may be shared across different regression tests under different programming environments, and for this reason, when using these two scopes, the fixture class ``cls`` is required to derive from :class:`~reframe.core.pipeline.RunOnlyRegressionTest`. - :param action: Set the behavior of a parameterized fixture to either ``'fork'`` or ``'join'``. - With a ``'fork'`` action, a parameterized fixture effectively parameterizes the regression test. - On the other hand, a ``'join'`` action gathers all the fixture variants into the same instance of the regression test. - By default, the ``action`` parameter is set to ``'fork'``. - :param variants: Filter or sub-select a subset of the variants from a parameterized fixture. - This argument can be either an iterable with the indices from the desired variants, or a mapping containing unary functions that return the value of a boolean expression on the values of a given parameter. - :param variables: Mapping to set the values of fixture's variables. The variables are set after the fixture class has been created (i.e. after the class body has executed) and before the fixture class is instantiated. - - - .. versionadded:: 3.9.0 - - ------------------- -Built-in functions ------------------- - -ReFrame provides the following built-in functions, which are only available in the class body of classes deriving from :class:`~reframe.core.pipeline.RegressionMixin`. - -.. py:decorator:: RegressionMixin.sanity_function(func) - - Decorate a member function as the sanity function of the test. - - This decorator will convert the given function into a :func:`~RegressionMixin.deferrable` and mark it to be executed during the test's sanity stage. - When this decorator is used, manually assigning a value to :attr:`~RegressionTest.sanity_patterns` in the test is not allowed. - - Decorated functions may be overridden by derived classes, and derived classes may also decorate a different method as the test's sanity function. - Decorating multiple member functions in the same class is not allowed. - However, a :class:`RegressionTest` may inherit from multiple :class:`RegressionMixin` classes with their own sanity functions. - In this case, the derived class will follow Python's `MRO `_ to find a suitable sanity function. - - .. versionadded:: 3.7.0 - -.. py:decorator:: RegressionMixin.performance_function(unit, *, perf_key=None) - - Decorate a member function as a performance function of the test. - - This decorator converts the decorated method into a performance deferrable function (see ":ref:`deferrable-performance-functions`" for more details) whose evaluation is deferred to the performance stage of the regression test. - The decorated function must take a single argument without a default value (i.e. ``self``) and any number of arguments with default values. - A test may decorate multiple member functions as performance functions, where each of the decorated functions must be provided with the units of the performance quantities to be extracted from the test. - These performance units must be of type :class:`str`. - Any performance function may be overridden in a derived class and multiple bases may define their own performance functions. - In the event of a name conflict, the derived class will follow Python's `MRO `_ to choose the appropriate performance function. - However, defining more than one performance function with the same name in the same class is disallowed. - - The full set of performance functions of a regression test is stored under :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` as key-value pairs, where, by default, the key is the name of the decorated member function, and the value is the deferred performance function itself. - Optionally, the key under which a performance function is stored in :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` can be customised by passing the desired key as the ``perf_key`` argument to this decorator. - - .. versionadded:: 3.8.0 +.. versionadded:: 3.4.2 -.. py:decorator:: RegressionMixin.deferrable(func) +ReFrame test base classes and, in particular, the :class:`reframe.core.pipeline.RegressionMixin` class, define a set of functions and decorators that can be used to define essential test elements, such as variables, parameters, fixtures, pipeline hooks etc. +These are called *builtins* because they are directly available for use inside the test class body that is being defined without the need to import any module. +However, almost all of these builtins are also available from the :obj:`reframe.core.builtins` module. +The use of this module is required only when creating new tests programmatically using the :func:`~reframe.core.meta.make_test` function. - Converts the decorated method into a deferrable function. +.. py:method:: reframe.core.pipeline.RegressionMixin.bind(func, name=None) - See :ref:`deferrable-functions` for further information on deferrable functions. + Bind a free function to a regression test. - .. versionadded:: 3.7.0 + By default, the function is bound with the same name as the free function. + However, the function can be bound using a different name with the ``name`` argument. -.. autodecorator:: reframe.core.pipeline.RegressionMixin.loggable_as(name) + :param func: external function to be bound to a class. + :param name: bind the function under a different name. -.. py:decorator:: reframe.core.pipeline.RegressionMixin.loggable + .. note:: + This is the only builtin that is not available through the :obj:`reframe.core.builtins` module. + The reason is that the :func:`bind` method needs to access the class namespace directly in order to bind the free function to the class. - Equivalent to :func:`@loggable_as(None) `. + .. versionadded:: 3.6.2 - .. versionadded:: 3.11.0 +.. autodecorator:: reframe.core.builtins.deferrable +.. autofunction:: reframe.core.builtins.fixture -.. py:decorator:: RegressionMixin.require_deps(func) +.. autodecorator:: reframe.core.builtins.loggable_as(name) - Decorator to denote that a function will use the test dependencies. +.. autodecorator:: reframe.core.builtins.loggable - The arguments of the decorated function must be named after the dependencies that the function intends to use. - The decorator will bind the arguments to a partial realization of the :func:`~reframe.core.pipeline.RegressionTest.getdep` function, such that conceptually the new function arguments will be the following: +.. autofunction:: reframe.core.builtins.parameter - .. code-block:: python +.. autodecorator:: reframe.core.builtins.performance_function - new_arg = functools.partial(getdep, orig_arg_name) +.. autodecorator:: reframe.core.builtins.require_deps - The converted arguments are essentially functions accepting a single argument, which is the target test's programming environment. - Additionally, this decorator will attach the function to run *after* the test's setup phase, but *before* any other "post-setup" pipeline hook. +.. autodecorator:: reframe.core.builtins.run_after(stage) - .. warning:: - .. versionchanged:: 3.7.0 - Using this function from the :py:mod:`reframe` or :py:mod:`reframe.core.decorators` modules is now deprecated. - You should use the built-in function described here. +.. autodecorator:: reframe.core.builtins.run_before(stage) -.. py:function:: RegressionMixin.bind(func, name=None) +.. autodecorator:: reframe.core.builtins.sanity_function - Bind a free function to a regression test. +.. autofunction:: reframe.core.builtins.variable - By default, the function is bound with the same name as the free function. - However, the function can be bound using a different name with the ``name`` argument. - :param func: external function to be bound to a class. - :param name: bind the function under a different name. +.. versionchanged:: 3.7.0 + Expose :func:`@deferrable ` as a builtin. - .. versionadded:: 3.6.2 +.. versionchanged:: 3.11.0 + Builtins are now available also through the :obj:`reframe.core.builtins` module. +.. _pipeline-hooks: -------------- Pipeline Hooks -------------- -ReFrame provides built-in functions that allow attaching arbitrary functions to run before and/or after a given stage of the execution pipeline. +ReFrame provides a mechanism to allow attaching arbitrary functions to run before or after a given stage of the execution pipeline. +This is achieved through the :func:`@run_before ` and :func:`@run_after ` test builtins. Once attached to a given stage, these functions are referred to as *pipeline hooks*. A hook may be attached to multiple pipeline stages and multiple hooks may also be attached to the same pipeline stage. Pipeline hooks attached to multiple stages will be executed on each pipeline stage the hook was attached to. @@ -599,8 +127,11 @@ In the following example, :func:`BaseTest.x` will execute before :func:`DerivedT '''Hook y''' +.. seealso:: + - :func:`@run_before `, :func:`@run_after ` decorators + .. note:: - Pipeline hooks do not execute in the test's stage directory. + Pipeline hooks do not execute in the test's stage directory, but in the directory that ReFrame executes in. However, the test's :attr:`~reframe.core.pipeline.RegressionTest.stagedir` can be accessed by explicitly changing the working directory from within the hook function itself (see the :class:`~reframe.utility.osext.change_dir` utility for further details): .. code:: python @@ -618,7 +149,7 @@ In the following example, :func:`BaseTest.x` will execute before :func:`DerivedT .. warning:: .. versionchanged:: 3.7.0 Declaring pipeline hooks using the same name functions from the :py:mod:`reframe` or :py:mod:`reframe.core.decorators` modules is now deprecated. - You should use the built-in functions described in this section instead. + You should use the builtin functions described in the :ref:`builtins` section.. .. warning:: .. versionchanged:: 3.9.2 @@ -630,42 +161,6 @@ In the following example, :func:`BaseTest.x` will execute before :func:`DerivedT Tests that relied on the execution order of hooks might break with this change. -.. py:decorator:: RegressionMixin.run_before(stage) - - Decorator for attaching a function to a given pipeline stage. - - The function will run just before the specified pipeline stage and it cannot accept any arguments except ``self``. - This decorator can be stacked, in which case the function will be attached to multiple pipeline stages. - See above for the valid ``stage`` argument values. - - -.. py:decorator:: RegressionMixin.run_after(stage) - - Decorator for attaching a function to a given pipeline stage. - - This is analogous to :func:`~RegressionMixin.run_before`, except that the hook will execute right after the stage it was attached to. - This decorator also supports ``'init'`` as a valid ``stage`` argument, where in this case, the hook will execute right after the test is initialized (i.e. after the :func:`__init__` method is called) and before entering the test's pipeline. - In essence, a post-init hook is equivalent to defining additional :func:`__init__` functions in the test. - The following code - - .. code-block:: python - - class MyTest(rfm.RegressionTest): - @run_after('init') - def foo(self): - self.x = 1 - - is equivalent to - - .. code-block:: python - - class MyTest(rfm.RegressionTest): - def __init__(self): - self.x = 1 - - .. versionchanged:: 3.5.2 - Add support for post-init hooks. - .. _test-variants: @@ -673,7 +168,7 @@ In the following example, :func:`BaseTest.x` will execute before :func:`DerivedT Test variants ------------- -Through the :func:`~reframe.core.pipeline.RegressionMixin.parameter` and :func:`~reframe.core.pipeline.RegressionMixin.fixture` builtins, a regression test may store multiple versions or `variants` of a regression test at the class level. +Through the :func:`~reframe.core.builtins.parameter` and :func:`~reframe.core.builtins.fixture` builtins, a regression test may store multiple versions or `variants` of a regression test at the class level. During class creation, the test's parameter and fixture spaces are constructed and combined, assigning a unique index to each of the available test variants. In most cases, the user does not need to be aware of all the internals related to this variant indexing, since ReFrame will run by default all the available variants for each of the registered tests. On the other hand, in more complex use cases such as setting dependencies across different test variants, or when performing some complex variant sub-selection on a fixture declaration, the user may need to access some of this low-level information related to the variant indexing. @@ -836,7 +331,7 @@ The :py:mod:`reframe` module offers direct access to the basic test classes, con .. py:decorator:: reframe.require_deps .. deprecated:: 3.7.0 - Please use the :func:`~reframe.core.pipeline.RegressionMixin.require_deps` built-in function + Please use the :func:`@require_deps ` builtin decorator. .. py:decorator:: reframe.required_version @@ -847,13 +342,13 @@ The :py:mod:`reframe` module offers direct access to the basic test classes, con .. py:decorator:: reframe.run_after .. deprecated:: 3.7.0 - Please use the :func:`~reframe.core.pipeline.RegressionMixin.run_after` built-in function + Please use the :func:`~reframe.core.builtins.run_after` built-in function .. py:decorator:: reframe.run_before .. deprecated:: 3.7.0 - Please use the :func:`~reframe.core.pipeline.RegressionMixin.run_before` built-in function + Please use the :func:`~reframe.core.builtins.run_before` built-in function .. py:decorator:: reframe.simple_test diff --git a/reframe/core/builtins.py b/reframe/core/builtins.py index b0e3809054..9a270386ad 100644 --- a/reframe/core/builtins.py +++ b/reframe/core/builtins.py @@ -38,17 +38,48 @@ def final(fn): # Hook-related builtins def run_before(stage): - '''Decorator for attaching a test method to a given stage. + '''Attach the decorated function before a certain pipeline stage. - See online docs for more information. + The function will run just before the specified pipeline stage and it + cannot accept any arguments except ``self``. This decorator can be + stacked, in which case the function will be attached to multiple pipeline + stages. See above for the valid ``stage`` argument values. + + :param stage: The pipeline stage where this function will be attached to. + See :ref:`pipeline-hooks` for the list of valid stage values. ''' return hooks.attach_to('pre_' + stage) def run_after(stage): - '''Decorator for attaching a test method to a given stage. + '''Attach the decorated function after a certain pipeline stage. + + This is analogous to :func:`~RegressionMixin.run_before`, except that the + hook will execute right after the stage it was attached to. This decorator + also supports ``'init'`` as a valid ``stage`` argument, where in this + case, the hook will execute right after the test is initialized (i.e. + after the :func:`__init__` method is called) and before entering the + test's pipeline. In essence, a post-init hook is equivalent to defining + additional :func:`__init__` functions in the test. The following code + + .. code-block:: python + + class MyTest(rfm.RegressionTest): + @run_after('init') + def foo(self): + self.x = 1 + + is equivalent to + + .. code-block:: python + + class MyTest(rfm.RegressionTest): + def __init__(self): + self.x = 1 + + .. versionchanged:: 3.5.2 + Add support for post-init hooks. - See online docs for more information. ''' return hooks.attach_to('post_' + stage) @@ -59,10 +90,24 @@ def run_after(stage): # Sanity and performance function builtins def sanity_function(fn): - '''Mark a function as the test's sanity function. - - Decorated functions must be unary and they will be converted into - deferred expressions. + '''Decorate a test member function to mark it as a sanity check. + + This decorator will convert the given function into a + :func:`~RegressionMixin.deferrable` and mark it to be executed during the + test's sanity stage. When this decorator is used, manually assigning a + value to :attr:`~RegressionTest.sanity_patterns` in the test is not + allowed. + + Decorated functions may be overridden by derived classes, and derived + classes may also decorate a different method as the test's sanity + function. Decorating multiple member functions in the same class is not + allowed. However, a :class:`RegressionTest` may inherit from multiple + :class:`RegressionMixin` classes with their own sanity functions. In this + case, the derived class will follow Python's `MRO + `_ to find + a suitable sanity function. + + .. versionadded:: 3.7.0 ''' _def_fn = deferrable(fn) @@ -70,17 +115,43 @@ def sanity_function(fn): return _def_fn -def performance_function(units, *, perf_key=None): - '''Decorate a function to extract a performance variable. +def performance_function(unit, *, perf_key=None): + '''Decorate a test member function to mark it as a performance metric + function. + + This decorator converts the decorated method into a performance deferrable + function (see ":ref:`deferrable-performance-functions`" for more details) + whose evaluation is deferred to the performance stage of the regression + test. The decorated function must take a single argument without a default + value (i.e. ``self``) and any number of arguments with default values. A + test may decorate multiple member functions as performance functions, + where each of the decorated functions must be provided with the unit of + the performance quantity to be extracted from the test. Any performance + function may be overridden in a derived class and multiple bases may + define their own performance functions. In the event of a name conflict, + the derived class will follow Python's `MRO + `_ to + choose the appropriate performance function. However, defining more than + one performance function with the same name in the same class is + disallowed. + + The full set of performance functions of a regression test is stored under + :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` as key-value + pairs, where, by default, the key is the name of the decorated member + function, and the value is the deferred performance function itself. + Optionally, the key under which a performance function is stored in + :attr:`~reframe.core.pipeline.RegressionTest.perf_variables` can be + customised by passing the desired key as the ``perf_key`` argument to this + decorator. + + :param unit: A string representing the measurement unit of this metric. + + .. versionadded:: 3.8.0 - The ``units`` argument indicates the units of the performance - variable to be extracted. - The ``perf_key`` optional arg will be used as the name of the - performance variable. If not provided, the function name will - be used as the performance variable name. ''' - if not isinstance(units, str): - raise TypeError('performance units must be a string') + + if not isinstance(unit, str): + raise TypeError('performance unit must be a string') if perf_key and not isinstance(perf_key, str): raise TypeError("'perf_key' must be a string") @@ -95,7 +166,7 @@ def _deco_wrapper(func): @functools.wraps(func) def _perf_fn(*args, **kwargs): return _DeferredPerformanceExpression( - func, units, *args, **kwargs + func, unit, *args, **kwargs ) _perf_key = perf_key if perf_key else func.__name__ @@ -135,3 +206,4 @@ def _loggable(fn): loggable = loggable_as(None) +loggable.__doc__ = '''Equivalent to :func:`loggable_as(None) `.''' diff --git a/reframe/core/deferrable.py b/reframe/core/deferrable.py index 0ecc789607..71c52a069b 100644 --- a/reframe/core/deferrable.py +++ b/reframe/core/deferrable.py @@ -8,8 +8,12 @@ def deferrable(func): - '''Function decorator for converting a function to a deferred - expression.''' + '''Convert the decorated function to a deferred expression. + + See :ref:`deferrable-functions` for further information on deferrable + functions. + ''' + @functools.wraps(func) def _deferred(*args, **kwargs): return _DeferredExpression(func, *args, **kwargs) diff --git a/reframe/core/fixtures.py b/reframe/core/fixtures.py index 4db9e05027..6e63639329 100644 --- a/reframe/core/fixtures.py +++ b/reframe/core/fixtures.py @@ -348,54 +348,338 @@ def __contains__(self, cls): class TestFixture: - '''Regression test fixture class. - - A fixture is a regression test that generates a resource that must exist - before the parent test is executed. A fixture is a class that derives from - the :class:`reframe.core.pipeline.RegressionTest` class and serves as a - building block to compose a more complex test structure. Since fixtures are - full ReFrame tests on their own, a fixture can have multiple fixtures, and - so on; building a directed acyclic graph. - - However, a given fixture may be shared by multiple regression tests that - need the same resource. This can be achieved by setting the appropriate - scope level on which the fixture should be shared. By default, fixtures - are registered with the ``'test'`` scope, which makes each fixture - `private` to each of the parent tests. Hence, if all fixtures use this - scope, the resulting fixture hierarchy can be thought of multiple - independent branches that emanate from each root regression test. On the - other hand, setting a more relaxed scope that allows resource sharing - across different regression tests will effectively interconnect the - fixture branches that share a resource. - - From a more to less restrictive scope, the valid scopes are ``'test'``, - ``'environment'``, ``'partition'`` and ``'session'``. Fixtures with - a scope set to either ``'partition'`` or ``'session'`` must derive from - the :class:`reframe.core.pipeline.RunOnlyRegressionTest` class, since the - generated resource must not depend on the programming environment. Fixtures - with scopes set to either ``'environment'`` or ``'test'`` can derive from - any derived class from :class:`reframe.core.pipeline.RegressionTest`. - - Fixtures may be parameterized, where a regression test that uses a - parameterized fixture is by extension a parameterized test. Hence, the - number of test variants of a test will depend on the test parameters and - the parameters of each of the fixtures that compose the parent test. Each - possible parameter-fixture combination has a unique `variant number`, which - is an index in the range from ``[0, N)``, where `N` is the total number of - test variants. This is the default behaviour and it is achieved when the - action argument is set to ``'fork'``. On the other hand, if this argument - is set to a ``'join'`` action, the parent test will reduce all the fixture + '''Insert a new fixture in the current test. + + A fixture is a regression test that creates, prepares and/or manages a + resource for another regression test. Fixtures may contain other fixtures + and so on, forming a directed acyclic graph. A parent fixture (or a + regular regression test) requires the resources managed by its child + fixtures in order to run, and it may only access these fixture resources + after its ``setup`` pipeline stage. The execution of parent fixtures is + postponed until all their respective children have completed execution. + However, the destruction of the resources managed by a fixture occurs in + reverse order, only after all the parent fixtures have been destroyed. + This destruction of resources takes place during the ``cleanup`` pipeline + stage of the regression test. Fixtures must not define the members + :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and + :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs`. These + variables are defined based on the values specified in the parent test, + ensuring that the fixture runs with a suitable system partition and + programming environment combination. A fixture's + :attr:`~reframe.core.pipeline.RegressionTest.name` attribute may be + internally mangled depending on the arguments passed during the fixture + declaration. Hence, manually setting or modifying the + :attr:`~reframe.core.pipeline.RegressionTest.name` attribute in the + fixture class is disallowed, and breaking this restriction will result in + undefined behavior. + + .. warning:: + The fixture name mangling is considered an internal framework mechanism + and it may change in future versions without any notice. Users must not + express any logic in their tests that relies on a given fixture name + mangling scheme. + + + By default, the resources managed by a fixture are private to the parent + test. However, it is possible to share these resources across different + tests by passing the appropriate fixture ``scope`` argument. The different + scope levels are independent from each other and a fixture only executes + once per scope, where all the tests that belong to that same scope may use + the same resources managed by a given fixture instance. The available + scopes are: + + - **session**: This scope encloses all the tests and fixtures that run + in the full ReFrame session. This may include tests that use different + system partition and programming environment combinations. The fixture + class must derive from + :class:`~reframe.core.pipeline.RunOnlyRegressionTest` to avoid any + implicit dependencies on the partition or the programming environment + used. + + - **partition**: This scope spans across a single system partition. This + may include different tests that run on the same partition but use + different programming environments. Fixtures with this scope must be + independent of the programming environment, which restricts the + fixture class to derive from + :class:`~reframe.core.pipeline.RunOnlyRegressionTest`. + + - **environment**: The extent of this scope covers a single combination + of system partition and programming environment. Since the fixture is + guaranteed to have the same partition and programming environment as + the parent test, the fixture class can be any derived class from + :class:`~reframe.core.pipeline.RegressionTest`. * **test**: This scope + covers a single instance of the parent test, where the resources + provided by the fixture are exclusive to each parent test instance. + The fixture class can be any derived class from + :class:`~reframe.core.pipeline.RegressionTest`. + + Rather than specifying the scope at the fixture class definition, ReFrame + fixtures set the scope level from the consumer side (i.e. when used by + another test or fixture). A test may declare multiple fixtures using the + same class, where fixtures with different scopes are guaranteed to point + to different instances of the fixture class. On the other hand, when two + or more fixtures use the same fixture class and have the same scope, these + different fixtures will point to the same underlying resource if the + fixtures refer to the same :ref:`variant` of the fixture + class. The example below illustrates the different fixture scope usages: + + .. code:: python + + class MyFixture(rfm.RunOnlyRegressionTest): + my_var = variable(int, value=1) + ... + + + @rfm.simple_test + class TestA(rfm.RegressionTest): + valid_systems = ['p1', 'p2'] + valid_prog_environs = ['e1', 'e2'] + f1 = fixture(MyFixture, scope='session') # Shared throughout the full session + f2 = fixture(MyFixture, scope='partition') # Shared for each supported partition + f3 = fixture(MyFixture, scope='environment') # Shared for each supported part+environ + f4 = fixture(MyFixture, scope='test') # Private evaluation of MyFixture + ... + + + @rfm.simple_test + class TestB(rfm.RegressionTest): + valid_systems = ['p1'] + valid_prog_environs = ['e1'] + f1 = fixture(MyFixture, scope='test') # Another private instance of MyFixture + f2 = fixture(MyFixture, scope='environment') # Same as f3 in TestA for p1 + e1 + f3 = fixture(MyFixture, scope='session') # Same as f1 in TestA + ... + + @run_after('setup') + def access_fixture_resources(self): + # Dummy pipeline hook to illustrate fixture resource access + assert self.f1.my_var is not self.f2.my_var + assert self.f1.my_var is not self.f3.my_var + + + :class:`TestA` supports two different valid systems and another two valid + programming environments. Assuming that both environments are supported by + each of the system partitions ``'p1'`` and ``'p2'``, this test will + execute a total of four times. This test uses the very simple + :class:`MyFixture` fixture multiple times using different scopes, where + fixture ``f1`` (session scope) will be shared across the four test + instances, and fixture ``f4`` (test scope) will be executed once per test + instance. On the other hand, ``f2`` (partition scope) will run once per + partition supported by test :class:`TestA`, and the multiple per-partition + executions (i.e. for each programming environment) will share the same + underlying resource for ``f2``. Lastly, ``f3`` will run a total of four + times, which is once per partition and environment combination. This + simple :class:`TestA` shows how multiple instances from the same test can + share resources, but the real power behind fixtures is illustrated with + :class:`TestB`, where this resource sharing is extended across different + tests. For simplicity, :class:`TestB` only supports a single partition + ``'p1'`` and programming environment ``'e1'``, and similarly to + :class:`TestA`, ``f1`` (test scope) causes a private evaluation of the + fixture :class:`MyFixture`. However, the resources managed by fixtures + ``f2`` (environment scope) and ``f3`` (session scope) are shared with + :class:`Test1`. + + Fixtures are treated by ReFrame as first-class ReFrame tests, which means + that these classes can use the same built-in functionalities as in regular + tests decorated with + :func:`@rfm.simple_test`. This + includes the :func:`~reframe.core.pipeline.RegressionMixin.parameter` + built-in, where fixtures may have more than one + :ref:`variant`. When this occurs, a parent test may select + to either treat a parameterized fixture as a test parameter, or instead, + to gather all the fixture variants from a single instance of the parent + test. In essence, fixtures implement `fork-join` model whose behavior may + be controlled through the ``action`` argument. This argument may be set to + one of the following options: + + - **fork**: This option parameterizes the parent test as a function of + the fixture variants. The fixture handle will resolve to a single + instance of the fixture. + + - **join**: This option gathers all the variants from a fixture into a + single instance of the parent test. The fixture handle will point to a + list containing all the fixture variants. + + A test may declare multiple fixtures with different ``action`` options, + where the default ``action`` option is ``'fork'``. The example below + illustrates the behavior of these two different options. + + .. code:: python + + class ParamFix(rfm.RegressionTest): + p = parameter(range(5)) # A simple test parameter + ... + + + @rfm.simple_test + class TestC(rfm.RegressionTest): + # Parameterize TestC for each ParamFix variant + f = fixture(ParamFix, action='fork') + ... + + @run_after('setup') + def access_fixture_resources(self): + print(self.f.p) # Prints the fixture's variant parameter value + + + @rfm.simple_test + class TestD(rfm.RegressionTest): + # Gather all fixture variants into a single test + f = fixture(ParamFix, action='join') + ... + + @run_after('setup') + def reduce_range(self): + # Sum all the values of p for each fixture variant + res = functools.reduce(lambda x, y: x+y, (fix.p for fix in self.f)) + n = len(self.f)-1 + assert res == (n*n + n)/2 + + Here :class:`ParamFix` is a simple fixture class with a single parameter. + When the test :class:`TestC` uses this fixture with a ``'fork'`` action, + the test is implicitly parameterized over each variant of + :class:`ParamFix`. Hence, when the :func:`access_fixture_resources` + post-setup hook accesses the fixture ``f``, it only access a single + instance of the :class:`ParamFix` fixture. On the other hand, when this + same fixture is used with a ``'join'`` action by :class:`TestD`, the test + is not parameterized and all the :class:`ParamFix` instances are gathered + into ``f`` as a list. Thus, the post-setup pipeline hook + :func:`reduce_range` can access all the fixture variants and compute a + reduction of the different ``p`` values. + + When declaring a fixture, a parent test may select a subset of the fixture + variants through the ``variants`` argument. This variant selection can be + done by either passing an iterable containing valid variant indices (see + :ref:`test-variants` for further information on how the test variants are + indexed), or instead, passing a mapping with the parameter name (of the + fixture class) as keys and filtering functions as values. These filtering + functions are unary functions that return the value of a boolean + expression on the values of the specified parameter, and they all must + evaluate to :class:`True` for at least one of the fixture class variants. + See the example below for an illustration on how to filter-out fixture variants. - The variants from a given fixture to be used by the parent test can be - filtered out through the ``variants`` optional argument. This can either be - a list of the variant numbers to be used, or it can be a dictionary with - conditions on the parameter space of the fixture. + .. code:: python + + class ComplexFixture(rfm.RegressionTest): + # A fixture with 400 different variants. + p0 = parameter(range(100)) + p1 = parameter(['a', 'b', 'c', 'd']) + ... + + @rfm.simple_test + class TestE(rfm.RegressionTest): + # Select the fixture variants with boolean conditions + foo = fixture(ComplexFixture, + variants={'p0': lambda x: x<10, 'p1': lambda x: x=='d'}) + + # Select the fixture variants by index + bar = fixture(ComplexFixture, variants=range(300,310)) + ... + + A parent test may also specify the value of different variables in the + fixture class to be set before its instantiation. Each variable must have + been declared in the fixture class with the + :func:`~reframe.core.pipeline.RegressionMixin.variable` built-in, + otherwise it is silently ignored. This variable specification is + equivalent to deriving a new class from the fixture class, and setting + these variable values in the class body of a newly derived class. + Therefore, when fixture declarations use the same fixture class and pass + different values to the ``variables`` argument, the fixture class is + interpreted as a different class for each of these fixture declarations. + See the example below. + + .. code:: python + + class Fixture(rfm.RegressionTest): + v = variable(int, value=1) + ... + + @rfm.simple_test + class TestF(rfm.RegressionTest): + foo = fixture(Fixture) + bar = fixture(Fixture, variables={'v':5}) + baz = fixture(Fixture, variables={'v':10}) + ... + + @run_after('setup') + def print_fixture_variables(self): + print(self.foo.v) # Prints 1 + print(self.bar.v) # Prints 5 + print(self.baz.v) # Prints 10 + + The test :class:`TestF` declares the fixtures ``foo``, ``bar`` and ``baz`` + using the same :class:`Fixture` class. If no variables were set in ``bar`` + and ``baz``, this would result into the same fixture being declared + multiple times in the same scope (implicitly set to ``'test'``), which + would lead to a single instance of :class:`Fixture` being referred to by + ``foo``, ``bar`` and ``baz``. However, in this case ReFrame identifies + that the declared fixtures pass different values to the ``variables`` + argument in the fixture declaration, and executes these three fixtures + separately. + + .. note:: + Mappings passed to the ``variables`` argument that define the same + class variables in different order are interpreted as the same value. + The two fixture declarations below are equivalent, and both ``foo`` and + ``bar`` will point to the same instance of the fixture class + :class:`MyResource`. + + .. code:: python + + foo = fixture(MyResource, variables={'a':1, 'b':2}) + bar = fixture(MyResource, variables={'b':2, 'a':1}) + + + + :param cls: A class derived from + :class:`~reframe.core.pipeline.RegressionTest` that manages a given + resource. The base from this class may be further restricted to other + derived classes of :class:`~reframe.core.pipeline.RegressionTest` + depending on the ``scope`` parameter. + + :param scope: Sets the extent to which other regression tests may share + the resources managed by a fixture. The available scopes are, from + more to less restrictive, ``'test'``, ``'environment'``, + ``'partition'`` and ``'session'``. By default a fixture's scope is set + to ``'test'``, which makes the resource private to the test that uses + the fixture. This means that when multiple regression tests use the + same fixture class with a ``'test'`` scope, the fixture will run once + per regression test. When the scope is set to ``'environment'``, the + resources managed by the fixture are shared across all the tests that + use the fixture and run on the same system partition and use the same + programming environment. When the scope is set to ``'partition'``, the + resources managed by the fixture are shared instead across all the + tests that use the fixture and run on the same system partition. + Lastly, when the scope is set to ``'session'``, the resources managed + by the fixture are shared across the full ReFrame session. Fixtures + with either ``'partition'`` or ``'session'`` scopes may be shared + across different regression tests under different programming + environments, and for this reason, when using these two scopes, the + fixture class ``cls`` is required to derive from + :class:`~reframe.core.pipeline.RunOnlyRegressionTest`. + + :param action: Set the behavior of a parameterized fixture to either + ``'fork'`` or ``'join'``. With a ``'fork'`` action, a parameterized + fixture effectively parameterizes the regression test. On the other + hand, a ``'join'`` action gathers all the fixture variants into the + same instance of the regression test. By default, the ``action`` + parameter is set to ``'fork'``. + + :param variants: Filter or sub-select a subset of the variants from a + parameterized fixture. This argument can be either an iterable with + the indices from the desired variants, or a mapping containing unary + functions that return the value of a boolean expression on the values + of a given parameter. + + :param variables: Mapping to set the values of fixture's variables. The + variables are set after the fixture class has been created (i.e. after + the class body has executed) and before the fixture class is + instantiated. + + + .. versionadded:: 3.9.0 - Also, a fixture may set or update the default value of a test variable - by passing the appropriate key-value mapping as the ``variables`` argument. - - :meta private: ''' def __init__(self, cls, *, scope='test', action='fork', variants='all', diff --git a/reframe/core/hooks.py b/reframe/core/hooks.py index 2ad0b99e21..cb00007bb4 100644 --- a/reframe/core/hooks.py +++ b/reframe/core/hooks.py @@ -37,9 +37,28 @@ def _fn(*args, **kwargs): def require_deps(func): - '''Denote that the decorated test method will use the test dependencies. + '''Decorator to denote that a function will use the test dependencies. - See online docs for more information. + The arguments of the decorated function must be named after the + dependencies that the function intends to use. The decorator will bind the + arguments to a partial realization of the + :func:`~reframe.core.pipeline.RegressionTest.getdep` function, such that + conceptually the new function arguments will be the following: + + .. code-block:: python + + new_arg = functools.partial(getdep, orig_arg_name) + + The converted arguments are essentially functions accepting a single + argument, which is the target test's programming environment. + Additionally, this decorator will attach the function to run *after* the + test's setup phase, but *before* any other "post-setup" pipeline hook. + + .. warning:: + .. versionchanged:: 3.7.0 + Using this functionality from the :py:mod:`reframe` or + :py:mod:`reframe.core.decorators` modules is now deprecated. You + should use the built-in function described here. ''' tests = inspect.getfullargspec(func).args[1:] diff --git a/reframe/core/parameters.py b/reframe/core/parameters.py index 5dce03bb6d..9c57255539 100644 --- a/reframe/core/parameters.py +++ b/reframe/core/parameters.py @@ -16,15 +16,116 @@ class TestParam: - '''Regression test paramter class. - - Stores the attributes of a regression test parameter as defined directly - in the test definition. These attributes are the parameter's name, - values, and inheritance behaviour. This class should be thought of as a - temporary storage for these parameter attributes, before the full final - parameter space is built. - - :meta private: + '''Inserts a new test parameter. + + At the class level, these parameters are stored in a separate namespace + referred to as the *parameter space*. If a parameter with a matching name + is already present in the parameter space of a parent class, the existing + parameter values will be combined with those provided by this method + following the inheritance behavior set by the arguments ``inherit_params`` + and ``filter_params``. Instead, if no parameter with a matching name + exists in any of the parent parameter spaces, a new regression test + parameter is created. A regression test can be parameterized as follows: + + .. code:: python + + class Foo(rfm.RegressionTest): + variant = parameter(['A', 'B']) + # print(variant) # Error: a parameter may only be accessed from the class instance. + + @run_after('init') + def do_something(self): + if self.variant == 'A': + do_this() + else: + do_other() + + One of the most powerful features of these built-in functions is that they + store their input information at the class level. However, a parameter may + only be accessed from the class instance and accessing it directly from + the class body is disallowed. With this approach, extending or + specializing an existing parameterized regression test becomes + straightforward, since the test attribute additions and modifications made + through built-in functions in the parent class are automatically inherited + by the child test. For instance, continuing with the example above, one + could override the :func:`do_something` hook in the :class:`Foo` + regression test as follows: + + .. code:: python + + class Bar(Foo): + @run_after('init') + def do_something(self): + if self.variant == 'A': + override_this() + else: + override_other() + + Moreover, a derived class may extend, partially extend and/or modify the + parameter values provided in the base class as shown below. + + .. code:: python + + class ExtendVariant(Bar): + # Extend the full set of inherited variant parameter values to ['A', 'B', 'C'] + variant = parameter(['C'], inherit_params=True) + + class PartiallyExtendVariant(Bar): + # Extend a subset of the inherited variant parameter values to ['A', 'D'] + variant = parameter(['D'], inherit_params=True, + filter_params=lambda x: x[:1]) + + class ModifyVariant(Bar): + # Modify the variant parameter values to ['AA', 'BA'] + variant = parameter(inherit_params=True, + filter_params=lambda x: map(lambda y: y+'A', x)) + + A parameter with no values is referred to as an *abstract parameter* (i.e. + a parameter that is declared but not defined). Therefore, classes with at + least one abstract parameter are considered abstract classes. + + .. code:: python + + class AbstractA(Bar): + variant = parameter() + + class AbstractB(Bar): + variant = parameter(inherit_params=True, filter_params=lambda x: []) + + :param values: An iterable containing the parameter values. + + :param inherit_params: If :obj:`True`, the parameter values defined in any + base class will be inherited. In this case, the parameter values + provided in the current class will extend the set of inherited + parameter values. If the parameter does not exist in any of the parent + parameter spaces, this option has no effect. + + :param filter_params: Function to filter/modify the inherited parameter + values that may have been provided in any of the parent parameter + spaces. This function must accept a single iterable argument and + return an iterable. It will be called with the inherited parameter + values and it must return the filtered set of parameter values. This + function will only have an effect if used with + ``inherit_params=True``. + + :param fmt: A formatting function that will be used to format the values + of this parameter in the test's + :attr:`~reframe.core.pipeline.RegressionTest.display_name`. This + function should take as argument the parameter value and return a + string representation of the value. If the returned value is not a + string, it will be converted using the :py:func:`str` function. + + :param loggable: Mark this parameter as loggable. If :obj:`True`, this + parameter will become a log record attribute under the name + ``check_NAME``, where ``NAME`` is the name of the parameter. + + :returns: A new test parameter. + + .. versionadded:: 3.10.0 + The ``fmt`` argument is added. + + .. versionadded:: 3.11.0 + The ``loggable`` argument is added. ''' def __init__(self, values=None, inherit_params=False, @@ -75,6 +176,8 @@ def update(self, other): The values from the other parameter will be filtered according to the filter function of this one and prepended to this parameter's values. + + :meta private: ''' try: diff --git a/reframe/core/variables.py b/reframe/core/variables.py index cad750e863..e6013205f8 100644 --- a/reframe/core/variables.py +++ b/reframe/core/variables.py @@ -33,16 +33,147 @@ def __deepcopy__(self, memo): class TestVar: - '''Regression test variable class. + '''Insert a new test variable. - Stores the attributes of a variable when defined directly in the class - body. Instances of this class are injected into the regression test - during class instantiation. + Declaring a test variable through the :func:`variable` built-in allows for + a more robust test implementation than if the variables were just defined + as regular test attributes (e.g. ``self.a = 10``). Using variables + declared through the :func:`variable` built-in guarantees that these + regression test variables will not be redeclared by any child class, while + also ensuring that any values that may be assigned to such variables + comply with its original declaration. In essence, declaring test variables + with the :func:`variable` built-in removes any potential test errors that + might be caused by accidentally overriding a class attribute. See the + example below. + + .. code:: python - To support injecting attributes into the variable, this class implements a - separate dict `__attrs__` where these will be stored. + class Foo(rfm.RegressionTest): + my_var = variable(int, value=8) + not_a_var = my_var - 4 + + @run_after('init') + def access_vars(self): + print(self.my_var) # prints 8. + # self.my_var = 'override' # Error: my_var must be an int! + self.not_a_var = 'override' # However, this would work. Dangerous! + self.my_var = 10 # tests may also assign values the standard way + + Here, the argument ``value`` in the :func:`variable` built-in sets the + default value for the variable. This value may be accessed directly from + the class body, as long as it was assigned before either in the same class + body or in the class body of a parent class. This behavior extends the + standard Python data model, where a regular class attribute from a parent + class is never available in the class body of a child class. Hence, using + the :func:`variable` built-in enables us to directly use or modify any + variables that may have been declared upstream the class inheritance + chain, without altering their original value at the parent class level. + + .. code:: python + + class Bar(Foo): + print(my_var) # prints 8 + # print(not_a_var) # This is standard Python and raises a NameError + + # Since my_var is available, we can also update its value: + my_var = 4 + + # Bar inherits the full declaration of my_var with the original type-checking. + # my_var = 'override' # Wrong type error again! + + @run_after('init') + def access_vars(self): + print(self.my_var) # prints 4 + print(self.not_a_var) # prints 4 + + + print(Foo.my_var) # prints 8 + print(Bar.my_var) # prints 4 + + + Here, :class:`Bar` inherits the variables from :class:`Foo` and can see + that ``my_var`` has already been declared in the parent class. Therefore, + the value of ``my_var`` is updated ensuring that the new value complies to + the original variable declaration. However, the value of ``my_var`` at + :class:`Foo` remains unchanged. + + These examples above assumed that a default value can be provided to the + variables in the bases tests, but that might not always be the case. For + example, when writing a test library, one might want to leave some + variables undefined and force the user to set these when using the test. + As shown in the example below, imposing such requirement is as simple as + not passing any ``value`` to the :func:`variable` built-in, which marks + the given variable as *required*. + + .. code:: python + + # Test as written in the library + class EchoBaseTest(rfm.RunOnlyRegressionTest): + what = variable(str) + + valid_systems = ['*'] + valid_prog_environs = ['*'] + + @run_before('run') + def set_executable(self): + self.executable = f'echo {self.what}' + + @sanity_function + def assert_what(self): + return sn.assert_found(fr'{self.what}') + + + # Test as written by the user + @rfm.simple_test + class HelloTest(EchoBaseTest): + what = 'Hello' + + + # A parameterized test with type-checking + @rfm.simple_test + class FoodTest(EchoBaseTest): + param = parameter(['Bacon', 'Eggs']) + + @run_after('init') + def set_vars_with_params(self): + self.what = self.param + + + Similarly to a variable with a value already assigned to it, the value of + a required variable may be set either directly in the class body, on the + :func:`__init__` method, or in any other hook before it is referenced. + Otherwise an error will be raised indicating that a required variable has + not been set. Conversely, a variable with a default value already assigned + to it can be made required by assigning it the ``required`` keyword. + However, this ``required`` keyword is only available in the class body. + + .. code:: python + + class MyRequiredTest(HelloTest): + what = required + + + Running the above test will cause the :func:`set_exec_and_sanity` hook + from :class:`EchoBaseTest` to throw an error indicating that the variable + ``what`` has not been set. + + :param `types`: the supported types for the variable. + :param value: the default value assigned to the variable. If no value is + provided, the variable is set as ``required``. + :param field: the field validator to be used for this variable. If no + field argument is provided, it defaults to + :attr:`reframe.core.fields.TypedField`. The provided field validator + by this argument must derive from :attr:`reframe.core.fields.Field`. + :param loggable: Mark this variable as loggable. If :obj:`True`, this + variable will become a log record attribute under the name + ``check_NAME``, where ``NAME`` is the name of the variable. + :param `kwargs`: keyword arguments to be forwarded to the constructor of the + field validator. + :returns: A new test variable. + + .. versionadded:: 3.10.2 + The ``loggable`` argument is added. - :meta private: ''' __slots__ = ('_default_value', '_field', '_loggable', '_name') From 4b2339ae6a132522c0341dd05c6f935f581258f2 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 12 Mar 2022 21:06:36 +0100 Subject: [PATCH 3/7] Update docs --- docs/regression_test_api.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index f38c3a548c..ce4f575e65 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -1,6 +1,6 @@ -=================== -Regression Test API -=================== +================== +Test API Reference +================== This page provides a reference guide of the ReFrame API for writing regression tests covering all the relevant details. Internal data structures and APIs are covered only to the extent that this might be helpful to the final user of the framework. From 950030fe5081b9e6441dd2e986b1d53b3f0f1081 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 12 Mar 2022 21:56:41 +0100 Subject: [PATCH 4/7] Update `make_test()` docs and make them public --- docs/regression_test_api.rst | 9 +++++++++ reframe/core/meta.py | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index ce4f575e65..b3faf06d36 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -185,6 +185,15 @@ Therefore, classes that derive from the base :class:`~reframe.core.pipeline.Regr .. automethod:: reframe.core.pipeline.RegressionMixin.variant_name +------------------------- +Dynamic Creation of Tests +------------------------- + +.. versionadded:: 3.10.0 + + +.. autofunction:: reframe.core.meta.make_test + ------------------------ Environments and Systems diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 0d8d58afbd..8291bab8b6 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -838,13 +838,45 @@ class HelloTest(rfm.RunOnlyRegressionTest): hello_cls = HelloTest + Test :ref:`builtins ` can also be used when defining the body of + the test by accessing them through the :obj:`reframe.core.builtins`. + Methods can also be bound to the newly created tests using the ``methods`` + argument. The following is an example: + + .. code-block:: python + + import reframe.core.builtins as builtins + + + def set_message(obj): + obj.executable_opts = [obj.message] + + def validate(obj): + return sn.assert_found(obj.message, obj.stdout) + + hello_cls = rfm.make_test( + 'HelloTest', (rfm.RunOnlyRegressionTest,), + { + 'valid_systems': ['*'], + 'valid_prog_environs': ['*'], + 'executable': 'echo', + 'message': builtins.variable(str) + }, + methods=[ + builtins.run_before('run')(set_message), + builtins.sanity_function(validate) + ] + ) + + :param name: The name of the new test class. :param bases: A tuple of the base classes of the class that is being created. :param body: A mapping of key/value pairs that will be inserted as class attributes in the newly created class. - :param methods: A list of functions to be added as methods to the class - that is being created. + :param methods: A list of functions to be bound as methods to the class + that is being created. The functions will be bound with their original + name. :param kwargs: Any keyword arguments to be passed to the :class:`RegressionTestMeta` metaclass. From d3fdc599e0e61356f22a3f45461fa80baf30dc33 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 12 Mar 2022 22:25:53 +0100 Subject: [PATCH 5/7] Fix PEP8 issues --- reframe/core/fixtures.py | 31 +++++++++++++++++++++++-------- reframe/core/meta.py | 2 +- reframe/core/parameters.py | 10 +++++++--- reframe/core/variables.py | 7 ++++--- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/reframe/core/fixtures.py b/reframe/core/fixtures.py index 6e63639329..fad2d8b3a3 100644 --- a/reframe/core/fixtures.py +++ b/reframe/core/fixtures.py @@ -434,10 +434,18 @@ class MyFixture(rfm.RunOnlyRegressionTest): class TestA(rfm.RegressionTest): valid_systems = ['p1', 'p2'] valid_prog_environs = ['e1', 'e2'] - f1 = fixture(MyFixture, scope='session') # Shared throughout the full session - f2 = fixture(MyFixture, scope='partition') # Shared for each supported partition - f3 = fixture(MyFixture, scope='environment') # Shared for each supported part+environ - f4 = fixture(MyFixture, scope='test') # Private evaluation of MyFixture + + # Fixture shared throughout the full session + f1 = fixture(MyFixture, scope='session') + + # Fixture shared for each supported partition + f2 = fixture(MyFixture, scope='partition') + + # Fixture shared for each supported part+environ + f3 = fixture(MyFixture, scope='environment') + + # Fixture private evaluation of MyFixture + f4 = fixture(MyFixture, scope='test') ... @@ -445,9 +453,15 @@ class TestA(rfm.RegressionTest): class TestB(rfm.RegressionTest): valid_systems = ['p1'] valid_prog_environs = ['e1'] - f1 = fixture(MyFixture, scope='test') # Another private instance of MyFixture - f2 = fixture(MyFixture, scope='environment') # Same as f3 in TestA for p1 + e1 - f3 = fixture(MyFixture, scope='session') # Same as f1 in TestA + + # Another private instance of MyFixture + f1 = fixture(MyFixture, scope='test') + + # Same as f3 in TestA for p1 + e1 + f2 = fixture(MyFixture, scope='environment') + + # Same as f1 in TestA + f3 = fixture(MyFixture, scope='session') ... @run_after('setup') @@ -531,7 +545,8 @@ class TestD(rfm.RegressionTest): @run_after('setup') def reduce_range(self): # Sum all the values of p for each fixture variant - res = functools.reduce(lambda x, y: x+y, (fix.p for fix in self.f)) + res = functools.reduce(lambda x, y: x+y, + (fix.p for fix in self.f)) n = len(self.f)-1 assert res == (n*n + n)/2 diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 8291bab8b6..f8ee51c839 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -19,7 +19,7 @@ import reframe.core.hooks as hooks import reframe.utility as utils -from reframe.core.exceptions import ReframeSyntaxError, ReframeFatalError +from reframe.core.exceptions import ReframeSyntaxError from reframe.core.runtime import runtime diff --git a/reframe/core/parameters.py b/reframe/core/parameters.py index 9c57255539..6047e513b1 100644 --- a/reframe/core/parameters.py +++ b/reframe/core/parameters.py @@ -31,7 +31,9 @@ class TestParam: class Foo(rfm.RegressionTest): variant = parameter(['A', 'B']) - # print(variant) # Error: a parameter may only be accessed from the class instance. + + # print(variant) + # Error: a parameter may only be accessed from the class instance @run_after('init') def do_something(self): @@ -67,11 +69,13 @@ def do_something(self): .. code:: python class ExtendVariant(Bar): - # Extend the full set of inherited variant parameter values to ['A', 'B', 'C'] + # Extend the full set of inherited variant parameter values + # to ['A', 'B', 'C'] variant = parameter(['C'], inherit_params=True) class PartiallyExtendVariant(Bar): - # Extend a subset of the inherited variant parameter values to ['A', 'D'] + # Extend a subset of the inherited variant parameter values + # to ['A', 'D'] variant = parameter(['D'], inherit_params=True, filter_params=lambda x: x[:1]) diff --git a/reframe/core/variables.py b/reframe/core/variables.py index e6013205f8..4c8ef6e32d 100644 --- a/reframe/core/variables.py +++ b/reframe/core/variables.py @@ -55,8 +55,8 @@ class Foo(rfm.RegressionTest): @run_after('init') def access_vars(self): print(self.my_var) # prints 8. - # self.my_var = 'override' # Error: my_var must be an int! - self.not_a_var = 'override' # However, this would work. Dangerous! + # self.my_var = 'override' # Error: my_var must be an int! + self.not_a_var = 'override' # This will work, but is dangerous! self.my_var = 10 # tests may also assign values the standard way Here, the argument ``value`` in the :func:`variable` built-in sets the @@ -78,7 +78,8 @@ class Bar(Foo): # Since my_var is available, we can also update its value: my_var = 4 - # Bar inherits the full declaration of my_var with the original type-checking. + # Bar inherits the full declaration of my_var with the original + # type-checking. # my_var = 'override' # Wrong type error again! @run_after('init') From 303405a721fa6d1fcd871fe401aca2acf1ffce33 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 12 Mar 2022 22:28:24 +0100 Subject: [PATCH 6/7] Fix PEP8 issues --- reframe/core/fixtures.py | 3 ++- reframe/core/variables.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/reframe/core/fixtures.py b/reframe/core/fixtures.py index fad2d8b3a3..d6f7a41d3d 100644 --- a/reframe/core/fixtures.py +++ b/reframe/core/fixtures.py @@ -586,7 +586,8 @@ class ComplexFixture(rfm.RegressionTest): class TestE(rfm.RegressionTest): # Select the fixture variants with boolean conditions foo = fixture(ComplexFixture, - variants={'p0': lambda x: x<10, 'p1': lambda x: x=='d'}) + variants={'p0': lambda x: x<10, + 'p1': lambda x: x=='d'}) # Select the fixture variants by index bar = fixture(ComplexFixture, variants=range(300,310)) diff --git a/reframe/core/variables.py b/reframe/core/variables.py index 4c8ef6e32d..48da55e387 100644 --- a/reframe/core/variables.py +++ b/reframe/core/variables.py @@ -168,8 +168,8 @@ class MyRequiredTest(HelloTest): :param loggable: Mark this variable as loggable. If :obj:`True`, this variable will become a log record attribute under the name ``check_NAME``, where ``NAME`` is the name of the variable. - :param `kwargs`: keyword arguments to be forwarded to the constructor of the - field validator. + :param `kwargs`: keyword arguments to be forwarded to the constructor of + the field validator. :returns: A new test variable. .. versionadded:: 3.10.2 From 6a8f36a8305e0bace896c0cbc298e9646a915460 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 15 Mar 2022 20:15:56 +0100 Subject: [PATCH 7/7] Remove stale comment --- reframe/core/meta.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reframe/core/meta.py b/reframe/core/meta.py index f8ee51c839..eb45fa40a9 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -893,10 +893,13 @@ def validate(obj): for m in methods: body[m.__name__] = m + # We update the namespace with the body of the class and we explicitly + # call reset on each namespace key to trigger the functionality of + # `__setitem__()` as if the body elements were actually being typed in the + # class definition namespace.update(body) for k in list(namespace.keys()): namespace.reset(k) - # namespace.update(body) cls = RegressionTestMeta(name, bases, namespace, **kwargs) return cls