diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 9458ab607c..7b558f5d39 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -428,16 +428,29 @@ System Partition Configuration .. versionadded:: 3.5.0 +.. js:attribute:: .systems[].partitions[].features + + :required: No + :default: ``[]`` + + User defined features of the partition. + These are accessible through the :attr:`~reframe.core.systems.SystemPartition.features` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.current_partition` and can also be selected through the extended syntax of :attr:`~reframe.core.pipeline.RegressionTest.valid_systems`. + The values of this list must be alphanumeric strings starting with a non-digit character and may also contain a ``-``. + + .. versionadded:: 3.11.0 + + .. js:attribute:: .systems[].partitions[].extras :required: No :default: ``{}`` - User defined attributes of the partition. This will be accessible through the :attr:`~reframe.core.systems.SystemPartition.extras` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.current_partition`. + User defined attributes of the partition. + These are accessible through the :attr:`~reframe.core.systems.SystemPartition.extras` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.current_partition` and can also be selected through the extended syntax of :attr:`~reframe.core.pipeline.RegressionTest.valid_systems`. + The attributes of this object must be alphanumeric strings starting with a non-digit character and their values can be of any type. .. versionadded:: 3.5.0 - .. _container-platform-configuration: @@ -598,12 +611,26 @@ They are associated with `system partitions <#system-partition-configuration>`__ Variables are set after the environment modules are loaded. +.. js:attribute:: .environments[].features + + :required: No + :default: ``[]`` + + User defined features of the environment. + These are accessible through the :attr:`~reframe.core.environments.Environment.features` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.current_environ` and can also be selected through the extended syntax of :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs`. + The values of this list must be alphanumeric strings starting with a non-digit character and may also contain a ``-``. + + .. versionadded:: 3.11.0 + + .. js:attribute:: .environments[].extras :required: No :default: ``{}`` - User defined attributes of the environment. This will be accessible through the :attr:`~reframe.core.environments.Environment.extras` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.current_environ`. + User defined attributes of the environment. + These are accessible through the :attr:`~reframe.core.environments.Environment.extras` attribute of the :attr:`~reframe.core.pipeline.RegressionTest.current_environ` and can also be selected through the extended syntax of :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs`. + The attributes of this object must be alphanumeric strings starting with a non-digit character and their values can be of any type. .. versionadded:: 3.9.1 diff --git a/reframe/core/decorators.py b/reframe/core/decorators.py index ed33b410ea..997cd25b67 100644 --- a/reframe/core/decorators.py +++ b/reframe/core/decorators.py @@ -64,8 +64,13 @@ def skip(self, test): '''Add a test to the skip set.''' self._skip_tests.add(test) - def instantiate_all(self): - '''Instantiate all the registered tests.''' + def instantiate_all(self, reset_sysenv=0): + '''Instantiate all the registered tests. + + :param reset_sysenv: Reset valid_systems and valid_prog_environs after + instantiating the tests. Bit 0 resets the valid_systems, bit 1 + resets the valid_prog_environs. + ''' # We first instantiate the leaf tests and then walk up their # dependencies to instantiate all the fixtures. Fixtures can only @@ -79,6 +84,7 @@ def instantiate_all(self): for args, kwargs in variants: try: + kwargs['reset_sysenv'] = reset_sysenv leaf_tests.append(test(*args, **kwargs)) except SkipTestError as e: getlogger().warning( diff --git a/reframe/core/environments.py b/reframe/core/environments.py index fc0ecd283f..3df845a2e2 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -37,7 +37,8 @@ class Environment(jsonext.JSONSerializable): Users may not create :class:`Environment` objects directly. ''' - def __init__(self, name, modules=None, variables=None, extras=None): + def __init__(self, name, modules=None, variables=None, + extras=None, features=None): modules = modules or [] variables = variables or [] self._name = name @@ -45,6 +46,7 @@ def __init__(self, name, modules=None, variables=None, extras=None): self._module_names = [m['name'] for m in self._modules] self._variables = collections.OrderedDict(variables) self._extras = extras or {} + self._features = features or [] @property def name(self): @@ -92,7 +94,7 @@ def variables(self): @property def extras(self): - '''User defined properties defined in the configuration. + '''User defined properties specified in the configuration. .. versionadded:: 3.9.1 @@ -101,6 +103,16 @@ def extras(self): return self._extras + @property + def features(self): + '''Used defined features specified in the configuration. + + .. versionadded:: 3.11.0 + + :type: :class:`List[str]` + ''' + return self._features + def __eq__(self, other): if not isinstance(other, type(self)): return NotImplemented @@ -116,7 +128,8 @@ def __repr__(self): return (f'{type(self).__name__}(' f'name={self._name!r}, ' f'modules={self._modules!r}, ' - f'variables={list(self._variables.items())!r})') + f'variables={list(self._variables.items())!r}, ' + f'extras={self._extras!r}, features={self._features!r})') class _EnvironmentSnapshot(Environment): @@ -174,6 +187,7 @@ def __init__(self, modules=None, variables=None, extras=None, + features=None, cc='cc', cxx='CC', ftn='ftn', @@ -184,7 +198,7 @@ def __init__(self, fflags=None, ldflags=None, **kwargs): - super().__init__(name, modules, variables, extras) + super().__init__(name, modules, variables, extras, features) self._cc = cc self._cxx = cxx self._ftn = ftn diff --git a/reframe/core/fixtures.py b/reframe/core/fixtures.py index 6141bca3ea..c52a068dd2 100644 --- a/reframe/core/fixtures.py +++ b/reframe/core/fixtures.py @@ -119,7 +119,7 @@ def __init__(self): # Store the system name for name-mangling purposes self._sys_name = runtime.runtime().system.name - def add(self, fixture, variant_num, parent_name, partitions, prog_envs): + def add(self, fixture, variant_num, parent_test): '''Register a fixture. This method mangles the fixture name, ensuring that different fixture @@ -161,11 +161,7 @@ def add(self, fixture, variant_num, parent_name, partitions, prog_envs): :param fixture: An instance of :class:`TestFixture`. :param variant_num: The variant index for the given ``fixture``. - :param parent_name: The full name of the parent test. This argument - is used to mangle the fixture name for those with a ``'test'`` - scope, such that the fixture is private to its parent test. - :param partitions: The system partitions supported by the parent test. - :param prog_envs: The valid programming environments from the parent. + :param parent_test: The parent test. ''' cls = fixture.cls @@ -187,51 +183,65 @@ def add(self, fixture, variant_num, parent_name, partitions, prog_envs): fname += vname # Select only the valid partitions - valid_partitions = self._filter_valid_partitions(partitions) + try: + valid_sysenv = runtime.valid_sysenv_comb( + parent_test.valid_systems, + parent_test.valid_prog_environs + ) + except AttributeError as e: + msg = e.args[0] + f' in test {parent_test.display_name!r}' + raise ReframeSyntaxError(msg) from None - # Return if not any valid partition - if not valid_partitions: + # Return if there are no valid system/environment combinations + if not valid_sysenv: return [] # Register the fixture if scope == 'session': # The name is mangled with the system name - # Select a valid environment supported by a partition - for part in valid_partitions: - valid_envs = self._filter_valid_environs(part, prog_envs) - if valid_envs: + # Pick the first valid system/environment combination + pname, ename = None, None + for part, environs in valid_sysenv.items(): + pname = part.fullname + for env in environs: + ename = env.name break - else: + + if ename is None: + # No valid environments found return [] # Register the fixture - fixt_data = FixtureData(variant_num, [valid_envs[0]], [part], + fixt_data = FixtureData(variant_num, [ename], [pname], variables, scope, self._sys_name) name = f'{cls.__name__}_{fixt_data.mashup()}' self._registry[cls][name] = fixt_data reg_names.append(name) elif scope == 'partition': - for part in valid_partitions: + for part, environs in valid_sysenv.items(): # The mangled name contains the full partition name - # Select an environment supported by the partition - valid_envs = self._filter_valid_environs(part, prog_envs) - if not valid_envs: + pname = part.fullname + try: + ename = environs[0].name + except IndexError: continue # Register the fixture - fixt_data = FixtureData(variant_num, [valid_envs[0]], [part], - variables, scope, part) + fixt_data = FixtureData(variant_num, [ename], [pname], + variables, scope, pname) name = f'{cls.__name__}_{fixt_data.mashup()}' self._registry[cls][name] = fixt_data reg_names.append(name) elif scope == 'environment': - for part in valid_partitions: - for env in self._filter_valid_environs(part, prog_envs): + for part, environs in valid_sysenv.items(): + for env in environs: # The mangled name contains the full part and env names # Register the fixture - fixt_data = FixtureData(variant_num, [env], [part], - variables, scope, f'{part}+{env}') + pname, ename = part.fullname, env.name + fixt_data = FixtureData(variant_num, [ename], [pname], + variables, scope, + f'{pname}+{ename}') name = f'{cls.__name__}_{fixt_data.mashup()}' self._registry[cls][name] = fixt_data reg_names.append(name) @@ -239,9 +249,10 @@ def add(self, fixture, variant_num, parent_name, partitions, prog_envs): # The mangled name contains the parent test name. # Register the fixture - fixt_data = FixtureData(variant_num, list(prog_envs), - list(valid_partitions), - variables, scope, parent_name) + fixt_data = FixtureData(variant_num, + list(parent_test.valid_prog_environs), + list(parent_test.valid_systems), + variables, scope, parent_test.unique_name) name = f'{cls.__name__}_{fixt_data.mashup()}' self._registry[cls][name] = fixt_data reg_names.append(name) @@ -986,18 +997,13 @@ def inject(self, obj, cls=None, fixtures_index=None): # Create the fixture registry obj._rfm_fixture_registry = FixtureRegistry() - # Prepare the partitions and prog_envs - part, prog_envs = self._expand_partitions_envs(obj) - # Register the fixtures for name, fixture in self.fixtures.items(): dep_names = [] for variant in fixture_variants[name]: # Register all the variants and track the fixture names dep_names += obj._rfm_fixture_registry.add(fixture, - variant, - obj.unique_name, - part, prog_envs) + variant, obj) # Add dependencies if fixture.scope == 'session': @@ -1011,38 +1017,6 @@ def inject(self, obj, cls=None, fixtures_index=None): for dep_name in dep_names: obj.depends_on(dep_name, dep_kind) - def _expand_partitions_envs(self, obj): - '''Process the partitions and programming environs of the parent.''' - - try: - part = tuple(obj.valid_systems) - except AttributeError: - raise ReframeSyntaxError( - f"'valid_systems' is undefined in test {obj.unique_name!r}" - ) - else: - rt = runtime.runtime() - if '*' in part or rt.system.name in part: - part = tuple(p.fullname for p in rt.system.partitions) - - try: - prog_envs = tuple(obj.valid_prog_environs) - except AttributeError: - raise ReframeSyntaxError( - f"'valid_prog_environs' is undefined " - f"in test {obj.unique_name!r}" - ) - else: - if '*' in prog_envs: - all_pes = set() - for p in runtime.runtime().system.partitions: - for e in p.environs: - all_pes.add(e.name) - - prog_envs = tuple(all_pes) - - return part, prog_envs - def __iter__(self): '''Walk through all index combinations for all fixtures.''' yield from self.__variant_combinations diff --git a/reframe/core/meta.py b/reframe/core/meta.py index 1df20fb752..cb23d60d97 100644 --- a/reframe/core/meta.py +++ b/reframe/core/meta.py @@ -397,6 +397,7 @@ def __call__(cls, *args, **kwargs): # Intercept the requested variant number (if any) and map it to the # respective points in the parameter and fixture spaces. variant_num = kwargs.pop('variant_num', None) + reset_sysenv = kwargs.pop('reset_sysenv', 0) param_index, fixt_index = cls._map_variant_num(variant_num) fixt_name = kwargs.pop('fixt_name', None) fixt_data = kwargs.pop('fixt_data', None) @@ -437,6 +438,11 @@ def __call__(cls, *args, **kwargs): setattr(obj, fname, fixtures.FixtureProxy(finfo)) obj.__init__(*args, **kwargs) + if reset_sysenv & 1: + obj.valid_systems = ['*'] + + if reset_sysenv & 2: + obj.valid_prog_environs = ['*'] # Register the fixtures # Fixtures must be injected after the object's initialisation because diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 85cc6b92c7..b711749ce9 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -82,6 +82,19 @@ DEPEND_FULLY = 3 +# Valid systems/environments mini-language +_N = r'(\w[-.\w]*)' # name +_NW = rf'(\*|{_N})' # name or wildcard +_F = rf'([+-]{_N})' # feature +_OP = r'([=<>]|!=|>=|<=)' # relational operator (unused for the moment) +_KV = rf'(%{_N}=\S+)' # key/value pair +_FKV = rf'({_F}|{_KV})' # feature | key/value pair +_VALID_ENV_SYNTAX = rf'^({_NW}|{_FKV}(\s+{_FKV})*)$' + +_S = rf'({_NW}(:{_NW})?)' # system/partition +_VALID_SYS_SYNTAX = rf'^({_S}|{_FKV}(\s+{_FKV})*)$' + + _PIPELINE_STAGES = ( '__init__', 'setup', @@ -215,11 +228,11 @@ def pipeline_hooks(cls): #: #: .. warning:: #: - #: Setting the name of a test is deprecated and will be disabled in the - #: future. If you were setting the name of a test to circumvent the old - #: long parameterized test names in order to reference them in - #: dependency chains, please refer to :ref:`param_deps` for more details on how - #: to achieve this. + #: Setting the name of a test is deprecated and will be disabled in the + #: future. If you were setting the name of a test to circumvent the old + #: long parameterized test names in order to reference them in + #: dependency chains, please refer to :ref:`param_deps` for more details + #: on how to achieve this. #: #: .. versionchanged:: 3.10.0 #: Setting the :attr:`name` attribute is deprecated. @@ -229,42 +242,101 @@ def pipeline_hooks(cls): "setting the 'name' attribute is deprecated and " "will be disabled in the future", DEPRECATE_WR) - #: List of programming environments supported by this test. + #: List of environments or environment features or environment properties + #: required by this test. #: - #: If ``*`` is in the list then all programming environments are supported - #: by this test. + #: The syntax of this attribute is exactly the same as of the + #: :attr:`valid_systems` except that the ``a:b`` entries are invalid. #: #: :type: :class:`List[str]` #: :default: ``required`` #: - #: .. note:: - #: .. versionchanged:: 2.12 - #: Programming environments can now be specified using wildcards. + #: .. seealso:: + #: - `Environment features + #: `__ + #: - `Environment extras + #: `__ #: - #: .. versionchanged:: 2.17 - #: Support for wildcards is dropped. + #: .. versionchanged:: 2.12 + #: Programming environments can now be specified using wildcards. + #: + #: .. versionchanged:: 2.17 + #: Support for wildcards is dropped. + #: + #: .. versionchanged:: 3.3 + #: Default value changed from ``[]`` to ``None``. #: - #: .. versionchanged:: 3.3 - #: Default value changed from ``[]`` to ``None``. + #: .. versionchanged:: 3.6 + #: Default value changed from ``None`` to ``required``. #: - #: .. versionchanged:: 3.6 - #: Default value changed from ``None`` to ``required``. - valid_prog_environs = variable(typ.List[str], loggable=True) + #: .. versionchanged:: 3.11.0 + #: Extend syntax to support features and key/value pairs. + valid_prog_environs = variable(typ.List[typ.Str[_VALID_ENV_SYNTAX]], + loggable=True) - #: List of systems supported by this test. - #: The general syntax for systems is ``[:]``. - #: Both and accept the value ``*`` to mean any value. - #: ``*`` is an alias of ``*:*`` + #: List of systems or system features or system properties required by this + #: test. + #: + #: Each entry in this list is a requirement and can have one of the + #: following forms: + #: + #: - ``sysname``: The test is valid for system named ``sysname``. + #: - ``sysname:partname``: The test is valid for the partition ``partname`` + #: of system ``sysname``. + #: - ``*``: The test is valid for any system. + #: - ``*:partname``: The test is valid for any partition named ``partname`` + #: in any system. + #: - ``+feat``: The test is valid for all partitions that define feature + #: ``feat`` as a feature. + #: - ``-feat``: The test is valid for all partitions that do not define + #: feature ``feat`` as a feature. + #: - ``%key=val``: The test is valid for all partitions that define the + #: extra property ``key`` with the value ``val``. + #: + #: Multiple features and key/value pairs can be included in a single entry + #: of the :attr:`valid_systems` list, in which case an AND operation on + #: these constraints is implied. For example, the test defining the + #: following will be valid for all systems that have define both ``feat1`` + #: and ``feat2`` and set ``foo=1`` + #: + #: .. code-block:: python + #: + #: valid_systems = ['+feat1 +feat2 %foo=1'] + #: + #: For key/value pairs comparisons, ReFrame will automatically convert the + #: value in the key/value spec to the type of the value of the + #: corresponding entry in the partitions ``extras`` property. In the above + #: example, if the type of ``foo`` property is integer, ``1`` will be + #: converted to an integer value. If a conversion to the target type is not + #: possible, then the requested key/value pair is not matched. + #: + #: Multiple entries in the :attr:`valid_systems` list are implicitly ORed, + #: such that the following example implies that the test is valid for + #: either ``sys1`` or for any other system that does not define ``feat``. + #: + #: .. code-block:: python + #: + #: valid_systems = ['sys1', '-feat'] #: #: :type: :class:`List[str]` #: :default: ``None`` #: - #: .. versionchanged:: 3.3 - #: Default value changed from ``[]`` to ``None``. + #: .. seealso:: + #: - `System partition features + #: `__ + #: - `System partition extras + #: `__ #: - #: .. versionchanged:: 3.6 - #: Default value changed from ``None`` to ``required``. - valid_systems = variable(typ.List[str], loggable=True) + #: .. versionchanged:: 3.3 + #: Default value changed from ``[]`` to ``None``. + #: + #: .. versionchanged:: 3.6 + #: Default value changed from ``None`` to ``required``. + #: + #: .. versionchanged:: 3.11.0 + #: Extend syntax to support features and key/value pairs. + valid_systems = variable(typ.List[typ.Str[_VALID_SYS_SYNTAX]], + loggable=True) #: A detailed description of the test. #: @@ -1000,7 +1072,7 @@ def _process_hook_registry(cls): '''Process and validate the pipeline hooks.''' _pipeline_hooks = {} - for stage, hooks in cls.pipeline_hooks().items(): + for stage, hks in cls.pipeline_hooks().items(): # Pop the stage pre_/post_ prefix stage_name = stage.split('_', maxsplit=1)[1] @@ -1020,7 +1092,7 @@ def _process_hook_registry(cls): elif stage == 'post_run': stage = 'post_run_wait' - _pipeline_hooks[stage] = hooks + _pipeline_hooks[stage] = hks return _pipeline_hooks @@ -1381,23 +1453,6 @@ def info(self): return ret - def supports_system(self, name): - if name.find(':') != -1: - system, partition = name.split(':') - else: - system, partition = self.current_system.name, name - - valid_matches = ['*', '*:*', system, f'{system}:*', - f'*:{partition}', f'{system}:{partition}'] - - return any(n in self.valid_systems for n in valid_matches) - - def supports_environ(self, env_name): - if '*' in self.valid_prog_environs: - return True - - return env_name in self.valid_prog_environs - def is_local(self): '''Check if the test will execute locally. diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index 698f82409e..1c34455d3c 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -267,6 +267,113 @@ def is_env_loaded(environ): for k, v in environ.variables.items())) +def _is_valid_part(part, valid_systems): + for spec in valid_systems: + if spec[0] not in ('+', '-', '%'): + # This is the standard case + sysname, partname = part.fullname.split(':') + valid_matches = ['*', '*:*', sysname, f'{sysname}:*', + f'*:{partname}', f'{part.fullname}'] + if spec in valid_matches: + return True + else: + plus_feats = [] + minus_feats = [] + props = {} + for subspec in spec.split(' '): + if subspec.startswith('+'): + plus_feats.append(subspec[1:]) + elif subspec.startswith('-'): + minus_feats.append(subspec[1:]) + elif subspec.startswith('%'): + key, val = subspec[1:].split('=') + props[key] = val + + have_plus_feats = all( + ft in part.features or ft in part.resources + for ft in plus_feats + ) + have_minus_feats = any( + ft in part.features or ft in part.resources + for ft in minus_feats + ) + try: + have_props = True + for k, v in props.items(): + extra_value = part.extras[k] + extra_type = type(extra_value) + if extra_value != extra_type(v): + have_props = False + break + except (KeyError, ValueError): + have_props = False + + if have_plus_feats and not have_minus_feats and have_props: + return True + + return False + + +def _is_valid_env(env, valid_prog_environs): + if '*' in valid_prog_environs: + return True + + for spec in valid_prog_environs: + if spec[0] not in ('+', '-', '%'): + # This is the standard case + if env.name == spec: + return True + else: + plus_feats = [] + minus_feats = [] + props = {} + for subspec in spec.split(' '): + if subspec.startswith('+'): + plus_feats.append(subspec[1:]) + elif subspec.startswith('-'): + minus_feats.append(subspec[1:]) + elif subspec.startswith('%'): + key, val = subspec[1:].split('=') + props[key] = val + + have_plus_feats = all(ft in env.features for ft in plus_feats) + have_minus_feats = any(ft in env.features + for ft in minus_feats) + try: + have_props = True + for k, v in props.items(): + extra_value = env.extras[k] + extra_type = type(extra_value) + if extra_value != extra_type(v): + have_props = False + break + except (KeyError, ValueError): + have_props = False + + if have_plus_feats and not have_minus_feats and have_props: + return True + + return False + + +def valid_sysenv_comb(valid_systems, valid_prog_environs, + check_systems=True, check_environs=True): + ret = {} + curr_sys = runtime().system + for part in curr_sys.partitions: + if check_systems and not _is_valid_part(part, valid_systems): + continue + + ret[part] = [] + for env in part.environs: + if check_environs and not _is_valid_env(env, valid_prog_environs): + continue + + ret[part].append(env) + + return ret + + class temp_environment: '''Context manager to temporarily change the environment.''' diff --git a/reframe/core/systems.py b/reframe/core/systems.py index ca31640f6f..f752c691c5 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -166,7 +166,7 @@ class SystemPartition(jsonext.JSONSerializable): def __init__(self, *, parent, name, sched_type, launcher_type, descr, access, container_environs, resources, local_env, environs, max_jobs, prepare_cmds, - processor, devices, extras, time_limit): + processor, devices, extras, features, time_limit): getlogger().debug(f'Initializing system partition {name!r}') self._parent_system = parent self._name = name @@ -184,6 +184,7 @@ def __init__(self, *, parent, name, sched_type, launcher_type, self._processor = ProcessorInfo(processor) self._devices = [DeviceInfo(d) for d in devices] self._extras = extras + self._features = features self._time_limit = time_limit @property @@ -377,7 +378,9 @@ def devices(self): @property def extras(self): - '''User defined properties defined in the configuration. + '''User defined properties associated with this partition. + + These extras are defined in the configuration. .. versionadded:: 3.5.0 @@ -385,6 +388,18 @@ def extras(self): ''' return self._extras + @property + def features(self): + '''User defined features associated with this partition. + + These features are defined in the configuration. + + .. versionadded:: 3.11.0 + + :type: :class:`List[str]` + ''' + return self._features + def __eq__(self, other): if not isinstance(other, type(self)): return NotImplemented @@ -492,6 +507,7 @@ def create(cls, site_config): modules=site_config.get(f'environments/@{e}/modules'), variables=site_config.get(f'environments/@{e}/variables'), extras=site_config.get(f'environments/@{e}/extras'), + features=site_config.get(f'environments/@{e}/features'), cc=site_config.get(f'environments/@{e}/cc'), cxx=site_config.get(f'environments/@{e}/cxx'), ftn=site_config.get(f'environments/@{e}/ftn'), @@ -524,6 +540,7 @@ def create(cls, site_config): processor=site_config.get(f'{partid}/processor'), devices=site_config.get(f'{partid}/devices'), extras=site_config.get(f'{partid}/extras'), + features=site_config.get(f'{partid}/features'), time_limit=site_config.get(f'{partid}/time_limit') ) ) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 336c8e3b04..1cc245edee 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -896,7 +896,9 @@ def restrict_logging(): loader = RegressionCheckLoader(check_search_path, check_search_recursive, - external_vars) + external_vars, + options.skip_system_check, + options.skip_prgenv_check) def print_infoline(param, value): param = param + ':' @@ -931,15 +933,16 @@ def print_infoline(param, value): print_infoline('output directory', repr(session_info['prefix_output'])) printer.info('') try: - # Locate and load checks - checks_found = loader.load_all() + # Locate and load checks; `force=True` is not needed for normal + # invocations from the command line and has practically no effect, but + # it is needed to better emulate the behavior of running reframe's CLI + # from within the unit tests, which call repeatedly `main()`. + checks_found = loader.load_all(force=True) printer.verbose(f'Loaded {len(checks_found)} test(s)') # Generate all possible test cases first; we will need them for # resolving dependencies after filtering - testcases_all = generate_testcases(checks_found, - options.skip_system_check, - options.skip_prgenv_check) + testcases_all = generate_testcases(checks_found) testcases = testcases_all printer.verbose(f'Generated {len(testcases)} test case(s)') @@ -1173,7 +1176,6 @@ def module_unuse(*paths): printer.error("unknown execution policy `%s': Exiting...") sys.exit(1) - exec_policy.skip_system_check = options.skip_system_check exec_policy.force_local = options.force_local exec_policy.strict_check = options.strict exec_policy.skip_sanity_check = options.skip_sanity_check diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 61707cf7d7..795d2869c5 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -97,27 +97,17 @@ def clone(self): return TestCase(self._check_orig, self._partition, self._environ) -def generate_testcases(checks, - skip_system_check=False, - skip_environ_check=False): +def generate_testcases(checks): '''Generate concrete test cases from checks.''' - def supports_partition(c, p): - return skip_system_check or c.supports_system(p.fullname) - - def supports_environ(c, e): - return skip_environ_check or c.supports_environ(e.name) - rt = runtime.runtime() cases = [] for c in checks: - for p in rt.system.partitions: - if not supports_partition(c, p): - continue - - for e in p.environs: - if supports_environ(c, e): - cases.append(TestCase(c, p, e)) + valid_comb = runtime.valid_sysenv_comb(c.valid_systems, + c.valid_prog_environs) + for part, environs in valid_comb.items(): + for env in environs: + cases.append(TestCase(c, part, env)) return cases @@ -549,9 +539,7 @@ class ExecutionPolicy(abc.ABC): def __init__(self): # Options controlling the check execution - self.skip_system_check = False self.force_local = False - self.skip_environ_check = False self.skip_sanity_check = False self.skip_performance_check = False self.keep_stage_files = False diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index 13f945269a..9fbee52ecc 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -40,7 +40,8 @@ def visit_ImportFrom(self, node): class RegressionCheckLoader: - def __init__(self, load_path, recurse=False, external_vars=None): + def __init__(self, load_path, recurse=False, external_vars=None, + skip_system_check=False, skip_prgenv_check=False): # Expand any environment variables and symlinks load_path = [os.path.realpath(osext.expandvars(p)) for p in load_path] self._load_path = osext.unique_abs_paths(load_path, recurse) @@ -51,6 +52,8 @@ def __init__(self, load_path, recurse=False, external_vars=None): # Variables set in the command line self._external_vars = external_vars or {} + self._skip_system_check = bool(skip_system_check) + self._skip_prgenv_check = bool(skip_prgenv_check) def _module_name(self, filename): '''Figure out a module name from filename. @@ -170,7 +173,12 @@ def load_from_module(self, module): return [] self._set_defaults(registry) - candidate_tests = registry.instantiate_all() if registry else [] + reset_sysenv = self._skip_prgenv_check << 1 | self._skip_system_check + if registry: + candidate_tests = registry.instantiate_all(reset_sysenv) + else: + candidate_tests = [] + legacy_tests = legacy_registry() if legacy_registry else [] if self._external_vars and legacy_tests: getlogger().warning( @@ -179,6 +187,15 @@ def load_from_module(self, module): "please use the 'parameter' builtin in your tests" ) + # Reset valid_systems and valid_prog_environs in all legacy tests + if reset_sysenv: + for t in legacy_tests: + if self._skip_system_check: + t.valid_systems = ['*'] + + if self._skip_prgenv_check: + t.valid_prog_environs = ['*'] + # Merge tests candidate_tests += legacy_tests diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 2f72158748..59c8286b4b 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -5,7 +5,11 @@ "defs": { "alphanum_string": { "type": "string", - "pattern": "([a-zA-Z0-9_]|-)+" + "pattern": "([a-zA-Z_][a-zA-Z0-9_]*)" + }, + "alphanum_ext_string": { + "type": "string", + "pattern": "([a-zA-Z_]([a-zA-Z0-9_]|-)*)" }, "system_ref": { "type": "array", @@ -15,13 +19,7 @@ "type": "array", "items": { "type": "array", - "items": [ - { - "type": "string", - "pattern": "([a-zA-Z_][a-zA-Z0-9_]*)" - }, - {"type": "string"} - ], + "items": [{"$ref": "#/defs/alphanum_ext_string"}], "additionalProperties": false } }, @@ -220,7 +218,7 @@ "items": { "type": "object", "properties": { - "name": {"$ref": "#/defs/alphanum_string"}, + "name": {"$ref": "#/defs/alphanum_ext_string"}, "descr": {"type": "string"}, "hostnames": { "type": "array", @@ -243,7 +241,7 @@ "items": { "type": "object", "properties": { - "name": {"$ref": "#/defs/alphanum_string"}, + "name": {"$ref": "#/defs/alphanum_ext_string"}, "descr": {"type": "string"}, "scheduler": { "type": "string", @@ -300,7 +298,16 @@ }, "processor": {"$ref": "#/defs/processor_info"}, "devices": {"$ref": "#/defs/devices"}, - "extras": {"type": "object"}, + "features": { + "type": "array", + "items": {"$ref": "#/defs/alphanum_string"} + }, + "extras": { + "type": "object", + "propertyNames": { + "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" + } + }, "resources": { "type": "array", "items": { @@ -332,7 +339,7 @@ "items": { "type": "object", "properties": { - "name": {"type": "string"}, + "name": {"$ref": "#/defs/alphanum_ext_string"}, "modules": {"$ref": "#/defs/modules_list"}, "variables": {"$ref": "#/defs/envvar_list"}, "cc": {"type": "string"}, @@ -358,7 +365,16 @@ "type": "array", "items": {"type": "string"} }, - "extras": {"type": "object"}, + "extras": { + "type": "object", + "propertyNames": { + "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$" + } + }, + "features": { + "type": "array", + "items": {"$ref": "#/defs/alphanum_string"} + }, "target_systems": {"$ref": "#/defs/system_ref"} }, "required": ["name"], @@ -499,6 +515,7 @@ "environments/fflags": [], "environments/ldflags": [], "environments/extras": {}, + "environments/features": [], "environments/target_systems": ["*"], "general/dump_pipeline_progress": false, "general/pipeline_timeout": null, @@ -564,6 +581,7 @@ "systems/partitions/container_platforms": [], "systems/partitions/container_platforms/*modules": [], "systems/partitions/container_platforms/*variables": [], + "systems/partitions/features": [], "systems/partitions/resources": [], "systems/partitions/resources/options": [], "systems/partitions/modules": [], diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py index cad6c4f8a3..5d2d349c13 100644 --- a/unittests/resources/settings.py +++ b/unittests/resources/settings.py @@ -58,7 +58,8 @@ 'scheduler': 'local', 'launcher': 'local', 'environs': ['PrgEnv-cray', 'PrgEnv-gnu'], - 'descr': 'Login nodes' + 'descr': 'Login nodes', + 'features': ['cross_compile'] }, { 'name': 'gpu', @@ -84,6 +85,10 @@ ] } ], + 'features': ['cuda', 'mpi'], + 'extras': { + 'gpu_arch': 'a100' + }, 'environs': ['PrgEnv-gnu', 'builtin'], 'max_jobs': 10, 'processor': { @@ -195,6 +200,7 @@ 'cc': 'gcc', 'cxx': 'g++', 'ftn': 'gfortran', + 'features': ['cxx14'], 'extras': { 'foo': 1, 'bar': 'x' @@ -204,6 +210,7 @@ { 'name': 'PrgEnv-cray', 'modules': ['PrgEnv-cray'], + 'features': ['cxx14', 'mpi'], }, { 'name': 'builtin', diff --git a/unittests/test_config.py b/unittests/test_config.py index c68a08e504..8cd93e111c 100644 --- a/unittests/test_config.py +++ b/unittests/test_config.py @@ -261,7 +261,10 @@ def test_select_subconfig(): [{'name': 'PrgEnv-cray', 'collection': False, 'path': None}]) assert site_config.get('environments/@PrgEnv-gnu/extras') == {'foo': 1, 'bar': 'x'} + assert site_config.get('environments/@PrgEnv-gnu/features') == ['cxx14'] assert site_config.get('environments/@PrgEnv-cray/extras') == {} + assert site_config.get('environments/@PrgEnv-cray/features') == ['cxx14', + 'mpi'] assert len(site_config.get('general')) == 1 assert site_config.get('general/0/check_search_path') == ['a:b'] diff --git a/unittests/test_fixtures.py b/unittests/test_fixtures.py index d7e9d42a9f..6563fb7ab8 100644 --- a/unittests/test_fixtures.py +++ b/unittests/test_fixtures.py @@ -282,6 +282,11 @@ def fixture_exec_ctx(make_exec_ctx_g): yield from make_exec_ctx_g(test_util.TEST_CONFIG_FILE, 'sys1') +@pytest.fixture +def testsys_exec_ctx(make_exec_ctx_g): + yield from make_exec_ctx_g(test_util.TEST_CONFIG_FILE, 'testsys') + + @pytest.fixture def ctx_sys(fixture_exec_ctx): yield rt.runtime().system @@ -319,9 +324,26 @@ def _fixture_wrapper(**kwargs): yield _fixture_wrapper +@pytest.fixture +def simple_test(): + def _make_test(vs, ve): + class _X(rfm.RegressionTest): + valid_systems = vs + valid_prog_environs = ve + + return _X() + + return _make_test + + def test_fixture_registry_all(ctx_sys, simple_fixture): '''Test with all valid partition and environments available.''' + class MyTest(rfm.RegressionTest): + valid_systems = ['*'] + valid_prog_environs = ['*'] + + test = MyTest() reg = fixtures.FixtureRegistry() # Get all part and environs @@ -332,8 +354,7 @@ def test_fixture_registry_all(ctx_sys, simple_fixture): def register(s, **kwargs): registered_fixt.update( - reg.add(simple_fixture(scope=s, **kwargs), - 0, 'base', all_part, set(all_env)) + reg.add(simple_fixture(scope=s, **kwargs), 0, test) ) register('test') @@ -358,40 +379,37 @@ class Foo: assert simple_fixture().cls in reg -def test_fixture_registry_edge_cases(ctx_sys, simple_fixture): +def test_fixture_registry_edge_cases(ctx_sys, simple_fixture, simple_test): '''Test edge cases.''' reg = fixtures.FixtureRegistry() registered_fixt = set() - def register(p, e, **kwargs): + def register(test, **kwargs): registered_fixt.update( - reg.add(simple_fixture(**kwargs), - 0, 'b', p, e) + reg.add(simple_fixture(**kwargs), 0, test) ) # Invalid partitions - NO-OP - register(['wrong_partition'], ['e1', 'e2']) + register(simple_test(['wrong_partition'], ['e1', 'e2'])) assert len(registered_fixt) == 0 # Valid partition but wrong environment - NO-OP (except test scope) - partitions = [p.fullname for p in ctx_sys.partitions] - register([partitions[0]], ['wrong_environment'], scope='session') + register(simple_test(['sys1:p0'], ['wrong_environment']), scope='session') assert len(registered_fixt) == 0 - register([partitions[0]], ['wrong_environment'], scope='partition') + register(simple_test(['sys1:p0'], [ + 'wrong_environment']), scope='partition') assert len(registered_fixt) == 0 - register([partitions[0]], ['wrong_environment'], scope='environment') + register( + simple_test(['sys1:p0'], ['wrong_environment']), scope='environment' + ) assert len(registered_fixt) == 0 - register([partitions[0]], ['wrong_environment'], scope='test') + register(simple_test(['sys1:p0'], ['wrong_environment']), scope='test') assert len(registered_fixt) == 1 registered_fixt.pop() # Environ 'e2' is not supported in 'sys1:p0', but is in 'sys1:p1' - register(['sys1:p0', 'sys1:p1'], ['e2'], scope='session') - assert 'e2' not in {env.name for p in ctx_sys.partitions - if p.fullname == 'sys1:p0' for env in p.environs} - assert 'e2' in {env.name for p in ctx_sys.partitions - if p.fullname == 'sys1:p1' for env in p.environs} + register(simple_test(['sys1:p0', 'sys1:p1'], ['e2']), scope='session') assert len(registered_fixt) == 1 # 'sys1:p0' is skipped on this fixture because env 'e2' is not supported @@ -400,14 +418,14 @@ def register(p, e, **kwargs): assert last_fixture.environments == ['e2'] # Similar behavior with the partition scope - register(['sys1:p0', 'sys1:p1'], ['e2'], scope='partition') + register(simple_test(['sys1:p0', 'sys1:p1'], ['e2']), scope='partition') assert len(registered_fixt) == 1 last_fixture = reg[simple_fixture().cls][registered_fixt.pop()] assert last_fixture.partitions == ['sys1:p1'] assert last_fixture.environments == ['e2'] # And also similar behavior with the environment scope - register(['sys1:p0', 'sys1:p1'], ['e2'], scope='environment') + register(simple_test(['sys1:p0', 'sys1:p1'], ['e2']), scope='environment') assert len(registered_fixt) == 1 last_fixture = reg[simple_fixture().cls][registered_fixt.pop()] assert last_fixture.partitions == ['sys1:p1'] @@ -415,26 +433,23 @@ def register(p, e, **kwargs): # However, with the test scope partitions and environments get copied # without any filtering. - register(['sys1:p0', 'sys1:p1'], ['e2'], scope='test') + register(simple_test(['sys1:p0', 'sys1:p1'], ['e2']), scope='test') assert len(registered_fixt) == 1 last_fixture = reg[simple_fixture().cls][registered_fixt.pop()] assert last_fixture.partitions == ['sys1:p0', 'sys1:p1'] assert last_fixture.environments == ['e2'] -def test_fixture_registry_variables(ctx_part_env, simple_fixture): +def test_fixture_registry_variables(ctx_sys, simple_fixture, simple_test): '''Test that the order of the variables does not matter.''' reg = fixtures.FixtureRegistry() - - # Get one valid part+env combination - part, env = ctx_part_env() registered_fixt = set() def register(**kwargs): registered_fixt.update( - reg.add(simple_fixture(**kwargs), - 0, 'b', [part], [env]) + reg.add(simple_fixture(**kwargs), 0, + simple_test(['sys1:p0'], ['e0'])) ) register(variables={'a': 1, 'b': 2}) @@ -451,24 +466,22 @@ def register(**kwargs): # Test also the format of the internal fixture tuple fixt_data = list(reg[simple_fixture().cls].values())[0] assert fixt_data.variant_num == 0 - assert fixt_data.environments == [env] - assert fixt_data.partitions == [part] + assert fixt_data.environments == ['e0'] + assert fixt_data.partitions == ['sys1:p0'] assert all(v in fixt_data.variables for v in ('b', 'a')) -def test_fixture_registry_variants(ctx_part_env, param_fixture): +def test_fixture_registry_variants(ctx_sys, param_fixture, simple_test): '''Test different fixture variants are registered separately.''' reg = fixtures.FixtureRegistry() - - # Get one valid part+env combination - part, env = ctx_part_env() registered_fixt = set() def register(scope='test', variant=0): + # We use a single valid part/env combination registered_fixt.update( reg.add(param_fixture(scope=scope), variant, - 'b', [part], [env]) + simple_test(['sys1:p0'], ['e0'])) ) register(scope='test', variant=0) @@ -489,60 +502,22 @@ def register(scope='test', variant=0): assert len(registered_fixt) == 8 -def test_fixture_registry_base_arg(ctx_part_env, simple_fixture): - '''The base argument argument only has an effect with test scope.''' - - reg = fixtures.FixtureRegistry() - - # Get one valid part+env combination - part, env = ctx_part_env() - registered_fixt = set() - - def register(scope, base): - registered_fixt.update( - reg.add(simple_fixture(scope=scope), 0, - base, [part], [env]) - ) - - # For a test scope, the base name is used for the fixture name mangling. - # So changing this base arg, leads to a new fixture being registered. - register(scope='test', base='b1') - assert len(registered_fixt) == 1 - register(scope='test', base='b2') - assert len(registered_fixt) == 2 - - # The base argument is not used with any of the other scopes. - register(scope='environment', base='b1') - assert len(registered_fixt) == 3 - register(scope='environment', base='b2') - assert len(registered_fixt) == 3 - register(scope='partition', base='b1') - assert len(registered_fixt) == 4 - register(scope='partition', base='b2') - assert len(registered_fixt) == 4 - register(scope='session', base='b3') - assert len(registered_fixt) == 5 - register(scope='session', base='b3') - assert len(registered_fixt) == 5 - - -def test_overlapping_registries(ctx_part_env, simple_fixture, param_fixture): +def test_overlapping_registries(ctx_sys, simple_test, + simple_fixture, param_fixture): '''Test instantiate_all(), update() and difference() registry methods.''' - # Get one valid part+env combination - part, env = ctx_part_env() - # Build base registry with some fixtures reg = fixtures.FixtureRegistry() - reg.add(simple_fixture(), 0, 'b', [part], [env]) + reg.add(simple_fixture(), 0, simple_test(['sys1:p0'], ['e0'])) for i in param_fixture().variants: - reg.add(param_fixture(), i, 'b', [part], [env]) + reg.add(param_fixture(), i, simple_test(['sys1:p0'], ['e0'])) # Build overlapping registry other = fixtures.FixtureRegistry() - other.add(simple_fixture(variables={'v': 2}), 0, 'b', [part], [env]) + other.add(simple_fixture(variables={'v': 2}), 0, + simple_test(['sys1:p0'], ['e0'])) for i in param_fixture().variants: - other.add(param_fixture(), i, 'b', [part], [env]) + other.add(param_fixture(), i, simple_test(['sys1:p0'], ['e0'])) assert len(reg.instantiate_all()) == len(param_fixture().variants) + 1 assert len(other.instantiate_all()) == len(param_fixture().variants) + 1 @@ -557,8 +532,8 @@ def test_overlapping_registries(ctx_part_env, simple_fixture, param_fixture): assert len(inst) == 1 assert inst[0].v == 2 assert inst[0].name == list(diff_reg[simple_fixture().cls].keys())[0] - assert inst[0].valid_systems == [part] - assert inst[0].valid_prog_environs == [env] + assert inst[0].valid_systems == ['sys1:p0'] + assert inst[0].valid_prog_environs == ['e0'] # Test the difference method in the opposite direction diff_reg = reg.difference(other) @@ -581,26 +556,23 @@ class Foo: reg.difference(Foo()) -def test_bad_fixture_inst(ctx_part_env): +def test_bad_fixture_inst(ctx_sys, simple_test): '''Test that instantiate_all does not raise an exception.''' - # Get one valid part+env combination - part, env = ctx_part_env() - class Foo(rfm.RegressionTest): def __init__(self): raise Exception('raise exception during instantiation') reg = fixtures.FixtureRegistry() - reg.add(fixtures.TestFixture(Foo), 0, 'b', [part], [env]) + reg.add(fixtures.TestFixture(Foo), 0, simple_test(['sys1:p0'], ['e0'])) reg.instantiate_all() -def test_expand_part_env(fixture_exec_ctx, simple_fixture): +def test_expand_part_env(testsys_exec_ctx, simple_fixture): '''Test expansion of partitions and environments.''' class MyTest(rfm.RegressionTest): - foo = simple_fixture(scope='test') + foo = simple_fixture(scope='environment') # The fixture registry is not injected when no variant_num is provided assert not hasattr(MyTest(), '_rfm_fixture_registry') @@ -609,7 +581,7 @@ class MyTest(rfm.RegressionTest): with pytest.raises(ReframeSyntaxError, match="'valid_systems'"): MyTest(variant_num=0) - MyTest.valid_systems = ['sys1'] + MyTest.valid_systems = ['testsys'] with pytest.raises(ReframeSyntaxError, match="'valid_prog_environs'"): MyTest(variant_num=0) @@ -625,15 +597,42 @@ def get_fixt_data(inst): )[simple_fixture().cls].values() )[0] + def _assert_fixture_partitions(expected): + reg = MyTest(variant_num=0)._rfm_fixture_registry + partitions = [] + for d in reg[simple_fixture().cls].values(): + partitions += d.partitions + + assert set(partitions) == set(expected) + + def _assert_fixture_environs(expected): + reg = MyTest(variant_num=0)._rfm_fixture_registry + environs = [] + for d in reg[simple_fixture().cls].values(): + environs += d.environments + + assert set(environs) == set(expected) + MyTest.valid_prog_environs = ['*'] - d = get_fixt_data(MyTest(variant_num=0)) - assert all(env in d.environments for env in ('e0', 'e1', 'e2', 'e3')) - assert all(part in d.partitions for part in ('sys1:p0', 'sys1:p1')) + _assert_fixture_partitions(['testsys:login', 'testsys:gpu']) + _assert_fixture_environs(['builtin', 'PrgEnv-cray', 'PrgEnv-gnu']) # Repeat now using * for the valid_systems MyTest.valid_systems = ['*'] - d = get_fixt_data(MyTest(variant_num=0)) - assert all(part in d.partitions for part in ('sys1:p0', 'sys1:p1')) + _assert_fixture_partitions(['testsys:login', 'testsys:gpu']) + + # Test the extended syntax of valid_systems and valid_prog_environs + MyTest.valid_systems = ['+cuda'] + _assert_fixture_partitions(['testsys:gpu']) + _assert_fixture_environs(['builtin', 'PrgEnv-gnu']) + + MyTest.valid_prog_environs = ['%bar=y'] + _assert_fixture_environs(['PrgEnv-gnu']) + + MyTest.valid_systems = ['testsys'] + MyTest.valid_prog_environs = ['+cxx14'] + _assert_fixture_partitions(['testsys:login']) + _assert_fixture_environs(['PrgEnv-cray', 'PrgEnv-gnu']) def test_fixture_injection(fixture_exec_ctx, simple_fixture, param_fixture): diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 7ff1103089..251fb0dae4 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -65,7 +65,7 @@ def generic_system(make_exec_ctx_g): @pytest.fixture -def testsys_system(make_exec_ctx_g): +def testsys_exec_ctx(make_exec_ctx_g): yield from make_exec_ctx_g(test_util.TEST_CONFIG_FILE, 'testsys') @@ -338,60 +338,374 @@ class MyTest(pinnedtest): assert pinned._prefix == expected_prefix -def test_supports_system(hellotest, testsys_system): +def test_valid_systems_syntax(hellotest): hellotest.valid_systems = ['*'] - assert hellotest.supports_system('gpu') - assert hellotest.supports_system('login') - assert hellotest.supports_system('testsys:gpu') - assert hellotest.supports_system('testsys:login') - hellotest.valid_systems = ['*:*'] - assert hellotest.supports_system('gpu') - assert hellotest.supports_system('login') - assert hellotest.supports_system('testsys:gpu') - assert hellotest.supports_system('testsys:login') - - hellotest.valid_systems = ['testsys'] - assert hellotest.supports_system('gpu') - assert hellotest.supports_system('login') - assert hellotest.supports_system('testsys:gpu') - assert hellotest.supports_system('testsys:login') - - hellotest.valid_systems = ['testsys:gpu'] - assert hellotest.supports_system('gpu') - assert not hellotest.supports_system('login') - assert hellotest.supports_system('testsys:gpu') - assert not hellotest.supports_system('testsys:login') - - hellotest.valid_systems = ['testsys:login'] - assert not hellotest.supports_system('gpu') - assert hellotest.supports_system('login') - assert not hellotest.supports_system('testsys:gpu') - assert hellotest.supports_system('testsys:login') - - hellotest.valid_systems = ['foo'] - assert not hellotest.supports_system('gpu') - assert not hellotest.supports_system('login') - assert not hellotest.supports_system('testsys:gpu') - assert not hellotest.supports_system('testsys:login') - - hellotest.valid_systems = ['*:gpu'] - assert hellotest.supports_system('testsys:gpu') - assert hellotest.supports_system('foo:gpu') - assert not hellotest.supports_system('testsys:cpu') - assert not hellotest.supports_system('testsys:login') - - hellotest.valid_systems = ['testsys:*'] - assert hellotest.supports_system('testsys:login') - assert hellotest.supports_system('gpu') - assert not hellotest.supports_system('foo:gpu') - - -def test_supports_environ(hellotest, generic_system): + hellotest.valid_systems = ['sys:*'] + hellotest.valid_systems = ['*:part'] + hellotest.valid_systems = ['sys'] + hellotest.valid_systems = ['sys:part'] + hellotest.valid_systems = ['sys-0'] + hellotest.valid_systems = ['sys:part-0'] + hellotest.valid_systems = ['+x0'] + hellotest.valid_systems = ['-y0'] + hellotest.valid_systems = ['%z0=w0'] + hellotest.valid_systems = ['+x0 -y0 %z0=w0'] + hellotest.valid_systems = ['-y0 +x0 %z0=w0'] + hellotest.valid_systems = ['%z0=w0 +x0 -y0'] + + with pytest.raises(TypeError): + hellotest.valid_systems = [''] + + with pytest.raises(TypeError): + hellotest.valid_systems = [' sys:part'] + + with pytest.raises(TypeError): + hellotest.valid_systems = [' sys:part '] + + with pytest.raises(TypeError): + hellotest.valid_systems = [':'] + + with pytest.raises(TypeError): + hellotest.valid_systems = [':foo'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['foo:'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['+'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['-'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['%'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['%foo'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['%foo='] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['+x0 -y0 %z0'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['+x0 - %z0=w0'] + + with pytest.raises(TypeError): + hellotest.valid_systems = ['%'] + + for sym in '!@#$^&()=<>': + with pytest.raises(TypeError): + hellotest.valid_systems = [f'{sym}foo'] + + for sym in '!@#$%^&*()+=<>': + with pytest.raises(TypeError): + hellotest.valid_systems = [f'foo{sym}'] + + +def test_valid_prog_environs_syntax(hellotest): hellotest.valid_prog_environs = ['*'] - assert hellotest.supports_environ('foo1') - assert hellotest.supports_environ('foo-env') - assert hellotest.supports_environ('*') + hellotest.valid_prog_environs = ['env'] + hellotest.valid_prog_environs = ['env-0'] + hellotest.valid_prog_environs = ['env.0'] + hellotest.valid_prog_environs = ['+x0'] + hellotest.valid_prog_environs = ['-y0'] + hellotest.valid_prog_environs = ['%z0=w0'] + hellotest.valid_prog_environs = ['+x0 -y0 %z0=w0'] + hellotest.valid_prog_environs = ['-y0 +x0 %z0=w0'] + hellotest.valid_prog_environs = ['%z0=w0 +x0 -y0'] + hellotest.valid_prog_environs = ['+foo.bar'] + hellotest.valid_prog_environs = ['%foo.bar=a$xx'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = [''] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = [' env0'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['env0 '] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = [':'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = [':foo'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['foo:'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['+'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['-'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['%'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['%foo'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['%foo='] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['+x0 -y0 %z0'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['+x0 - %z0=w0'] + + with pytest.raises(TypeError): + hellotest.valid_prog_environs = ['%'] + + for sym in '!@#$^&()=<>:': + with pytest.raises(TypeError): + hellotest.valid_prog_environs = [f'{sym}foo'] + + for sym in '!@#$%^&*()+=<>:': + with pytest.raises(TypeError): + hellotest.valid_prog_environs = [f'foo{sym}'] + + +def test_supports_sysenv(testsys_exec_ctx): + def _named_comb(valid_sysenv): + ret = {} + for part, environs in valid_sysenv.items(): + ret[part.fullname] = [env.name for env in environs] + + return ret + + def _assert_supported(valid_systems, valid_prog_environs, + expected, **kwargs): + valid_comb = _named_comb( + rt.valid_sysenv_comb(valid_systems, valid_prog_environs, **kwargs) + ) + assert expected == valid_comb + + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + _assert_supported( + valid_systems=['*:*'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + _assert_supported( + valid_systems=['testsys'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + _assert_supported( + valid_systems=['testsys:*'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + _assert_supported( + valid_systems=['testsys:gpu'], + valid_prog_environs=['*'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + _assert_supported( + valid_systems=['testsys:login'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + } + ) + _assert_supported( + valid_systems=['foo'], + valid_prog_environs=['*'], + expected={} + ) + _assert_supported( + valid_systems=['*:gpu'], + valid_prog_environs=['*'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + + # Check feature support + _assert_supported( + valid_systems=['+cuda'], + valid_prog_environs=['*'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + + # Check AND in features and extras + _assert_supported( + valid_systems=['+cuda +mpi %gpu_arch=v100'], + valid_prog_environs=['*'], + expected={} + ) + _assert_supported( + valid_systems=['+cuda -mpi'], + valid_prog_environs=['*'], + expected={} + ) + + # Check OR in features ad extras + _assert_supported( + valid_systems=['+cuda +mpi', '%gpu_arch=v100'], + valid_prog_environs=['*'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + + # Check that resources are taken into account + _assert_supported( + valid_systems=['+gpu +datawarp'], + valid_prog_environs=['*'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + + # Check negation + _assert_supported( + valid_systems=['-mpi -gpu'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'] + } + ) + _assert_supported( + valid_systems=['-mpi -foo'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'] + } + ) + _assert_supported( + valid_systems=['+gpu -datawarp'], + valid_prog_environs=['*'], + expected={} + ) + + # Test environment scoping + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['PrgEnv-cray'], + expected={ + 'testsys:gpu': [], + 'testsys:login': ['PrgEnv-cray'] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['+cxx14'], + expected={ + 'testsys:gpu': [], + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['+cxx14 -cxx14'], + expected={ + 'testsys:gpu': [], + 'testsys:login': [] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['+cxx14', '-cxx14'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'], + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['%bar=x'], + expected={ + 'testsys:gpu': [], + 'testsys:login': ['PrgEnv-gnu'] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['%foo=2'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu'], + 'testsys:login': [] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['%foo=bar'], + expected={ + 'testsys:gpu': [], + 'testsys:login': [] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['-cxx14'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'], + 'testsys:login': [] + } + ) + + # Check valid_systems / valid_prog_environs combinations + _assert_supported( + valid_systems=['testsys:login'], + valid_prog_environs=['-cxx14'], + expected={ + 'testsys:login': [] + } + ) + _assert_supported( + valid_systems=['+cross_compile'], + valid_prog_environs=['-cxx14'], + expected={ + 'testsys:login': [] + } + ) + + # Test skipping validity checks + _assert_supported( + valid_systems=['foo'], + valid_prog_environs=['*'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + }, + check_systems=False + ) + _assert_supported( + valid_systems=['foo'], + valid_prog_environs=['xxx'], + expected={ + 'testsys:login': ['PrgEnv-cray', 'PrgEnv-gnu'], + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + }, + check_systems=False, + check_environs=False + ) def test_sourcesdir_none(local_exec_ctx): @@ -508,7 +822,7 @@ class MyTest(rfm.CompileOnlyRegressionTest): test.compile_wait() -def test_extra_resources(HelloTest, testsys_system): +def test_extra_resources(HelloTest, testsys_exec_ctx): @test_util.custom_prefix('unittests/resources/checks') class MyTest(HelloTest): local = True @@ -1016,7 +1330,7 @@ def _run_sanity(test, *exec_ctx, skip_perf=False): @pytest.fixture -def dummy_gpu_exec_ctx(testsys_system): +def dummy_gpu_exec_ctx(testsys_exec_ctx): partition = test_util.partition_by_name('gpu') environ = test_util.environment_by_name('builtin', partition) yield partition, environ @@ -1036,7 +1350,7 @@ def sanity_file(tmp_path): # should not change to the `@performance_function` syntax` @pytest.fixture -def dummytest(testsys_system, perf_file, sanity_file): +def dummytest(testsys_exec_ctx, perf_file, sanity_file): class MyTest(rfm.RunOnlyRegressionTest): def __init__(self): self.perf_file = perf_file @@ -1264,7 +1578,7 @@ def extract_perf(patt, tag): @pytest.fixture -def perftest(testsys_system, perf_file, sanity_file): +def perftest(testsys_exec_ctx, perf_file, sanity_file): class MyTest(rfm.RunOnlyRegressionTest): sourcesdir = None diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 5c31cd5cbf..717c7bf1d4 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -63,8 +63,8 @@ def timestamps(self): @pytest.fixture def make_loader(): - def _make_loader(check_search_path): - return RegressionCheckLoader(check_search_path) + def _make_loader(check_search_path, *args, **kwargs): + return RegressionCheckLoader(check_search_path, *args, **kwargs) return _make_loader @@ -115,9 +115,11 @@ def _make_runner(*args, **kwargs): def make_cases(make_loader): def _make_cases(checks=None, sort=False, *args, **kwargs): if checks is None: - checks = make_loader(['unittests/resources/checks']).load_all() + checks = make_loader( + ['unittests/resources/checks'], *args, **kwargs + ).load_all(force=True) - cases = executors.generate_testcases(checks, *args, **kwargs) + cases = executors.generate_testcases(checks) if sort: depgraph, _ = dependencies.build_deps(cases) dependencies.validate_deps(depgraph) @@ -304,7 +306,7 @@ def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx): def test_runall_skip_prgenv_check(make_runner, make_cases, common_exec_ctx): runner = make_runner() - runner.runall(make_cases(skip_environ_check=True)) + runner.runall(make_cases(skip_prgenv_check=True)) stats = runner.stats assert 10 == stats.num_cases() assert_runall(runner)