[feat] Build configuration incrementally by allowing to combine multiple configuration files

reframe/core/config.py

@@ -81,9 +81,9 @@ def _hostname(use_fqdn, use_xthostname):
 
 
 class _SiteConfig:
-    def __init__(self, site_config, filename):
-        self._site_config = copy.deepcopy(site_config)
-        self._filename = filename
+    def __init__(self):
+        self._site_config = None
+        self._filenames = []
         self._subconfigs = {}
         self._local_system = None
         self._sticky_options = {}
@@ -106,6 +106,23 @@ def __init__(self, site_config, filename):
                     f'invalid configuration schema: {schema_filename!r}'
                 ) from e
 
+    def add_config(self, config, filename):
+        self._filenames.append(filename)
+        nc = copy.deepcopy(config)
+        if self._site_config is None:
+            self._site_config = nc
+            return self
+
+        for i in ['systems', 'environments', 'scheduler', 'logging',
+                  'modes', 'general']:
+            if i in nc:
+                if i in self._site_config:
+                    self._site_config[i] += nc[i]
+                else:
+                    self._site_config[i] = nc[i]
+
+        return self
+
     def _pick_config(self):
         if self._local_system:
             return self._subconfigs[self._local_system]
@@ -114,7 +131,7 @@ def _pick_config(self):
 
     def __repr__(self):
         return (f'{type(self).__name__}(site_config={self._site_config!r}, '
-                f'filename={self._filename!r})')
+                f'filenames={self._filenames!r})')
 
     def __str__(self):
         return json.dumps(self._pick_config(), indent=2)
@@ -242,25 +259,14 @@ def get(self, option, default=None):
         return value
 
     @property
-    def filename(self):
-        return self._filename
+    def filenames(self):
+        return self._filenames
 
     @property
     def subconfig_system(self):
         return self._local_system
 
-    @classmethod
-    def create(cls, filename):
-        _, ext = os.path.splitext(filename)
-        if ext == '.py':
-            return cls._create_from_python(filename)
-        elif ext == '.json':
-            return cls._create_from_json(filename)
-        else:
-            raise ConfigError(f"unknown configuration file type: '{filename}'")
-
-    @classmethod
-    def _create_from_python(cls, filename):
+    def add_python_config(self, filename):
         try:
             mod = util.import_module_from_file(filename)
         except ImportError as e:
@@ -284,10 +290,9 @@ def _create_from_python(cls, filename):
                 f"not a valid Python configuration file: '{filename}'"
             )
 
-        return _SiteConfig(mod.site_configuration, filename)
+        return self.add_config(mod.site_configuration, filename)
 
-    @classmethod
-    def _create_from_json(cls, filename):
+    def add_json_config(self, filename):
         with open(filename) as fp:
             try:
                 config = json.loads(fp.read())
@@ -296,7 +301,7 @@ def _create_from_json(cls, filename):
                     f"invalid JSON syntax in configuration file '{filename}'"
                 ) from e
 
-        return _SiteConfig(config, filename)
+        self.add_config(config, filename)
 
     def _detect_system(self):
         getlogger().debug(
@@ -325,27 +330,8 @@ def validate(self):
         try:
             jsonschema.validate(site_config, self._schema)
         except jsonschema.ValidationError as e:
-            raise ConfigError(f"could not validate configuration file: "
-                              f"'{self._filename}'") from e
-
-        # Make sure that system and partition names are unique
-        system_names = set()
-        for system in self._site_config['systems']:
-            sysname = system['name']
-            if sysname in system_names:
-                raise ConfigError(f"system '{sysname}' already defined")
-
-            system_names.add(sysname)
-            partition_names = set()
-            for part in system['partitions']:
-                partname = part['name']
-                if partname in partition_names:
-                    raise ConfigError(
-                        f"partition '{partname}' already defined "
-                        f"for system '{sysname}'"
-                    )
-
-                partition_names.add(partname)
+            raise ConfigError(f"could not validate configuration files: "
+                              f"'{self._filenames}'") from e
 
     def select_subconfig(self, system_fullname=None,
                          ignore_resolve_errors=False):
@@ -457,6 +443,7 @@ def select_subconfig(self, system_fullname=None,
 
 
 def convert_old_config(filename, newfilename=None):
+    #TODO fix this to get many files
     old_config = util.import_module_from_file(filename).settings
     converted = {
         'systems': [],
@@ -632,37 +619,21 @@ def handler_list(handler_config, basedir=None):
     return fp.name
 
 
-def _find_config_file():
-    # The order of elements is important, since it defines the priority
-    homedir = os.getenv('HOME')
-    prefixes = [os.path.join(homedir, '.reframe')] if homedir else []
-    prefixes += [
-        reframe.INSTALL_PREFIX,
-        '/etc/reframe.d'
-    ]
-    valid_exts = ['py', 'json']
-    getlogger().debug('Looking for a suitable configuration file')
-    for d in prefixes:
-        if not d:
-            continue
-
-        for ext in valid_exts:
-            filename = os.path.join(d, f'settings.{ext}')
-            getlogger().debug(f'Trying {filename!r}')
-            if os.path.exists(filename):
-                return filename
-
-    return None
-
-
-def load_config(filename=None):
-    if filename is None:
-        filename = _find_config_file()
-        if filename is None:
-            # Return the generic configuration
-            getlogger().debug('No configuration found; '
-                              'falling back to a generic one')
-            return _SiteConfig(settings.site_configuration, '<builtin>')
-
-    getlogger().debug(f'Loading configuration file: {filename!r}')
-    return _SiteConfig.create(filename)
+
+def load_config(filenames=None):
+    ret = _SiteConfig()
+    getlogger().debug('Loading the generic configuration')
+    ret.add_config(settings.site_configuration, '<builtin>')
+    if filenames:
+        getlogger().debug(f'Loading configuration files: {filenames!r}')
+        for filename in filenames:
+            _, ext = os.path.splitext(filename)
+            if ext == '.py':
+                ret.add_python_config(filename)
+            elif ext == '.json':
+                ret.add_json_config(filename)
+            else:
+                raise ConfigError(f"unknown configuration file type: "
+                                    f"'{filename}'")
+
+    return ret

reframe/core/runtime.py

@@ -420,7 +420,7 @@ def __init__(self, config_file, sysname=None, options=None):
         if config_file is None:
             _runtime_context = None
         else:
-            site_config = config.load_config(config_file)
+            site_config = config.load_config([config_file])
             site_config.select_subconfig(sysname, ignore_resolve_errors=True)
             for opt, value in options.items():
                 site_config.add_sticky_option(opt, value)

reframe/frontend/ci.py

@@ -22,8 +22,8 @@ def _emit_gitlab_pipeline(testcases, child_pipeline_opts):
     verbosity = 'v' * config.get('general/0/verbose')
 
     def rfm_command(testcase):
-        if config.filename != '<builtin>':
-            config_opt = f'-C {config.filename}'
+        if config.filenames != ['<builtin>']:
+            config_opt = f'-C {config.filenames}'
         else:
             config_opt = ''
 

reframe/frontend/cli.py

@@ -496,8 +496,8 @@ def main():
 
     # Miscellaneous options
     misc_options.add_argument(
-        '-C', '--config-file', action='store',
-        dest='config_file', metavar='FILE',
+        '-C', '--config-file', action='append', metavar='FILE',
+        dest='config_file',
         help='Set configuration file',
         envvar='RFM_CONFIG_FILE'
     )
@@ -688,7 +688,7 @@ def restrict_logging():
     # to print pretty messages; logging will be reconfigured by user's
     # configuration later
     site_config = config.load_config(
-        os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py')
+        [os.path.join(reframe.INSTALL_PREFIX, 'reframe/core/settings.py')]
     )
     site_config.select_subconfig('generic')
     options.update_config(site_config)
@@ -727,15 +727,18 @@ def restrict_logging():
     try:
         try:
             printer.debug('Loading user configuration')
-            site_config = config.load_config(options.config_file)
+            if options.config_file is None:
+                site_config = config.load_config(None)
+            else:
+                site_config = config.load_config(options.config_file)
         except warnings.ReframeDeprecationWarning as e:
             printer.warning(e)
             converted = config.convert_old_config(options.config_file)
             printer.warning(
                 f"configuration file has been converted "
                 f"to the new syntax here: '{converted}'"
             )
-            site_config = config.load_config(converted)
+            site_config = config.load_config([converted])
 
         site_config.validate()
         site_config.set_autodetect_meth(
@@ -926,7 +929,7 @@ def print_infoline(param, value):
 
     session_info = {
         'cmdline': ' '.join(sys.argv),
-        'config_file': rt.site_config.filename,
+        'config_files': rt.site_config.filenames,
         'data_version': runreport.DATA_VERSION,
         'hostname': socket.gethostname(),
         'prefix_output': rt.output_prefix,
@@ -945,7 +948,7 @@ def print_infoline(param, value):
         f"{session_info['user'] or '<unknown>'}@{session_info['hostname']}"
     )
     print_infoline('working directory', repr(session_info['workdir']))
-    print_infoline('settings file', f"{session_info['config_file']!r}")
+    print_infoline('settings files', f"{session_info['config_files']!r}")
     print_infoline('check search path',
                    f"{'(R) ' if loader.recurse else ''}"
                    f"{':'.join(loader.load_path)!r}")

reframe/frontend/runreport.py

@@ -18,7 +18,7 @@
 # The schema data version
 # Major version bumps are expected to break the validation of previous schemas
 
-DATA_VERSION = '2.2'
+DATA_VERSION = '3.0'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
 
 

reframe/schemas/runreport.json

@@ -110,7 +110,10 @@
             "type": "object",
             "properties": {
                 "cmdline": {"type": "string"},
-                "config_file": {"type": ["string", "null"]},
+                "config_files": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
                 "data_version": {"type": "string"},
                 "hostname": {"type": "string"},
                 "num_cases": {"type": "number"},