Refactor templates v2

ssg/build_sce.py

@@ -166,8 +166,8 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output):
 
                 # While we don't _write_ it, we still need to parse SCE
                 # metadata from the templated content. Render it internally.
-                raw_sce_content = template_builder.get_lang_contents(
-                    rule_id, rule.title, rule.template, langs['sce-bash'])
+                raw_sce_content = template_builder.get_templatable_lang_contents(rule,
+                                                                                 langs['sce-bash'])
 
                 ext = '.sh'
                 filename = rule_id + ext

ssg/build_yaml.py

@@ -850,6 +850,11 @@ def load_policy_specific_content(self, rule_filename, env_yaml):
                 env_yaml, policy_specific_content_files)
         self.policy_specific_content = policy_specific_content
 
+    def get_template_context(self, env_yaml):
+        ctx = super(Rule, self).get_template_context(env_yaml)
+        if self.identifiers:
+            ctx["cce_identifiers"] = self.identifiers
+        return ctx
 
     def make_refs_and_identifiers_product_specific(self, product):
         product_suffix = "@{0}".format(product)

ssg/entities/common.py

@@ -363,6 +363,59 @@ def __init__(self):
     def is_templated(self):
         return isinstance(self.template, dict)
 
+    def get_template_context(self, env_yaml):
+        # TODO: The first two variables, 'rule_id' and 'rule_title' are expected by some
+        #       templates and macros even if they are not rendered in a rule context.
+        #       Better name for these variables are 'entity_id' and 'entity_title'.
+        return {
+            "rule_id": self.id_,
+            "rule_title": self.title,
+            "products": env_yaml["product"],
+        }
+
+    def get_template_name(self):
+        """
+        Given a template dictionary from a Rule instance, determine the name
+        of the template (from templates) this rule uses.
+        """
+        try:
+            template_name = self.template["name"]
+        except KeyError:
+            raise ValueError(
+                "Templatable {0} is missing template name under template key".format(self))
+        return template_name
+
+    def get_template_vars(self, env_yaml):
+        if "vars" not in self.template:
+            raise ValueError(
+                "Templatable {0} does not contain mandatory 'vars:' key under "
+                "'template:' key.".format(self))
+        template_vars = self.template["vars"]
+
+        # Add the rule ID which will be used in template preprocessors (template.py)
+        # as a unique sub-element for a variety of composite IDs.
+        # TODO: The name _rule_id is a legacy from the era when rule was the only
+        #       context for a template. Preprocessors implicitly depend on this name.
+        #       A better name is '_entity_id' (as in XCCDF Entity).
+        template_vars["_rule_id"] = self.id_
+
+        return make_items_product_specific(template_vars, env_yaml["product"])
+
+    def get_template_backend_langs(self):
+        """
+        Returns list of languages that should be generated from a template
+        configuration, controlled by backends.
+        """
+        from ..templates import LANGUAGES
+        if "backends" in self.template:
+            backends = self.template["backends"]
+            for lang in backends:
+                if lang not in LANGUAGES:
+                    raise RuntimeError("Templatable {0} wants to generate unknown language '{1}"
+                                       .format(self, lang))
+            return [lang for name, lang in LANGUAGES.items() if backends.get(name, "on") == "on"]
+        return LANGUAGES.values()
+
     def make_template_product_specific(self, product):
         if not self.is_templated():
             return

ssg/templates.py

@@ -12,22 +12,23 @@
 from ssg.build_cpe import ProductCPEs
 from collections import namedtuple
 
-templating_lang = namedtuple(
+
+TemplatingLang = namedtuple(
     "templating_language_attributes",
     ["name", "file_extension", "template_type", "lang_specific_dir"])
 
-template_type = ssg.utils.enum("remediation", "check")
-
-languages = {
-    "anaconda": templating_lang("anaconda", ".anaconda", template_type.remediation, "anaconda"),
-    "ansible": templating_lang("ansible", ".yml", template_type.remediation, "ansible"),
-    "bash": templating_lang("bash", ".sh", template_type.remediation, "bash"),
-    "blueprint": templating_lang("blueprint", ".toml", template_type.remediation, "blueprint"),
-    "ignition": templating_lang("ignition", ".yml", template_type.remediation, "ignition"),
-    "kubernetes": templating_lang("kubernetes", ".yml", template_type.remediation, "kubernetes"),
-    "oval": templating_lang("oval", ".xml", template_type.check, "oval"),
-    "puppet": templating_lang("puppet", ".pp", template_type.remediation, "puppet"),
-    "sce-bash": templating_lang("sce-bash", ".sh", template_type.remediation, "sce")
+TemplateType = ssg.utils.enum("REMEDIATION", "CHECK")
+
+LANGUAGES = {
+    "anaconda": TemplatingLang("anaconda", ".anaconda", TemplateType.REMEDIATION, "anaconda"),
+    "ansible": TemplatingLang("ansible", ".yml",        TemplateType.REMEDIATION, "ansible"),
+    "bash": TemplatingLang("bash", ".sh",               TemplateType.REMEDIATION, "bash"),
+    "blueprint": TemplatingLang("blueprint", ".toml",   TemplateType.REMEDIATION, "blueprint"),
+    "ignition": TemplatingLang("ignition", ".yml",      TemplateType.REMEDIATION, "ignition"),
+    "kubernetes": TemplatingLang("kubernetes", ".yml",  TemplateType.REMEDIATION, "kubernetes"),
+    "oval": TemplatingLang("oval", ".xml",              TemplateType.CHECK,       "oval"),
+    "puppet": TemplatingLang("puppet", ".pp",           TemplateType.REMEDIATION, "puppet"),
+    "sce-bash": TemplatingLang("sce-bash", ".sh",       TemplateType.REMEDIATION, "sce")
 }
 
 PREPROCESSING_FILE_NAME = "template.py"
@@ -57,12 +58,12 @@ def _load(self):
 
         template_yaml = ssg.yaml.open_raw(self.template_yaml_path)
         for supported_lang in template_yaml["supported_languages"]:
-            if supported_lang not in languages.keys():
+            if supported_lang not in LANGUAGES.keys():
                 raise ValueError(
                     "The template {0} declares to support the {1} language,"
                     "but this language is not supported by the content.".format(
                         self.name, supported_lang))
-            lang = languages[supported_lang]
+            lang = LANGUAGES[supported_lang]
             langfilename = lang.name + ".template"
             if not os.path.exists(os.path.join(self.template_path, langfilename)):
                 raise ValueError(
@@ -134,147 +135,86 @@ def _init_and_load_templates(self):
                 self.templates[item] = maybe_template
 
     def _init_lang_output_dirs(self):
-        for lang_name, lang in languages.items():
+        for lang_name, lang in LANGUAGES.items():
             lang_dir = lang.lang_specific_dir
-            if lang.template_type == template_type.check:
+            if lang.template_type == TemplateType.CHECK:
                 output_dir = self.checks_dir
             else:
                 output_dir = self.remediations_dir
             dir_ = os.path.join(output_dir, lang_dir)
             self.output_dirs[lang_name] = dir_
 
-    def get_template_backend_langs(self, template, rule_id):
-        """
-        Returns list of languages that should be generated from a template
-        configuration, controlled by backends.
+    def get_resolved_langs_to_generate(self, templatable):
         """
-        if "backends" in template:
-            backends = template["backends"]
-            for lang in backends:
-                if lang not in languages:
-                    raise RuntimeError("Rule {0} wants to generate unknown language '{1}"
-                                       "from a template.".format(rule_id, lang))
-            return [lang for name, lang in languages.items() if backends.get(name, "on") == "on"]
-        return languages.values()
-
-    def get_template_name(self, template, rule_id):
+        Given a specific Templatable instance, determine which languages are
+        generated by the combination of the template supported_languages AND
+        the Templatable's template configuration 'backends'.
         """
-        Given a template dictionary from a Rule instance, determine the name
-        of the template (from templates) this rule uses.
-        """
-        try:
-            template_name = template["name"]
-        except KeyError:
-            raise ValueError(
-                "Rule {0} is missing template name under template key".format(
-                    rule_id))
+        if not templatable.is_templated():
+            return []
+
+        rule_langs = set(templatable.get_template_backend_langs())
+        template_name = templatable.get_template_name()
         if template_name not in self.templates.keys():
             raise ValueError(
-                "Rule {0} uses template {1} which does not exist.".format(
-                    rule_id, template_name))
-        return template_name
-
-    def get_resolved_langs_to_generate(self, rule):
-        """
-        Given a specific Rule instance, determine which languages are
-        generated by the combination of the rule's template supported_languages AND
-        the rule's template configuration backends.
-        """
-        if rule.template is None:
-            return None
-
-        rule_langs = set(self.get_template_backend_langs(rule.template, rule.id_))
-        template_name = self.get_template_name(rule.template, rule.id_)
+                "Templatable {0} uses template {1} which does not exist."
+                .format(templatable, template_name))
         template_langs = set(self.templates[template_name].langs)
         return rule_langs.intersection(template_langs)
 
-    def process_product_vars(self, all_variables):
-        """
-        Given a dictionary with the format key[@<product>]=value, filter out
-        and only take keys that apply to this product (unqualified or qualified
-        to exactly this product). Returns a new dict.
-        """
-        processed = dict(filter(lambda item: '@' not in item[0], all_variables.items()))
-        suffix = '@' + self.env_yaml['product']
-        for variable in filter(lambda key: key.endswith(suffix), all_variables):
-            new_variable = variable[:-len(suffix)]
-            value = all_variables[variable]
-            processed[new_variable] = value
-
-        return processed
-
-    def render_lang_file(self, template_name, template_vars, lang, local_env_yaml):
+    def process_template_lang_file(self, template_name, template_vars, lang, local_env_yaml):
         """
-        Builds and returns templated content for a given rule for a given
-        language; does not write the output to disk.
+        Processes template for a given template name and language and returns rendered content.
         """
         if lang not in self.templates[template_name].langs:
-            return None
+            raise ValueError("Language {0} is not available for template {1}."
+                             .format(lang.name, template_name))
 
         template_file_name = lang.name + ".template"
         template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
         template_parameters = self.templates[template_name].preprocess(template_vars, lang.name)
         env_yaml = self.env_yaml.copy()
         env_yaml.update(local_env_yaml)
         jinja_dict = ssg.utils.merge_dicts(env_yaml, template_parameters)
-        try:
-            filled_template = ssg.jinja.process_file_with_macros(template_file_path, jinja_dict)
-        except Exception as e:
-            print("Error in template: %s (lang: %s)" % (template_name, lang.name))
-            raise e
+        return ssg.jinja.process_file_with_macros(template_file_path, jinja_dict)
 
-        return filled_template
-
-    def get_lang_contents(self, rule_id, rule_title, template, language, extra_env=None):
+    def get_templatable_lang_contents(self, templatable, language):
         """
-        For the specified template, build and return only the specified language
-        content.
+        For the specified Templatable, build and return only the specified language content.
         """
-        template_name = self.get_template_name(template, rule_id)
-        try:
-            template_vars = self.process_product_vars(template["vars"])
-        except KeyError:
-            raise ValueError(
-                "Rule {0} does not contain mandatory 'vars:' key under "
-                "'template:' key.".format(rule_id))
-        # Add the rule ID which will be reused in OVAL templates as OVAL
-        # definition ID so that the build system matches the generated
-        # check with the rule.
-        template_vars["_rule_id"] = rule_id
+        template_name = templatable.get_template_name()
+        template_vars = templatable.get_template_vars(self.env_yaml)
 
         # Checks and remediations are processed with a custom YAML dict
-        local_env_yaml = {"rule_id": rule_id, "rule_title": rule_title,
-                          "products": self.env_yaml["product"]}
-        if extra_env is not None:
-            local_env_yaml.update(extra_env)
-
-        return self.render_lang_file(template_name, template_vars, language, local_env_yaml)
-
-    def build_lang(self, rule_id, rule_title, template, lang, extra_env=None):
-        """
-        Builds templated content for a given rule for a given language.
-        Writes the output to the correct build directories.
-        """
-        filled_template = self.get_lang_contents(rule_id, rule_title, template, lang, extra_env)
+        local_env_yaml = templatable.get_template_context(self.env_yaml)
+        try:
+            return self.process_template_lang_file(template_name, template_vars, language, local_env_yaml)
+        except Exception as e:
+            raise RuntimeError("Unable to generate {0} template language for Templatable {1}: {2}"
+                               .format(language.name, templatable, e))
 
-        output_file_name = rule_id + lang.file_extension
+    def write_templatable_lang_contents(self, filled_template, lang, templatable):
+        output_file_name = templatable.id_ + lang.file_extension
         output_filepath = os.path.join(self.output_dirs[lang.name], output_file_name)
-
         with open(output_filepath, "w") as f:
             f.write(filled_template)
 
-    def build_rule(self, rule):
+    def build_templatable_lang(self, templatable, lang):
         """
-        Builds templated content for a given rule for selected languages,
+        Builds templated content of a given Templatable for a selected language,
         writing the output to the correct build directories.
         """
-        extra_env = {}
-        if rule.identifiers is not None:
-            extra_env["cce_identifiers"] = rule.identifiers
+        filled_template = self.get_templatable_lang_contents(templatable, lang)
+        self.write_templatable_lang_contents(filled_template, lang, templatable)
 
+    def build_rule(self, rule):
+        """
+        Builds templated content of a given Rule for all available languages,
+        writing the output to the correct build directories.
+        """
         for lang in self.get_resolved_langs_to_generate(rule):
             if lang.name != "sce-bash":
-                self.build_lang(rule.id_, rule.title, rule.template, lang, extra_env)
+                self.build_templatable_lang(rule, lang)
 
     def build_extra_ovals(self):
         declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
@@ -283,7 +223,12 @@ def build_extra_ovals(self):
             # Since OVAL definition ID in shorthand format is always the same
             # as rule ID, we can use it instead of the rule ID even if no rule
             # with that ID exists
-            self.build_lang(oval_def_id, oval_def_id, template, languages["oval"])
+            rule = ssg.build_yaml.Rule.get_instance_from_full_dict({
+                "id_": oval_def_id,
+                "title": oval_def_id,
+                "template": template,
+            })
+            self.build_templatable_lang(rule, LANGUAGES["oval"])
 
     def build_all_rules(self):
         for rule_file in sorted(os.listdir(self.resolved_rules_dir)):
@@ -293,13 +238,13 @@ def build_all_rules(self):
             except ssg.build_yaml.DocumentationNotComplete:
                 # Happens on non-debug build when a rule is "documentation-incomplete"
                 continue
-            if rule.template is not None:
+            if rule.is_templated():
                 self.build_rule(rule)
 
     def build(self):
         """
-        Builds all templated content for all languages, writing
-        the output to the correct build directories.
+        Builds all templated content for all languages,
+        writing the output to the correct build directories.
         """
         for dir_ in self.output_dirs.values():
             if not os.path.exists(dir_):

tests/unit/ssg-module/test_templates.py

@@ -27,6 +27,11 @@ def test_render_extra_ovals():
     declaration_path = os.path.join(builder.templates_dir, "extra_ovals.yml")
     declaration = ssg.yaml.open_raw(declaration_path)
     for oval_def_id, template in declaration.items():
-        oval_content = builder.get_lang_contents(oval_def_id, oval_def_id, template,
-                                                 ssg.templates.languages["oval"])
+        rule = ssg.build_yaml.Rule.get_instance_from_full_dict({
+            "id_": oval_def_id,
+            "title": oval_def_id,
+            "template": template,
+        })
+        oval_content = builder.get_templatable_lang_contents(rule,
+                                                             ssg.templates.LANGUAGES["oval"])
         assert "<title>%s</title>" % (oval_def_id,) in oval_content