Refactor templates v2

ssg/build_cpe.py

@@ -140,7 +140,6 @@ class CPEItem(XCCDFEntity):
 
     KEYS = dict(
         name=lambda: "",
-        title=lambda: "",
         check_id=lambda: "",
         bash_conditional=lambda: "",
         ansible_conditional=lambda: "",

ssg/build_sce.py

@@ -166,8 +166,8 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output):
 
                 # While we don't _write_ it, we still need to parse SCE
                 # metadata from the templated content. Render it internally.
-                raw_sce_content = template_builder.get_lang_for_rule(
-                    rule_id, rule.title, rule.template, 'sce-bash')
+                raw_sce_content = template_builder.get_templatable_lang_contents(rule,
+                                                                                 langs['sce-bash'])
 
                 ext = '.sh'
                 filename = rule_id + ext

ssg/build_yaml.py

@@ -44,51 +44,11 @@
 from .shims import unicode_func
 import ssg.build_stig
 
-from .entities.common import (
-    XCCDFEntity,
-    add_sub_element,
-)
+from .entities.common import add_sub_element, make_items_product_specific, \
+                             XCCDFEntity, Templatable
 from .entities.profile import Profile, ProfileWithInlinePolicies
 
 
-def add_sub_element(parent, tag, ns, data):
-    """
-    Creates a new child element under parent with tag tag, and sets
-    data as the content under the tag. In particular, data is a string
-    to be parsed as an XML tree, allowing sub-elements of children to be
-    added.
-
-    If data should not be parsed as an XML tree, either escape the contents
-    before passing into this function, or use ElementTree.SubElement().
-
-    Returns the newly created subelement of type tag.
-    """
-    namespaced_data = add_xhtml_namespace(data)
-    # This is used because our YAML data contain XML and XHTML elements
-    # ET.SubElement() escapes the < > characters by &lt; and &gt;
-    # and therefore it does not add child elements
-    # we need to do a hack instead
-    # TODO: Remove this function after we move to Markdown everywhere in SSG
-    ustr = unicode_func('<{0} xmlns="{3}" xmlns:xhtml="{2}">{1}</{0}>').format(
-        tag, namespaced_data, xhtml_namespace, ns)
-
-    try:
-        element = ET.fromstring(ustr.encode("utf-8"))
-    except Exception:
-        msg = ("Error adding subelement to an element '{0}' from string: '{1}'"
-               .format(parent.tag, ustr))
-        raise RuntimeError(msg)
-
-    # Apart from HTML and XML elements the rule descriptions and similar
-    # also contain <xccdf:sub> elements, where we need to add the prefix
-    # to create a full reference.
-    for x in element.findall(".//{%s}sub" % XCCDF12_NS):
-        x.set("idref", OSCAP_VALUE + x.get("idref"))
-        x.set("use", "legacy")
-    parent.append(element)
-    return element
-
-
 def reorder_according_to_ordering(unordered, ordering, regex=None):
     ordered = []
     if regex is None:
@@ -187,7 +147,6 @@ class Value(XCCDFEntity):
     """Represents XCCDF Value
     """
     KEYS = dict(
-        title=lambda: "",
         description=lambda: "",
         type=lambda: "",
         operator=lambda: "equals",
@@ -260,7 +219,6 @@ class Benchmark(XCCDFEntity):
     """Represents XCCDF Benchmark
     """
     KEYS = dict(
-        title=lambda: "",
         status=lambda: "",
         description=lambda: "",
         notice_id=lambda: "",
@@ -480,7 +438,6 @@ class Group(XCCDFEntity):
 
     KEYS = dict(
         prodtype=lambda: "all",
-        title=lambda: "",
         description=lambda: "",
         warnings=lambda: list(),
         requires=lambda: list(),
@@ -692,12 +649,11 @@ def filterfunc(rule):
     return filterfunc
 
 
-class Rule(XCCDFEntity):
+class Rule(XCCDFEntity, Templatable):
     """Represents XCCDF Rule
     """
     KEYS = dict(
         prodtype=lambda: "all",
-        title=lambda: "",
         description=lambda: "",
         rationale=lambda: "",
         severity=lambda: "",
@@ -718,12 +674,12 @@ class Rule(XCCDFEntity):
         platforms=lambda: set(),
         sce_metadata=lambda: dict(),
         inherited_platforms=lambda: set(),
-        template=lambda: None,
         cpe_platform_names=lambda: set(),
         inherited_cpe_platform_names=lambda: set(),
         bash_conditional=lambda: None,
         fixes=lambda: dict(),
-        ** XCCDFEntity.KEYS
+        ** XCCDFEntity.KEYS,
+        ** Templatable.KEYS
     )
 
     MANDATORY_KEYS = {
@@ -737,7 +693,6 @@ class Rule(XCCDFEntity):
     ID_LABEL = "rule_id"
 
     PRODUCT_REFERENCES = ("stigid", "cis",)
-    GLOBAL_REFERENCES = ("srg", "vmmsrg", "disa", "cis-csc",)
 
     def __init__(self, id_):
         super(Rule, self).__init__(id_)
@@ -895,21 +850,11 @@ def load_policy_specific_content(self, rule_filename, env_yaml):
                 env_yaml, policy_specific_content_files)
         self.policy_specific_content = policy_specific_content
 
-    def make_template_product_specific(self, product):
-        product_suffix = "@{0}".format(product)
-
-        if not self.template:
-            return
-
-        not_specific_vars = self.template.get("vars", dict())
-        specific_vars = self._make_items_product_specific(
-            not_specific_vars, product_suffix, True)
-        self.template["vars"] = specific_vars
-
-        not_specific_backends = self.template.get("backends", dict())
-        specific_backends = self._make_items_product_specific(
-            not_specific_backends, product_suffix, True)
-        self.template["backends"] = specific_backends
+    def get_template_context(self, env_yaml):
+        ctx = super(Rule, self).get_template_context(env_yaml)
+        if self.identifiers:
+            ctx["cce_identifiers"] = self.identifiers
+        return ctx
 
     def make_refs_and_identifiers_product_specific(self, product):
         product_suffix = "@{0}".format(product)
@@ -933,7 +878,7 @@ def make_refs_and_identifiers_product_specific(self, product):
         )
         for name, (dic, allow_overwrites) in to_set.items():
             try:
-                new_items = self._make_items_product_specific(
+                new_items = make_items_product_specific(
                     dic, product_suffix, allow_overwrites)
             except ValueError as exc:
                 msg = (
@@ -950,43 +895,6 @@ def make_refs_and_identifiers_product_specific(self, product):
 
         self._verify_stigid_format(product)
 
-    def _make_items_product_specific(self, items_dict, product_suffix, allow_overwrites=False):
-        new_items = dict()
-        for full_label, value in items_dict.items():
-            if "@" not in full_label and full_label not in new_items:
-                new_items[full_label] = value
-                continue
-
-            label = full_label.split("@")[0]
-
-            # this test should occur before matching product_suffix with the product qualifier
-            # present in the reference, so it catches problems even for products that are not
-            # being built at the moment
-            if label in Rule.GLOBAL_REFERENCES:
-                msg = (
-                    "You cannot use product-qualified for the '{item_u}' reference. "
-                    "Please remove the product-qualifier and merge values with the "
-                    "existing reference if there is any. Original line: {item_q}: {value_q}"
-                    .format(item_u=label, item_q=full_label, value_q=value)
-                )
-                raise ValueError(msg)
-
-            if not full_label.endswith(product_suffix):
-                continue
-
-            if label in items_dict and not allow_overwrites and value != items_dict[label]:
-                msg = (
-                    "There is a product-qualified '{item_q}' item, "
-                    "but also an unqualified '{item_u}' item "
-                    "and those two differ in value - "
-                    "'{value_q}' vs '{value_u}' respectively."
-                    .format(item_q=full_label, item_u=label,
-                            value_q=value, value_u=items_dict[label])
-                )
-                raise ValueError(msg)
-            new_items[label] = value
-        return new_items
-
     def validate_identifiers(self, yaml_file):
         if self.identifiers is None:
             raise ValueError("Empty identifier section in file %s" % yaml_file)

ssg/constants.py

@@ -454,6 +454,8 @@
     'eks': 'Amazon Elastic Kubernetes Service',
 }
 
+# References that can not be used with product-qualifiers
+GLOBAL_REFERENCES = ("srg", "vmmsrg", "disa", "cis-csc",)
 
 # Application constants
 DEFAULT_GID_MIN = 1000

ssg/entities/common.py

@@ -15,9 +15,48 @@
     XCCDF_REFINABLE_PROPERTIES,
     XCCDF12_NS,
     OSCAP_VALUE,
+    GLOBAL_REFERENCES
 )
 
 
+def make_items_product_specific(items_dict, product_suffix, allow_overwrites=False):
+    new_items = dict()
+    for full_label, value in items_dict.items():
+        if "@" not in full_label and full_label not in new_items:
+            new_items[full_label] = value
+            continue
+
+        label = full_label.split("@")[0]
+
+        # This test should occur before matching product_suffix with the product qualifier
+        # present in the reference, so it catches problems even for products that are not
+        # being built at the moment
+        if label in GLOBAL_REFERENCES:
+            msg = (
+                "You cannot use product-qualified for the '{item_u}' reference. "
+                "Please remove the product-qualifier and merge values with the "
+                "existing reference if there is any. Original line: {item_q}: {value_q}"
+                .format(item_u=label, item_q=full_label, value_q=value)
+            )
+            raise ValueError(msg)
+
+        if not full_label.endswith(product_suffix):
+            continue
+
+        if label in items_dict and not allow_overwrites and value != items_dict[label]:
+            msg = (
+                "There is a product-qualified '{item_q}' item, "
+                "but also an unqualified '{item_u}' item "
+                "and those two differ in value - "
+                "'{value_q}' vs '{value_u}' respectively."
+                .format(item_q=full_label, item_u=label,
+                        value_q=value, value_u=items_dict[label])
+            )
+            raise ValueError(msg)
+        new_items[label] = value
+    return new_items
+
+
 def add_sub_element(parent, tag, ns, data):
     """
     Creates a new child element under parent with tag tag, and sets
@@ -81,6 +120,7 @@ class XCCDFEntity(object):
     """
     KEYS = dict(
             id_=lambda: "",
+            title=lambda: "",
             definition_location=lambda: "",
     )
 
@@ -309,3 +349,85 @@ def update_with(self, rhs):
         updated_refinements = self._subtract_refinements(extended_refinements)
         updated_refinements.update(self.refine_rules)
         self.refine_rules = updated_refinements
+
+
+class Templatable(object):
+
+    KEYS = dict(
+        template=lambda: None,
+    )
+
+    def __init__(self):
+        pass
+
+    def is_templated(self):
+        return isinstance(self.template, dict)
+
+    def get_template_context(self, env_yaml):
+        # TODO: The first two variables, 'rule_id' and 'rule_title' are expected by some
+        #       templates and macros even if they are not rendered in a rule context.
+        #       Better name for these variables are 'entity_id' and 'entity_title'.
+        return {
+            "rule_id": self.id_,
+            "rule_title": self.title,
+            "products": env_yaml["product"],
+        }
+
+    def get_template_name(self):
+        """
+        Given a template dictionary from a Rule instance, determine the name
+        of the template (from templates) this rule uses.
+        """
+        try:
+            template_name = self.template["name"]
+        except KeyError:
+            raise ValueError(
+                "Templatable {0} is missing template name under template key".format(self))
+        return template_name
+
+    def get_template_vars(self, env_yaml):
+        if "vars" not in self.template:
+            raise ValueError(
+                "Templatable {0} does not contain mandatory 'vars:' key under "
+                "'template:' key.".format(self))
+        template_vars = self.template["vars"]
+
+        # Add the rule ID which will be used in template preprocessors (template.py)
+        # as a unique sub-element for a variety of composite IDs.
+        # TODO: The name _rule_id is a legacy from the era when rule was the only
+        #       context for a template. Preprocessors implicitly depend on this name.
+        #       A better name is '_entity_id' (as in XCCDF Entity).
+        template_vars["_rule_id"] = self.id_
+
+        return make_items_product_specific(template_vars, env_yaml["product"])
+
+    def get_template_backend_langs(self):
+        """
+        Returns list of languages that should be generated from a template
+        configuration, controlled by backends.
+        """
+        from ..templates import LANGUAGES
+        if "backends" in self.template:
+            backends = self.template["backends"]
+            for lang in backends:
+                if lang not in LANGUAGES:
+                    raise RuntimeError("Templatable {0} wants to generate unknown language '{1}"
+                                       .format(self, lang))
+            return [lang for name, lang in LANGUAGES.items() if backends.get(name, "on") == "on"]
+        return LANGUAGES.values()
+
+    def make_template_product_specific(self, product):
+        if not self.is_templated():
+            return
+
+        product_suffix = "@{0}".format(product)
+
+        not_specific_vars = self.template.get("vars", dict())
+        specific_vars = make_items_product_specific(
+            not_specific_vars, product_suffix, True)
+        self.template["vars"] = specific_vars
+
+        not_specific_backends = self.template.get("backends", dict())
+        specific_backends = make_items_product_specific(
+            not_specific_backends, product_suffix, True)
+        self.template["backends"] = specific_backends

ssg/entities/profile_base.py

@@ -33,7 +33,6 @@ class Profile(XCCDFEntity, SelectionHandler):
     """Represents XCCDF profile
     """
     KEYS = dict(
-        title=lambda: "",
         description=lambda: "",
         extends=lambda: "",
         metadata=lambda: None,