From f6dfe665040a35f17076e72a6716e7f5b8aee514 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Wed, 16 Nov 2022 14:06:15 +0100 Subject: [PATCH 01/13] Refactor template builder Unify the code and remove duplication. Introduce a basic test for template builder. --- ssg/build_sce.py | 4 +- ssg/templates.py | 265 ++++++++---------- tests/ssg_test_suite/common.py | 5 +- .../ssg-module/data/templates/extra_ovals.yml | 4 + .../templates/package_installed/oval.template | 11 + .../templates/package_installed/template.py | 12 + .../templates/package_installed/template.yml | 2 + tests/unit/ssg-module/test_templates.py | 32 +++ 8 files changed, 185 insertions(+), 150 deletions(-) create mode 100644 tests/unit/ssg-module/data/templates/extra_ovals.yml create mode 100644 tests/unit/ssg-module/data/templates/package_installed/oval.template create mode 100644 tests/unit/ssg-module/data/templates/package_installed/template.py create mode 100644 tests/unit/ssg-module/data/templates/package_installed/template.yml create mode 100644 tests/unit/ssg-module/test_templates.py diff --git a/ssg/build_sce.py b/ssg/build_sce.py index 633eb579ff3..69cca9f4688 100644 --- a/ssg/build_sce.py +++ b/ssg/build_sce.py @@ -166,8 +166,8 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output): # While we don't _write_ it, we still need to parse SCE # metadata from the templated content. Render it internally. - raw_sce_content = template_builder.get_lang_for_rule( - rule_id, rule.title, rule.template, 'sce-bash') + raw_sce_content = template_builder.get_lang_contents( + rule_id, rule.title, rule.template, langs['sce-bash']) ext = '.sh' filename = rule_id + ext diff --git a/ssg/templates.py b/ssg/templates.py index 05384a6664f..b41f9437469 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -8,12 +8,14 @@ import ssg.build_yaml import ssg.utils import ssg.yaml +import ssg.jinja from ssg.build_cpe import ProductCPEs from collections import namedtuple templating_lang = namedtuple( "templating_language_attributes", ["name", "file_extension", "template_type", "lang_specific_dir"]) + template_type = ssg.utils.enum("remediation", "check") languages = { @@ -27,23 +29,32 @@ "puppet": templating_lang("puppet", ".pp", template_type.remediation, "puppet"), "sce-bash": templating_lang("sce-bash", ".sh", template_type.remediation, "sce") } -preprocessing_file_name = "template.py" -templates = dict() +PREPROCESSING_FILE_NAME = "template.py" +TEMPLATE_YAML_FILE_NAME = "template.yml" -class Template(): - def __init__(self, template_root_directory, name): - self.template_root_directory = template_root_directory +class Template: + def __init__(self, templates_root_directory, name): + self.langs = [] + self.templates_root_directory = templates_root_directory self.name = name - self.template_path = os.path.join(self.template_root_directory, self.name) - self.template_yaml_path = os.path.join(self.template_path, "template.yml") - self.preprocessing_file_path = os.path.join(self.template_path, preprocessing_file_name) - - def load(self): + self.template_path = os.path.join(self.templates_root_directory, self.name) + self.template_yaml_path = os.path.join(self.template_path, TEMPLATE_YAML_FILE_NAME) + self.preprocessing_file_path = os.path.join(self.template_path, PREPROCESSING_FILE_NAME) + + @classmethod + def load_template(cls, template_root_directory, name): + maybe_template = cls(template_root_directory, name) + if maybe_template._looks_like_template(): + maybe_template._load() + return maybe_template + return None + + def _load(self): if not os.path.exists(self.preprocessing_file_path): self.preprocessing_file_path = None - self.langs = [] + template_yaml = ssg.yaml.open_raw(self.template_yaml_path) for supported_lang in template_yaml["supported_languages"]: if supported_lang not in languages.keys(): @@ -60,11 +71,15 @@ def load(self): self.langs.append(lang) def preprocess(self, parameters, lang): - # if no template.py file exists, skip this preprocessing part + parameters = self._preprocess_with_template_module(parameters, lang) + # TODO: Remove this right after the variables in templates are renamed to lowercase + parameters = {k.upper(): v for k, v in parameters.items()} + return parameters + + def _preprocess_with_template_module(self, parameters, lang): if self.preprocessing_file_path is not None: unique_dummy_module_name = "template_" + self.name - preprocess_mod = imp.load_source( - unique_dummy_module_name, self.preprocessing_file_path) + preprocess_mod = imp.load_source(unique_dummy_module_name, self.preprocessing_file_path) if not hasattr(preprocess_mod, "preprocess"): msg = ( "The '{name}' template's preprocessing file {preprocessing_file} " @@ -73,17 +88,12 @@ def preprocess(self, parameters, lang): ) raise ValueError(msg) parameters = preprocess_mod.preprocess(parameters.copy(), lang) - # TODO: Remove this right after the variables in templates are renamed - # to lowercase - uppercases = dict() - for k, v in parameters.items(): - uppercases[k.upper()] = v - return uppercases - - def looks_like_template(self): - if not os.path.isdir(self.template_root_directory): + return parameters + + def _looks_like_template(self): + if not os.path.isdir(self.template_path): return False - if os.path.islink(self.template_root_directory): + if os.path.islink(self.template_path): return False template_sources = sorted(glob.glob(os.path.join(self.template_path, "*.template"))) if not os.path.isfile(self.template_yaml_path) and not template_sources: @@ -101,9 +111,8 @@ class Builder(object): output directory for remediations into the constructor. Then, call the method build() to perform a build. """ - def __init__( - self, env_yaml, resolved_rules_dir, templates_dir, - remediations_dir, checks_dir, platforms_dir, cpe_items_dir): + def __init__(self, env_yaml, resolved_rules_dir, templates_dir, + remediations_dir, checks_dir, platforms_dir, cpe_items_dir): self.env_yaml = env_yaml self.resolved_rules_dir = resolved_rules_dir self.templates_dir = templates_dir @@ -112,6 +121,19 @@ def __init__( self.platforms_dir = platforms_dir self.cpe_items_dir = cpe_items_dir self.output_dirs = dict() + self.templates = dict() + self._init_lang_output_dirs() + self._init_and_load_templates() + self.product_cpes = ProductCPEs() + self.product_cpes.load_cpes_from_directory_tree(cpe_items_dir, self.env_yaml) + + def _init_and_load_templates(self): + for item in sorted(os.listdir(self.templates_dir)): + maybe_template = Template.load_template(self.templates_dir, item) + if maybe_template is not None: + self.templates[item] = maybe_template + + def _init_lang_output_dirs(self): for lang_name, lang in languages.items(): lang_dir = lang.lang_specific_dir if lang.template_type == template_type.check: @@ -120,75 +142,20 @@ def __init__( output_dir = self.remediations_dir dir_ = os.path.join(output_dir, lang_dir) self.output_dirs[lang_name] = dir_ - # scan directory structure and dynamically create list of templates - for item in sorted(os.listdir(self.templates_dir)): - maybe_template = Template(templates_dir, item) - if maybe_template.looks_like_template(): - maybe_template.load() - templates[item] = maybe_template - self.product_cpes = ProductCPEs() - self.product_cpes.load_cpes_from_directory_tree(cpe_items_dir, self.env_yaml) - - def build_lang_file( - self, rule_id, template_name, template_vars, lang, local_env_yaml): - """ - Builds and returns templated content for a given rule for a given - language; does not write the output to disk. - """ - if lang not in templates[template_name].langs: - return None - - template_file_name = lang.name + ".template" - template_file_path = os.path.join(self.templates_dir, template_name, template_file_name) - template_parameters = templates[template_name].preprocess(template_vars, lang.name) - jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters) - filled_template = ssg.jinja.process_file_with_macros( - template_file_path, jinja_dict) - return filled_template - - def build_lang( - self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None): + def get_template_backend_langs(self, template, rule_id): """ - Builds templated content for a given rule for a given language. - Writes the output to the correct build directories. + Returns list of languages that should be generated from a template + configuration, controlled by backends. """ - if lang not in templates[template_name].langs or lang.name == "sce-bash": - return - - filled_template = self.build_lang_file(rule_id, template_name, - template_vars, lang, - local_env_yaml) - - ext = lang.file_extension - output_file_name = rule_id + ext - output_filepath = os.path.join( - self.output_dirs[lang.name], output_file_name) - - with open(output_filepath, "w") as f: - f.write(filled_template) - - def get_langs_to_generate(self, rule): - """ - For a given rule returns list of languages that should be generated - from templates. This is controlled by "template_backends" in rule.yml. - """ - if "backends" in rule.template: - backends = rule.template["backends"] + if "backends" in template: + backends = template["backends"] for lang in backends: - if lang not in languages.keys(): - raise RuntimeError( - "Rule {0} wants to generate unknown language '{1}" - "from a template.".format(rule.id_, lang) - ) - langs_to_generate = [] - for lang_name, lang in languages.items(): - backend = backends.get(lang_name, "on") - if backend == "on": - langs_to_generate.append(lang) - return langs_to_generate - else: - return languages.values() + if lang not in languages: + raise RuntimeError("Rule {0} wants to generate unknown language '{1}" + "from a template.".format(rule_id, lang)) + return [lang for name, lang in languages.items() if backends.get(name, "on") == "on"] + return languages.values() def get_template_name(self, template, rule_id): """ @@ -201,7 +168,7 @@ def get_template_name(self, template, rule_id): raise ValueError( "Rule {0} is missing template name under template key".format( rule_id)) - if template_name not in templates.keys(): + if template_name not in self.templates.keys(): raise ValueError( "Rule {0} uses template {1} which does not exist.".format( rule_id, template_name)) @@ -210,15 +177,15 @@ def get_template_name(self, template, rule_id): def get_resolved_langs_to_generate(self, rule): """ Given a specific Rule instance, determine which languages are - generated by the combination of the rule's template_backends AND - the rule's template keys. + generated by the combination of the rule's template supported_languages AND + the rule's template configuration backends. """ if rule.template is None: return None - rule_langs = set(self.get_langs_to_generate(rule)) + rule_langs = set(self.get_template_backend_langs(rule.template, rule.id_)) template_name = self.get_template_name(rule.template, rule.id_) - template_langs = set(templates[template_name].langs) + template_langs = set(self.templates[template_name].langs) return rule_langs.intersection(template_langs) def process_product_vars(self, all_variables): @@ -236,42 +203,31 @@ def process_product_vars(self, all_variables): return processed - def build_rule(self, rule_id, rule_title, template, langs_to_generate, identifiers, - platforms=None): + def render_lang_file(self, template_name, template_vars, lang, local_env_yaml): """ - Builds templated content for a given rule for selected languages, - writing the output to the correct build directories. + Builds and returns templated content for a given rule for a given + language; does not write the output to disk. """ - template_name = self.get_template_name(template, rule_id) + if lang not in self.templates[template_name].langs: + return None + + template_file_name = lang.name + ".template" + template_file_path = os.path.join(self.templates_dir, template_name, template_file_name) + template_parameters = self.templates[template_name].preprocess(template_vars, lang.name) + env_yaml = self.env_yaml.copy() + env_yaml.update(local_env_yaml) + jinja_dict = ssg.utils.merge_dicts(env_yaml, template_parameters) try: - template_vars = self.process_product_vars(template["vars"]) - except KeyError: - raise ValueError( - "Rule {0} does not contain mandatory 'vars:' key under " - "'template:' key.".format(rule_id)) - # Add the rule ID which will be reused in OVAL templates as OVAL - # definition ID so that the build system matches the generated - # check with the rule. - template_vars["_rule_id"] = rule_id - # checks and remediations are processed with a custom YAML dict - local_env_yaml = self.env_yaml.copy() - local_env_yaml["rule_id"] = rule_id - local_env_yaml["rule_title"] = rule_title - local_env_yaml["products"] = self.env_yaml["product"] - if identifiers is not None: - local_env_yaml["cce_identifiers"] = identifiers - - for lang in langs_to_generate: - try: - self.build_lang( - rule_id, template_name, template_vars, lang, local_env_yaml, platforms) - except Exception as e: - raise e( - "Error building templated {0} content for rule {1}".format(lang, rule_id)) + filled_template = ssg.jinja.process_file_with_macros(template_file_path, jinja_dict) + except Exception as e: + print("Error in template: %s (lang: %s)" % (template_name, lang.name)) + raise e + + return filled_template - def get_lang_for_rule(self, rule_id, rule_title, template, language): + def get_lang_contents(self, rule_id, rule_title, template, language, extra_env=None): """ - For the specified rule, build and return only the specified language + For the specified template, build and return only the specified language content. """ template_name = self.get_template_name(template, rule_id) @@ -285,25 +241,49 @@ def get_lang_for_rule(self, rule_id, rule_title, template, language): # definition ID so that the build system matches the generated # check with the rule. template_vars["_rule_id"] = rule_id - # checks and remediations are processed with a custom YAML dict - local_env_yaml = self.env_yaml.copy() - local_env_yaml["rule_id"] = rule_id - local_env_yaml["rule_title"] = rule_title - local_env_yaml["products"] = self.env_yaml["product"] - return self.build_lang_file(rule_id, template_name, template_vars, - language, local_env_yaml) + # Checks and remediations are processed with a custom YAML dict + local_env_yaml = {"rule_id": rule_id, "rule_title": rule_title, + "products": self.env_yaml["product"]} + if extra_env is not None: + local_env_yaml.update(extra_env) + + return self.render_lang_file(template_name, template_vars, language, local_env_yaml) + + def build_lang(self, rule_id, rule_title, template, lang, extra_env=None): + """ + Builds templated content for a given rule for a given language. + Writes the output to the correct build directories. + """ + filled_template = self.get_lang_contents(rule_id, rule_title, template, lang, extra_env) + + output_file_name = rule_id + lang.file_extension + output_filepath = os.path.join(self.output_dirs[lang.name], output_file_name) + + with open(output_filepath, "w") as f: + f.write(filled_template) + + def build_rule(self, rule): + """ + Builds templated content for a given rule for selected languages, + writing the output to the correct build directories. + """ + extra_env = {} + if rule.identifiers is not None: + extra_env["cce_identifiers"] = rule.identifiers + + for lang in self.get_resolved_langs_to_generate(rule): + if lang.name != "sce-bash": + self.build_lang(rule.id_, rule.title, rule.template, lang, extra_env) def build_extra_ovals(self): declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml") declaration = ssg.yaml.open_raw(declaration_path) for oval_def_id, template in declaration.items(): - langs_to_generate = [languages["oval"]] # Since OVAL definition ID in shorthand format is always the same # as rule ID, we can use it instead of the rule ID even if no rule # with that ID exists - self.build_rule( - oval_def_id, oval_def_id, template, langs_to_generate, None) + self.build_lang(oval_def_id, oval_def_id, template, languages["oval"]) def build_all_rules(self): for rule_file in sorted(os.listdir(self.resolved_rules_dir)): @@ -313,19 +293,14 @@ def build_all_rules(self): except ssg.build_yaml.DocumentationNotComplete: # Happens on non-debug build when a rule is "documentation-incomplete" continue - if rule.template is None: - # rule is not templated, skipping - continue - langs_to_generate = self.get_langs_to_generate(rule) - self.build_rule(rule.id_, rule.title, rule.template, langs_to_generate, - rule.identifiers, platforms=rule.platforms) + if rule.template is not None: + self.build_rule(rule) def build(self): """ Builds all templated content for all languages, writing the output to the correct build directories. """ - for dir_ in self.output_dirs.values(): if not os.path.exists(dir_): os.makedirs(dir_) diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py index 30a4113da04..c055a4f3bc6 100644 --- a/tests/ssg_test_suite/common.py +++ b/tests/ssg_test_suite/common.py @@ -505,9 +505,8 @@ def load_test(absolute_path, rule_template, local_env_yaml): template_name = rule_template['name'] template_vars = rule_template['vars'] # Load template parameters and apply it to the test case. - maybe_template = ssg.templates.Template(_SHARED_TEMPLATES, template_name) - if maybe_template.looks_like_template(): - maybe_template.load() + maybe_template = ssg.templates.Template.load_template(_SHARED_TEMPLATES, template_name) + if maybe_template is not None: template_parameters = maybe_template.preprocess(template_vars, "tests") else: raise ValueError("Rule uses template '{}' " diff --git a/tests/unit/ssg-module/data/templates/extra_ovals.yml b/tests/unit/ssg-module/data/templates/extra_ovals.yml new file mode 100644 index 00000000000..78a223af8e4 --- /dev/null +++ b/tests/unit/ssg-module/data/templates/extra_ovals.yml @@ -0,0 +1,4 @@ +package_avahi_installed: + name: package_installed + vars: + pkgname: avahi diff --git a/tests/unit/ssg-module/data/templates/package_installed/oval.template b/tests/unit/ssg-module/data/templates/package_installed/oval.template new file mode 100644 index 00000000000..51a5cb4a08e --- /dev/null +++ b/tests/unit/ssg-module/data/templates/package_installed/oval.template @@ -0,0 +1,11 @@ + + + {{{ oval_metadata("The " + pkg_system|upper + " package " + PKGNAME + " should be installed.", affected_platforms=["multi_platform_all"]) }}} + + + + +{{{ oval_test_package_installed(package=PKGNAME, evr=EVR, test_id="test_package_"+PKGNAME+"_installed") }}} + diff --git a/tests/unit/ssg-module/data/templates/package_installed/template.py b/tests/unit/ssg-module/data/templates/package_installed/template.py new file mode 100644 index 00000000000..cfb47b7af5d --- /dev/null +++ b/tests/unit/ssg-module/data/templates/package_installed/template.py @@ -0,0 +1,12 @@ +import re + + +def preprocess(data, lang): + if "evr" in data: + evr = data["evr"] + if evr and not re.match(r'\d:\d[\d\w+.]*-\d[\d\w+.]*', evr, 0): + raise RuntimeError( + "ERROR: input violation: evr key should be in " + "epoch:version-release format, but package {0} has set " + "evr to {1}".format(data["pkgname"], evr)) + return data diff --git a/tests/unit/ssg-module/data/templates/package_installed/template.yml b/tests/unit/ssg-module/data/templates/package_installed/template.yml new file mode 100644 index 00000000000..2f6f2d2c7cb --- /dev/null +++ b/tests/unit/ssg-module/data/templates/package_installed/template.yml @@ -0,0 +1,2 @@ +supported_languages: + - oval diff --git a/tests/unit/ssg-module/test_templates.py b/tests/unit/ssg-module/test_templates.py new file mode 100644 index 00000000000..87972406fe5 --- /dev/null +++ b/tests/unit/ssg-module/test_templates.py @@ -0,0 +1,32 @@ +import os + +import ssg.templates as tpl +from ssg.environment import open_environment +import ssg.utils +import ssg.products +from ssg.yaml import ordered_load +import ssg.build_yaml +import ssg.build_cpe + + +ssg_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +DATADIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "data")) +templates_dir = os.path.join(DATADIR, "templates") +cpe_items_dir = os.path.join(DATADIR, "applicability") + +build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") +product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") +env_yaml = open_environment(build_config_yaml, product_yaml) + + +def test_render_extra_ovals(): + builder = ssg.templates.Builder( + env_yaml, '', templates_dir, + '', '', '', cpe_items_dir) + + declaration_path = os.path.join(builder.templates_dir, "extra_ovals.yml") + declaration = ssg.yaml.open_raw(declaration_path) + for oval_def_id, template in declaration.items(): + oval_content = builder.get_lang_contents(oval_def_id, oval_def_id, template, + ssg.templates.languages["oval"]) + assert "%s" % (oval_def_id,) in oval_content From 219ea259a4a648d55b4d42380e0853e60a01be79 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Thu, 24 Nov 2022 05:00:24 +0100 Subject: [PATCH 02/13] Refactor templates and XCCDFEntities Move 'title' attribute to the base class XCCDFEntity. It is a base type's (Item/Entity) element according the specs. Introduce sidekick Templatable class for XCCDFEntities that could be templated. It's a mix-in class that will help in rendering templates for various XCCDFEntities in the future. --- ssg/build_cpe.py | 1 - ssg/build_yaml.py | 8 ++------ ssg/entities/common.py | 14 ++++++++++++++ ssg/entities/profile_base.py | 1 - 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ssg/build_cpe.py b/ssg/build_cpe.py index fc19bcd9f76..0a694490ffe 100644 --- a/ssg/build_cpe.py +++ b/ssg/build_cpe.py @@ -140,7 +140,6 @@ class CPEItem(XCCDFEntity): KEYS = dict( name=lambda: "", - title=lambda: "", check_id=lambda: "", bash_conditional=lambda: "", ansible_conditional=lambda: "", diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 2dc17ac067b..84af1588778 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -187,7 +187,6 @@ class Value(XCCDFEntity): """Represents XCCDF Value """ KEYS = dict( - title=lambda: "", description=lambda: "", type=lambda: "", operator=lambda: "equals", @@ -260,7 +259,6 @@ class Benchmark(XCCDFEntity): """Represents XCCDF Benchmark """ KEYS = dict( - title=lambda: "", status=lambda: "", description=lambda: "", notice_id=lambda: "", @@ -480,7 +478,6 @@ class Group(XCCDFEntity): KEYS = dict( prodtype=lambda: "all", - title=lambda: "", description=lambda: "", warnings=lambda: list(), requires=lambda: list(), @@ -697,7 +694,6 @@ class Rule(XCCDFEntity): """ KEYS = dict( prodtype=lambda: "all", - title=lambda: "", description=lambda: "", rationale=lambda: "", severity=lambda: "", @@ -718,12 +714,12 @@ class Rule(XCCDFEntity): platforms=lambda: set(), sce_metadata=lambda: dict(), inherited_platforms=lambda: set(), - template=lambda: None, cpe_platform_names=lambda: set(), inherited_cpe_platform_names=lambda: set(), bash_conditional=lambda: None, fixes=lambda: dict(), - ** XCCDFEntity.KEYS + ** XCCDFEntity.KEYS, + ** Templatable.KEYS ) MANDATORY_KEYS = { diff --git a/ssg/entities/common.py b/ssg/entities/common.py index ccb3786067c..b0666304b27 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -81,6 +81,7 @@ class XCCDFEntity(object): """ KEYS = dict( id_=lambda: "", + title=lambda: "", definition_location=lambda: "", ) @@ -309,3 +310,16 @@ def update_with(self, rhs): updated_refinements = self._subtract_refinements(extended_refinements) updated_refinements.update(self.refine_rules) self.refine_rules = updated_refinements + + +class Templatable(object): + + KEYS = dict( + template=lambda: None, + ) + + def __init__(self): + pass + + def is_templated(self): + return isinstance(self.template, dict) diff --git a/ssg/entities/profile_base.py b/ssg/entities/profile_base.py index da0a5319f89..f9233414864 100644 --- a/ssg/entities/profile_base.py +++ b/ssg/entities/profile_base.py @@ -33,7 +33,6 @@ class Profile(XCCDFEntity, SelectionHandler): """Represents XCCDF profile """ KEYS = dict( - title=lambda: "", description=lambda: "", extends=lambda: "", metadata=lambda: None, From 395af0ed286aca99c2c379bac60a8d6878644710 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Thu, 24 Nov 2022 05:17:19 +0100 Subject: [PATCH 03/13] Refactor templates: Rule and Templatable and products Move make_items_product_specific method from Rule into common module as static function along with GLOBAL_REFERENCES (constants). This would allow us to reuse code beyond Rule component. Now Templatable can normalize/productify template variables on its own. Clean up duplicate definition of add_sub_element function. --- ssg/build_yaml.py | 101 ++--------------------------------------- ssg/constants.py | 2 + ssg/entities/common.py | 55 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 97 deletions(-) diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 84af1588778..15415c4d9d7 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -44,51 +44,11 @@ from .shims import unicode_func import ssg.build_stig -from .entities.common import ( - XCCDFEntity, - add_sub_element, -) +from .entities.common import add_sub_element, make_items_product_specific, \ + XCCDFEntity, Templatable from .entities.profile import Profile, ProfileWithInlinePolicies -def add_sub_element(parent, tag, ns, data): - """ - Creates a new child element under parent with tag tag, and sets - data as the content under the tag. In particular, data is a string - to be parsed as an XML tree, allowing sub-elements of children to be - added. - - If data should not be parsed as an XML tree, either escape the contents - before passing into this function, or use ElementTree.SubElement(). - - Returns the newly created subelement of type tag. - """ - namespaced_data = add_xhtml_namespace(data) - # This is used because our YAML data contain XML and XHTML elements - # ET.SubElement() escapes the < > characters by < and > - # and therefore it does not add child elements - # we need to do a hack instead - # TODO: Remove this function after we move to Markdown everywhere in SSG - ustr = unicode_func('<{0} xmlns="{3}" xmlns:xhtml="{2}">{1}').format( - tag, namespaced_data, xhtml_namespace, ns) - - try: - element = ET.fromstring(ustr.encode("utf-8")) - except Exception: - msg = ("Error adding subelement to an element '{0}' from string: '{1}'" - .format(parent.tag, ustr)) - raise RuntimeError(msg) - - # Apart from HTML and XML elements the rule descriptions and similar - # also contain elements, where we need to add the prefix - # to create a full reference. - for x in element.findall(".//{%s}sub" % XCCDF12_NS): - x.set("idref", OSCAP_VALUE + x.get("idref")) - x.set("use", "legacy") - parent.append(element) - return element - - def reorder_according_to_ordering(unordered, ordering, regex=None): ordered = [] if regex is None: @@ -689,7 +649,7 @@ def filterfunc(rule): return filterfunc -class Rule(XCCDFEntity): +class Rule(XCCDFEntity, Templatable): """Represents XCCDF Rule """ KEYS = dict( @@ -733,7 +693,6 @@ class Rule(XCCDFEntity): ID_LABEL = "rule_id" PRODUCT_REFERENCES = ("stigid", "cis",) - GLOBAL_REFERENCES = ("srg", "vmmsrg", "disa", "cis-csc",) def __init__(self, id_): super(Rule, self).__init__(id_) @@ -891,21 +850,6 @@ def load_policy_specific_content(self, rule_filename, env_yaml): env_yaml, policy_specific_content_files) self.policy_specific_content = policy_specific_content - def make_template_product_specific(self, product): - product_suffix = "@{0}".format(product) - - if not self.template: - return - - not_specific_vars = self.template.get("vars", dict()) - specific_vars = self._make_items_product_specific( - not_specific_vars, product_suffix, True) - self.template["vars"] = specific_vars - - not_specific_backends = self.template.get("backends", dict()) - specific_backends = self._make_items_product_specific( - not_specific_backends, product_suffix, True) - self.template["backends"] = specific_backends def make_refs_and_identifiers_product_specific(self, product): product_suffix = "@{0}".format(product) @@ -929,7 +873,7 @@ def make_refs_and_identifiers_product_specific(self, product): ) for name, (dic, allow_overwrites) in to_set.items(): try: - new_items = self._make_items_product_specific( + new_items = make_items_product_specific( dic, product_suffix, allow_overwrites) except ValueError as exc: msg = ( @@ -946,43 +890,6 @@ def make_refs_and_identifiers_product_specific(self, product): self._verify_stigid_format(product) - def _make_items_product_specific(self, items_dict, product_suffix, allow_overwrites=False): - new_items = dict() - for full_label, value in items_dict.items(): - if "@" not in full_label and full_label not in new_items: - new_items[full_label] = value - continue - - label = full_label.split("@")[0] - - # this test should occur before matching product_suffix with the product qualifier - # present in the reference, so it catches problems even for products that are not - # being built at the moment - if label in Rule.GLOBAL_REFERENCES: - msg = ( - "You cannot use product-qualified for the '{item_u}' reference. " - "Please remove the product-qualifier and merge values with the " - "existing reference if there is any. Original line: {item_q}: {value_q}" - .format(item_u=label, item_q=full_label, value_q=value) - ) - raise ValueError(msg) - - if not full_label.endswith(product_suffix): - continue - - if label in items_dict and not allow_overwrites and value != items_dict[label]: - msg = ( - "There is a product-qualified '{item_q}' item, " - "but also an unqualified '{item_u}' item " - "and those two differ in value - " - "'{value_q}' vs '{value_u}' respectively." - .format(item_q=full_label, item_u=label, - value_q=value, value_u=items_dict[label]) - ) - raise ValueError(msg) - new_items[label] = value - return new_items - def validate_identifiers(self, yaml_file): if self.identifiers is None: raise ValueError("Empty identifier section in file %s" % yaml_file) diff --git a/ssg/constants.py b/ssg/constants.py index ed68c053325..7f88a087487 100644 --- a/ssg/constants.py +++ b/ssg/constants.py @@ -454,6 +454,8 @@ 'eks': 'Amazon Elastic Kubernetes Service', } +# References that can not be used with product-qualifiers +GLOBAL_REFERENCES = ("srg", "vmmsrg", "disa", "cis-csc",) # Application constants DEFAULT_GID_MIN = 1000 diff --git a/ssg/entities/common.py b/ssg/entities/common.py index b0666304b27..76235d4bc5c 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -15,9 +15,48 @@ XCCDF_REFINABLE_PROPERTIES, XCCDF12_NS, OSCAP_VALUE, + GLOBAL_REFERENCES ) +def make_items_product_specific(items_dict, product_suffix, allow_overwrites=False): + new_items = dict() + for full_label, value in items_dict.items(): + if "@" not in full_label and full_label not in new_items: + new_items[full_label] = value + continue + + label = full_label.split("@")[0] + + # This test should occur before matching product_suffix with the product qualifier + # present in the reference, so it catches problems even for products that are not + # being built at the moment + if label in GLOBAL_REFERENCES: + msg = ( + "You cannot use product-qualified for the '{item_u}' reference. " + "Please remove the product-qualifier and merge values with the " + "existing reference if there is any. Original line: {item_q}: {value_q}" + .format(item_u=label, item_q=full_label, value_q=value) + ) + raise ValueError(msg) + + if not full_label.endswith(product_suffix): + continue + + if label in items_dict and not allow_overwrites and value != items_dict[label]: + msg = ( + "There is a product-qualified '{item_q}' item, " + "but also an unqualified '{item_u}' item " + "and those two differ in value - " + "'{value_q}' vs '{value_u}' respectively." + .format(item_q=full_label, item_u=label, + value_q=value, value_u=items_dict[label]) + ) + raise ValueError(msg) + new_items[label] = value + return new_items + + def add_sub_element(parent, tag, ns, data): """ Creates a new child element under parent with tag tag, and sets @@ -323,3 +362,19 @@ def __init__(self): def is_templated(self): return isinstance(self.template, dict) + + def make_template_product_specific(self, product): + if not self.is_templated(): + return + + product_suffix = "@{0}".format(product) + + not_specific_vars = self.template.get("vars", dict()) + specific_vars = make_items_product_specific( + not_specific_vars, product_suffix, True) + self.template["vars"] = specific_vars + + not_specific_backends = self.template.get("backends", dict()) + specific_backends = make_items_product_specific( + not_specific_backends, product_suffix, True) + self.template["backends"] = specific_backends From b7ff233b7d82d6d0d6c083703cda1c35bc27fe9a Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Thu, 24 Nov 2022 05:35:57 +0100 Subject: [PATCH 04/13] Refactor templates: move XCCDFEntity-specific code into Templatable Clean up template.Builder and delegate entity-specific functions to Templatable. The problem with legacy 'rule_id', 'rule_title' and '_rule_id' template variables is localized in Templatable. Rename LANGUAGES constant, TemplatingLang and TemplateType.XXX in accordance with PEP8 and co. Clean up and decompose methods of template.Builder. --- ssg/build_sce.py | 4 +- ssg/build_yaml.py | 5 + ssg/entities/common.py | 53 +++++++ ssg/templates.py | 185 +++++++++--------------- tests/unit/ssg-module/test_templates.py | 9 +- 5 files changed, 132 insertions(+), 124 deletions(-) diff --git a/ssg/build_sce.py b/ssg/build_sce.py index 69cca9f4688..5bfe437650c 100644 --- a/ssg/build_sce.py +++ b/ssg/build_sce.py @@ -166,8 +166,8 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output): # While we don't _write_ it, we still need to parse SCE # metadata from the templated content. Render it internally. - raw_sce_content = template_builder.get_lang_contents( - rule_id, rule.title, rule.template, langs['sce-bash']) + raw_sce_content = template_builder.get_templatable_lang_contents(rule, + langs['sce-bash']) ext = '.sh' filename = rule_id + ext diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 15415c4d9d7..4b9e4abc430 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -850,6 +850,11 @@ def load_policy_specific_content(self, rule_filename, env_yaml): env_yaml, policy_specific_content_files) self.policy_specific_content = policy_specific_content + def get_template_context(self, env_yaml): + ctx = super(Rule, self).get_template_context(env_yaml) + if self.identifiers: + ctx["cce_identifiers"] = self.identifiers + return ctx def make_refs_and_identifiers_product_specific(self, product): product_suffix = "@{0}".format(product) diff --git a/ssg/entities/common.py b/ssg/entities/common.py index 76235d4bc5c..58043c7c667 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -363,6 +363,59 @@ def __init__(self): def is_templated(self): return isinstance(self.template, dict) + def get_template_context(self, env_yaml): + # TODO: The first two variables, 'rule_id' and 'rule_title' are expected by some + # templates and macros even if they are not rendered in a rule context. + # Better name for these variables are 'entity_id' and 'entity_title'. + return { + "rule_id": self.id_, + "rule_title": self.title, + "products": env_yaml["product"], + } + + def get_template_name(self): + """ + Given a template dictionary from a Rule instance, determine the name + of the template (from templates) this rule uses. + """ + try: + template_name = self.template["name"] + except KeyError: + raise ValueError( + "Templatable {0} is missing template name under template key".format(self)) + return template_name + + def get_template_vars(self, env_yaml): + if "vars" not in self.template: + raise ValueError( + "Templatable {0} does not contain mandatory 'vars:' key under " + "'template:' key.".format(self)) + template_vars = self.template["vars"] + + # Add the rule ID which will be used in template preprocessors (template.py) + # as a unique sub-element for a variety of composite IDs. + # TODO: The name _rule_id is a legacy from the era when rule was the only + # context for a template. Preprocessors implicitly depend on this name. + # A better name is '_entity_id' (as in XCCDF Entity). + template_vars["_rule_id"] = self.id_ + + return make_items_product_specific(template_vars, env_yaml["product"]) + + def get_template_backend_langs(self): + """ + Returns list of languages that should be generated from a template + configuration, controlled by backends. + """ + from ..templates import LANGUAGES + if "backends" in self.template: + backends = self.template["backends"] + for lang in backends: + if lang not in LANGUAGES: + raise RuntimeError("Templatable {0} wants to generate unknown language '{1}" + .format(self, lang)) + return [lang for name, lang in LANGUAGES.items() if backends.get(name, "on") == "on"] + return LANGUAGES.values() + def make_template_product_specific(self, product): if not self.is_templated(): return diff --git a/ssg/templates.py b/ssg/templates.py index b41f9437469..16336b38922 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -12,22 +12,23 @@ from ssg.build_cpe import ProductCPEs from collections import namedtuple -templating_lang = namedtuple( + +TemplatingLang = namedtuple( "templating_language_attributes", ["name", "file_extension", "template_type", "lang_specific_dir"]) -template_type = ssg.utils.enum("remediation", "check") - -languages = { - "anaconda": templating_lang("anaconda", ".anaconda", template_type.remediation, "anaconda"), - "ansible": templating_lang("ansible", ".yml", template_type.remediation, "ansible"), - "bash": templating_lang("bash", ".sh", template_type.remediation, "bash"), - "blueprint": templating_lang("blueprint", ".toml", template_type.remediation, "blueprint"), - "ignition": templating_lang("ignition", ".yml", template_type.remediation, "ignition"), - "kubernetes": templating_lang("kubernetes", ".yml", template_type.remediation, "kubernetes"), - "oval": templating_lang("oval", ".xml", template_type.check, "oval"), - "puppet": templating_lang("puppet", ".pp", template_type.remediation, "puppet"), - "sce-bash": templating_lang("sce-bash", ".sh", template_type.remediation, "sce") +TemplateType = ssg.utils.enum("REMEDIATION", "CHECK") + +LANGUAGES = { + "anaconda": TemplatingLang("anaconda", ".anaconda", TemplateType.REMEDIATION, "anaconda"), + "ansible": TemplatingLang("ansible", ".yml", TemplateType.REMEDIATION, "ansible"), + "bash": TemplatingLang("bash", ".sh", TemplateType.REMEDIATION, "bash"), + "blueprint": TemplatingLang("blueprint", ".toml", TemplateType.REMEDIATION, "blueprint"), + "ignition": TemplatingLang("ignition", ".yml", TemplateType.REMEDIATION, "ignition"), + "kubernetes": TemplatingLang("kubernetes", ".yml", TemplateType.REMEDIATION, "kubernetes"), + "oval": TemplatingLang("oval", ".xml", TemplateType.CHECK, "oval"), + "puppet": TemplatingLang("puppet", ".pp", TemplateType.REMEDIATION, "puppet"), + "sce-bash": TemplatingLang("sce-bash", ".sh", TemplateType.REMEDIATION, "sce") } PREPROCESSING_FILE_NAME = "template.py" @@ -57,12 +58,12 @@ def _load(self): template_yaml = ssg.yaml.open_raw(self.template_yaml_path) for supported_lang in template_yaml["supported_languages"]: - if supported_lang not in languages.keys(): + if supported_lang not in LANGUAGES.keys(): raise ValueError( "The template {0} declares to support the {1} language," "but this language is not supported by the content.".format( self.name, supported_lang)) - lang = languages[supported_lang] + lang = LANGUAGES[supported_lang] langfilename = lang.name + ".template" if not os.path.exists(os.path.join(self.template_path, langfilename)): raise ValueError( @@ -134,82 +135,40 @@ def _init_and_load_templates(self): self.templates[item] = maybe_template def _init_lang_output_dirs(self): - for lang_name, lang in languages.items(): + for lang_name, lang in LANGUAGES.items(): lang_dir = lang.lang_specific_dir - if lang.template_type == template_type.check: + if lang.template_type == TemplateType.CHECK: output_dir = self.checks_dir else: output_dir = self.remediations_dir dir_ = os.path.join(output_dir, lang_dir) self.output_dirs[lang_name] = dir_ - def get_template_backend_langs(self, template, rule_id): - """ - Returns list of languages that should be generated from a template - configuration, controlled by backends. + def get_resolved_langs_to_generate(self, templatable): """ - if "backends" in template: - backends = template["backends"] - for lang in backends: - if lang not in languages: - raise RuntimeError("Rule {0} wants to generate unknown language '{1}" - "from a template.".format(rule_id, lang)) - return [lang for name, lang in languages.items() if backends.get(name, "on") == "on"] - return languages.values() - - def get_template_name(self, template, rule_id): + Given a specific Templatable instance, determine which languages are + generated by the combination of the template supported_languages AND + the Templatable's template configuration 'backends'. """ - Given a template dictionary from a Rule instance, determine the name - of the template (from templates) this rule uses. - """ - try: - template_name = template["name"] - except KeyError: - raise ValueError( - "Rule {0} is missing template name under template key".format( - rule_id)) + if not templatable.is_templated(): + return [] + + rule_langs = set(templatable.get_template_backend_langs()) + template_name = templatable.get_template_name() if template_name not in self.templates.keys(): raise ValueError( - "Rule {0} uses template {1} which does not exist.".format( - rule_id, template_name)) - return template_name - - def get_resolved_langs_to_generate(self, rule): - """ - Given a specific Rule instance, determine which languages are - generated by the combination of the rule's template supported_languages AND - the rule's template configuration backends. - """ - if rule.template is None: - return None - - rule_langs = set(self.get_template_backend_langs(rule.template, rule.id_)) - template_name = self.get_template_name(rule.template, rule.id_) + "Templatable {0} uses template {1} which does not exist." + .format(templatable, template_name)) template_langs = set(self.templates[template_name].langs) return rule_langs.intersection(template_langs) - def process_product_vars(self, all_variables): - """ - Given a dictionary with the format key[@]=value, filter out - and only take keys that apply to this product (unqualified or qualified - to exactly this product). Returns a new dict. - """ - processed = dict(filter(lambda item: '@' not in item[0], all_variables.items())) - suffix = '@' + self.env_yaml['product'] - for variable in filter(lambda key: key.endswith(suffix), all_variables): - new_variable = variable[:-len(suffix)] - value = all_variables[variable] - processed[new_variable] = value - - return processed - - def render_lang_file(self, template_name, template_vars, lang, local_env_yaml): + def process_template_lang_file(self, template_name, template_vars, lang, local_env_yaml): """ - Builds and returns templated content for a given rule for a given - language; does not write the output to disk. + Processes template for a given template name and language and returns rendered content. """ if lang not in self.templates[template_name].langs: - return None + raise ValueError("Language {0} is not available for template {1}." + .format(lang.name, template_name)) template_file_name = lang.name + ".template" template_file_path = os.path.join(self.templates_dir, template_name, template_file_name) @@ -217,64 +176,45 @@ def render_lang_file(self, template_name, template_vars, lang, local_env_yaml): env_yaml = self.env_yaml.copy() env_yaml.update(local_env_yaml) jinja_dict = ssg.utils.merge_dicts(env_yaml, template_parameters) - try: - filled_template = ssg.jinja.process_file_with_macros(template_file_path, jinja_dict) - except Exception as e: - print("Error in template: %s (lang: %s)" % (template_name, lang.name)) - raise e + return ssg.jinja.process_file_with_macros(template_file_path, jinja_dict) - return filled_template - - def get_lang_contents(self, rule_id, rule_title, template, language, extra_env=None): + def get_templatable_lang_contents(self, templatable, language): """ - For the specified template, build and return only the specified language - content. + For the specified Templatable, build and return only the specified language content. """ - template_name = self.get_template_name(template, rule_id) - try: - template_vars = self.process_product_vars(template["vars"]) - except KeyError: - raise ValueError( - "Rule {0} does not contain mandatory 'vars:' key under " - "'template:' key.".format(rule_id)) - # Add the rule ID which will be reused in OVAL templates as OVAL - # definition ID so that the build system matches the generated - # check with the rule. - template_vars["_rule_id"] = rule_id + template_name = templatable.get_template_name() + template_vars = templatable.get_template_vars(self.env_yaml) # Checks and remediations are processed with a custom YAML dict - local_env_yaml = {"rule_id": rule_id, "rule_title": rule_title, - "products": self.env_yaml["product"]} - if extra_env is not None: - local_env_yaml.update(extra_env) - - return self.render_lang_file(template_name, template_vars, language, local_env_yaml) - - def build_lang(self, rule_id, rule_title, template, lang, extra_env=None): - """ - Builds templated content for a given rule for a given language. - Writes the output to the correct build directories. - """ - filled_template = self.get_lang_contents(rule_id, rule_title, template, lang, extra_env) + local_env_yaml = templatable.get_template_context(self.env_yaml) + try: + return self.process_template_lang_file(template_name, template_vars, language, local_env_yaml) + except Exception as e: + raise RuntimeError("Unable to generate {0} template language for Templatable {1}: {2}" + .format(language.name, templatable, e)) - output_file_name = rule_id + lang.file_extension + def write_templatable_lang_contents(self, filled_template, lang, templatable): + output_file_name = templatable.id_ + lang.file_extension output_filepath = os.path.join(self.output_dirs[lang.name], output_file_name) - with open(output_filepath, "w") as f: f.write(filled_template) - def build_rule(self, rule): + def build_templatable_lang(self, templatable, lang): """ - Builds templated content for a given rule for selected languages, + Builds templated content of a given Templatable for a selected language, writing the output to the correct build directories. """ - extra_env = {} - if rule.identifiers is not None: - extra_env["cce_identifiers"] = rule.identifiers + filled_template = self.get_templatable_lang_contents(templatable, lang) + self.write_templatable_lang_contents(filled_template, lang, templatable) + def build_rule(self, rule): + """ + Builds templated content of a given Rule for all available languages, + writing the output to the correct build directories. + """ for lang in self.get_resolved_langs_to_generate(rule): if lang.name != "sce-bash": - self.build_lang(rule.id_, rule.title, rule.template, lang, extra_env) + self.build_templatable_lang(rule, lang) def build_extra_ovals(self): declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml") @@ -283,7 +223,12 @@ def build_extra_ovals(self): # Since OVAL definition ID in shorthand format is always the same # as rule ID, we can use it instead of the rule ID even if no rule # with that ID exists - self.build_lang(oval_def_id, oval_def_id, template, languages["oval"]) + rule = ssg.build_yaml.Rule.get_instance_from_full_dict({ + "id_": oval_def_id, + "title": oval_def_id, + "template": template, + }) + self.build_templatable_lang(rule, LANGUAGES["oval"]) def build_all_rules(self): for rule_file in sorted(os.listdir(self.resolved_rules_dir)): @@ -293,13 +238,13 @@ def build_all_rules(self): except ssg.build_yaml.DocumentationNotComplete: # Happens on non-debug build when a rule is "documentation-incomplete" continue - if rule.template is not None: + if rule.is_templated(): self.build_rule(rule) def build(self): """ - Builds all templated content for all languages, writing - the output to the correct build directories. + Builds all templated content for all languages, + writing the output to the correct build directories. """ for dir_ in self.output_dirs.values(): if not os.path.exists(dir_): diff --git a/tests/unit/ssg-module/test_templates.py b/tests/unit/ssg-module/test_templates.py index 87972406fe5..3c9c925d775 100644 --- a/tests/unit/ssg-module/test_templates.py +++ b/tests/unit/ssg-module/test_templates.py @@ -27,6 +27,11 @@ def test_render_extra_ovals(): declaration_path = os.path.join(builder.templates_dir, "extra_ovals.yml") declaration = ssg.yaml.open_raw(declaration_path) for oval_def_id, template in declaration.items(): - oval_content = builder.get_lang_contents(oval_def_id, oval_def_id, template, - ssg.templates.languages["oval"]) + rule = ssg.build_yaml.Rule.get_instance_from_full_dict({ + "id_": oval_def_id, + "title": oval_def_id, + "template": template, + }) + oval_content = builder.get_templatable_lang_contents(rule, + ssg.templates.LANGUAGES["oval"]) assert "%s" % (oval_def_id,) in oval_content From 3e0d2c0a0c8d0cb367920ec57058e2a57d825b88 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Thu, 24 Nov 2022 05:43:27 +0100 Subject: [PATCH 05/13] Refactor build_yaml Clean up imports and fix small problems. --- ssg/build_yaml.py | 11 +++++------ ssg/entities/common.py | 2 +- tests/unit/ssg-module/data/templates/extra_ovals.yml | 3 +++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 4b9e4abc430..c87ccf8444c 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -12,7 +12,7 @@ import ssg.build_remediations -from .build_cpe import CPEDoesNotExist, CPEALLogicalTest, CPEALFactRef, ProductCPEs +from .build_cpe import CPEALLogicalTest, CPEALFactRef, ProductCPEs from .constants import (XCCDF12_NS, OSCAP_BENCHMARK, OSCAP_GROUP, @@ -40,8 +40,7 @@ from .yaml import DocumentationNotComplete, open_and_macro_expand from .utils import required_key, mkdir_p -from .xml import ElementTree as ET, add_xhtml_namespace, register_namespaces, parse_file -from .shims import unicode_func +from .xml import ElementTree as ET, register_namespaces, parse_file import ssg.build_stig from .entities.common import add_sub_element, make_items_product_specific, \ @@ -280,7 +279,7 @@ def process_input_dict(cls, input_contents, env_yaml, product_cpes): return data def represent_as_dict(self): - data = super(Benchmark, cls).represent_as_dict() + data = super(Benchmark, self).represent_as_dict() data["rear-matter"] = data["rear_matter"] del data["rear_matter"] @@ -678,9 +677,9 @@ class Rule(XCCDFEntity, Templatable): inherited_cpe_platform_names=lambda: set(), bash_conditional=lambda: None, fixes=lambda: dict(), - ** XCCDFEntity.KEYS, - ** Templatable.KEYS + **XCCDFEntity.KEYS ) + KEYS.update(**Templatable.KEYS) MANDATORY_KEYS = { "title", diff --git a/ssg/entities/common.py b/ssg/entities/common.py index 58043c7c667..d7179ac4b95 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -399,7 +399,7 @@ def get_template_vars(self, env_yaml): # A better name is '_entity_id' (as in XCCDF Entity). template_vars["_rule_id"] = self.id_ - return make_items_product_specific(template_vars, env_yaml["product"]) + return make_items_product_specific(template_vars, env_yaml["product"], allow_overwrites=True) def get_template_backend_langs(self): """ diff --git a/tests/unit/ssg-module/data/templates/extra_ovals.yml b/tests/unit/ssg-module/data/templates/extra_ovals.yml index 78a223af8e4..c9b7e00725b 100644 --- a/tests/unit/ssg-module/data/templates/extra_ovals.yml +++ b/tests/unit/ssg-module/data/templates/extra_ovals.yml @@ -2,3 +2,6 @@ package_avahi_installed: name: package_installed vars: pkgname: avahi + pkgname@ubuntu2004: avahi-daemon + pkgname@rhel8: avahi8 + From b19366e65af554dff9cc341b8aff4dc39153318c Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 12:22:34 +0100 Subject: [PATCH 06/13] Refactor templates Better names for templatable-related methods of the Bulder class. --- ssg/build_sce.py | 4 ++-- ssg/templates.py | 14 +++++++------- tests/unit/ssg-module/test_templates.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ssg/build_sce.py b/ssg/build_sce.py index 5bfe437650c..22a38e486f4 100644 --- a/ssg/build_sce.py +++ b/ssg/build_sce.py @@ -166,8 +166,8 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output): # While we don't _write_ it, we still need to parse SCE # metadata from the templated content. Render it internally. - raw_sce_content = template_builder.get_templatable_lang_contents(rule, - langs['sce-bash']) + raw_sce_content = template_builder.get_lang_contents_for_templatable(rule, + langs['sce-bash']) ext = '.sh' filename = rule_id + ext diff --git a/ssg/templates.py b/ssg/templates.py index 16336b38922..9f8220c4aeb 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -178,7 +178,7 @@ def process_template_lang_file(self, template_name, template_vars, lang, local_e jinja_dict = ssg.utils.merge_dicts(env_yaml, template_parameters) return ssg.jinja.process_file_with_macros(template_file_path, jinja_dict) - def get_templatable_lang_contents(self, templatable, language): + def get_lang_contents_for_templatable(self, templatable, language): """ For the specified Templatable, build and return only the specified language content. """ @@ -193,19 +193,19 @@ def get_templatable_lang_contents(self, templatable, language): raise RuntimeError("Unable to generate {0} template language for Templatable {1}: {2}" .format(language.name, templatable, e)) - def write_templatable_lang_contents(self, filled_template, lang, templatable): + def write_lang_contents_for_templatable(self, filled_template, lang, templatable): output_file_name = templatable.id_ + lang.file_extension output_filepath = os.path.join(self.output_dirs[lang.name], output_file_name) with open(output_filepath, "w") as f: f.write(filled_template) - def build_templatable_lang(self, templatable, lang): + def build_lang_for_templatable(self, templatable, lang): """ Builds templated content of a given Templatable for a selected language, writing the output to the correct build directories. """ - filled_template = self.get_templatable_lang_contents(templatable, lang) - self.write_templatable_lang_contents(filled_template, lang, templatable) + filled_template = self.get_lang_contents_for_templatable(templatable, lang) + self.write_lang_contents_for_templatable(filled_template, lang, templatable) def build_rule(self, rule): """ @@ -214,7 +214,7 @@ def build_rule(self, rule): """ for lang in self.get_resolved_langs_to_generate(rule): if lang.name != "sce-bash": - self.build_templatable_lang(rule, lang) + self.build_lang_for_templatable(rule, lang) def build_extra_ovals(self): declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml") @@ -228,7 +228,7 @@ def build_extra_ovals(self): "title": oval_def_id, "template": template, }) - self.build_templatable_lang(rule, LANGUAGES["oval"]) + self.build_lang_for_templatable(rule, LANGUAGES["oval"]) def build_all_rules(self): for rule_file in sorted(os.listdir(self.resolved_rules_dir)): diff --git a/tests/unit/ssg-module/test_templates.py b/tests/unit/ssg-module/test_templates.py index 3c9c925d775..5d6ca842067 100644 --- a/tests/unit/ssg-module/test_templates.py +++ b/tests/unit/ssg-module/test_templates.py @@ -32,6 +32,6 @@ def test_render_extra_ovals(): "title": oval_def_id, "template": template, }) - oval_content = builder.get_templatable_lang_contents(rule, - ssg.templates.LANGUAGES["oval"]) + oval_content = builder.get_lang_contents_for_templatable(rule, + ssg.templates.LANGUAGES["oval"]) assert "%s" % (oval_def_id,) in oval_content From 3952bda15d49286b63c615bf28a72b65d6a221a3 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 12:25:30 +0100 Subject: [PATCH 07/13] Refactor common.Templatable Add/improve documentation. Rearrange funtions. Fix formatting. Remove dependency on templates.LANGUAGES from extract_configured_backend_lang method. This method would now take language dictionary from the caller and filter it based on the template configuration. --- ssg/entities/common.py | 50 +++++++++++++++++++++++++----------------- ssg/templates.py | 5 +---- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/ssg/entities/common.py b/ssg/entities/common.py index d7179ac4b95..b424f1c3ab2 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -352,6 +352,15 @@ def update_with(self, rhs): class Templatable(object): + """ + The Templatable is a mix-in sidekick for XCCDFEntity-based classes + that have templates. It contains methods used by the template Builder + class. + + Methods `get_template_context` and `get_template_vars` are subject for + overloading by XCCDFEntity subclasses that want to customize template + input. + """ KEYS = dict( template=lambda: None, @@ -363,6 +372,15 @@ def __init__(self): def is_templated(self): return isinstance(self.template, dict) + def get_template_name(self): + if not self.is_templated(): + return None + try: + return self.template["name"] + except KeyError: + raise ValueError( + "Templatable {0} is missing template name under template key".format(self)) + def get_template_context(self, env_yaml): # TODO: The first two variables, 'rule_id' and 'rule_title' are expected by some # templates and macros even if they are not rendered in a rule context. @@ -373,18 +391,6 @@ def get_template_context(self, env_yaml): "products": env_yaml["product"], } - def get_template_name(self): - """ - Given a template dictionary from a Rule instance, determine the name - of the template (from templates) this rule uses. - """ - try: - template_name = self.template["name"] - except KeyError: - raise ValueError( - "Templatable {0} is missing template name under template key".format(self)) - return template_name - def get_template_vars(self, env_yaml): if "vars" not in self.template: raise ValueError( @@ -399,22 +405,26 @@ def get_template_vars(self, env_yaml): # A better name is '_entity_id' (as in XCCDF Entity). template_vars["_rule_id"] = self.id_ - return make_items_product_specific(template_vars, env_yaml["product"], allow_overwrites=True) + return make_items_product_specific(template_vars, env_yaml["product"], + allow_overwrites=True) - def get_template_backend_langs(self): + def extract_configured_backend_lang(self, avail_langs): """ - Returns list of languages that should be generated from a template - configuration, controlled by backends. + Returns list of languages that should be generated + based on the Templatable's template option `template.backends`. """ - from ..templates import LANGUAGES + if not self.is_templated(): + return [] + if "backends" in self.template: backends = self.template["backends"] for lang in backends: - if lang not in LANGUAGES: + if lang not in avail_langs: raise RuntimeError("Templatable {0} wants to generate unknown language '{1}" .format(self, lang)) - return [lang for name, lang in LANGUAGES.items() if backends.get(name, "on") == "on"] - return LANGUAGES.values() + return [lang for name, lang in avail_langs.items() if backends.get(name, "on") == "on"] + + return avail_langs.values() def make_template_product_specific(self, product): if not self.is_templated(): diff --git a/ssg/templates.py b/ssg/templates.py index 9f8220c4aeb..5c34a3df0d2 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -150,16 +150,13 @@ def get_resolved_langs_to_generate(self, templatable): generated by the combination of the template supported_languages AND the Templatable's template configuration 'backends'. """ - if not templatable.is_templated(): - return [] - - rule_langs = set(templatable.get_template_backend_langs()) template_name = templatable.get_template_name() if template_name not in self.templates.keys(): raise ValueError( "Templatable {0} uses template {1} which does not exist." .format(templatable, template_name)) template_langs = set(self.templates[template_name].langs) + rule_langs = set(templatable.extract_configured_backend_lang(LANGUAGES)) return rule_langs.intersection(template_langs) def process_template_lang_file(self, template_name, template_vars, lang, local_env_yaml): From 35e075b017566a2fbad37f131127dadffb376538 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 12:31:17 +0100 Subject: [PATCH 08/13] Refactor templates Rearrange imports. Fix PEP8 formatting. Fix inconsistent argument name in Template class factory. --- ssg/entities/common.py | 2 +- ssg/templates.py | 17 ++++++++++------- tests/unit/ssg-module/test_templates.py | 7 ++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ssg/entities/common.py b/ssg/entities/common.py index b424f1c3ab2..c3394a2b223 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -7,11 +7,11 @@ from copy import deepcopy from ..xml import ElementTree as ET, add_xhtml_namespace -from ..constants import xhtml_namespace from ..yaml import DocumentationNotComplete, open_and_macro_expand from ..shims import unicode_func from ..constants import ( + xhtml_namespace, XCCDF_REFINABLE_PROPERTIES, XCCDF12_NS, OSCAP_VALUE, diff --git a/ssg/templates.py b/ssg/templates.py index 5c34a3df0d2..268c30cabc9 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -5,13 +5,14 @@ import imp import glob -import ssg.build_yaml +from collections import namedtuple + import ssg.utils import ssg.yaml import ssg.jinja -from ssg.build_cpe import ProductCPEs -from collections import namedtuple +import ssg.build_yaml +from ssg.build_cpe import ProductCPEs TemplatingLang = namedtuple( "templating_language_attributes", @@ -45,8 +46,8 @@ def __init__(self, templates_root_directory, name): self.preprocessing_file_path = os.path.join(self.template_path, PREPROCESSING_FILE_NAME) @classmethod - def load_template(cls, template_root_directory, name): - maybe_template = cls(template_root_directory, name) + def load_template(cls, templates_root_directory, name): + maybe_template = cls(templates_root_directory, name) if maybe_template._looks_like_template(): maybe_template._load() return maybe_template @@ -80,7 +81,8 @@ def preprocess(self, parameters, lang): def _preprocess_with_template_module(self, parameters, lang): if self.preprocessing_file_path is not None: unique_dummy_module_name = "template_" + self.name - preprocess_mod = imp.load_source(unique_dummy_module_name, self.preprocessing_file_path) + preprocess_mod = imp.load_source(unique_dummy_module_name, + self.preprocessing_file_path) if not hasattr(preprocess_mod, "preprocess"): msg = ( "The '{name}' template's preprocessing file {preprocessing_file} " @@ -185,7 +187,8 @@ def get_lang_contents_for_templatable(self, templatable, language): # Checks and remediations are processed with a custom YAML dict local_env_yaml = templatable.get_template_context(self.env_yaml) try: - return self.process_template_lang_file(template_name, template_vars, language, local_env_yaml) + return self.process_template_lang_file(template_name, template_vars, + language, local_env_yaml) except Exception as e: raise RuntimeError("Unable to generate {0} template language for Templatable {1}: {2}" .format(language.name, templatable, e)) diff --git a/tests/unit/ssg-module/test_templates.py b/tests/unit/ssg-module/test_templates.py index 5d6ca842067..723e749df1a 100644 --- a/tests/unit/ssg-module/test_templates.py +++ b/tests/unit/ssg-module/test_templates.py @@ -1,12 +1,13 @@ import os -import ssg.templates as tpl -from ssg.environment import open_environment import ssg.utils import ssg.products -from ssg.yaml import ordered_load import ssg.build_yaml import ssg.build_cpe +import ssg.templates as tpl + +from ssg.environment import open_environment +from ssg.yaml import ordered_load ssg_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) From 5c7cf430a1012bf3151872519f7873b99218f5cb Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 12:33:25 +0100 Subject: [PATCH 09/13] Refactor entities/common Remove redundant check in make_items_product_specific. If anything this will lower the cognitive complexity of the function. --- ssg/entities/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ssg/entities/common.py b/ssg/entities/common.py index c3394a2b223..27eb9e0a727 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -22,7 +22,7 @@ def make_items_product_specific(items_dict, product_suffix, allow_overwrites=False): new_items = dict() for full_label, value in items_dict.items(): - if "@" not in full_label and full_label not in new_items: + if "@" not in full_label: new_items[full_label] = value continue @@ -53,6 +53,7 @@ def make_items_product_specific(items_dict, product_suffix, allow_overwrites=Fal value_q=value, value_u=items_dict[label]) ) raise ValueError(msg) + new_items[label] = value return new_items From c0b411e509d2b1f96ccdcfcd13f0c1f6a8aa7664 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 12:35:46 +0100 Subject: [PATCH 10/13] Add max_line_length parameter to .editorconfig This parameter is an extension to the standard specification: https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties#supported-by-a-limited-number-of-editors The value 99 is in sync with sanity CI configuration for PEP8: setup.cfg: [pycodestyle] max-line-length = 99 --- .editorconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/.editorconfig b/.editorconfig index d076e156cfc..c104d5e006e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,6 +14,7 @@ indentation_guess = true indent_style = space indent_size = unset tab_width = 4 +max_line_length = 99 [{**.yml, **.yaml, **.jinja}] indent_style = space From cc7ecd79db008ad5b9f16fa21ac4569010e80a11 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 14:01:08 +0100 Subject: [PATCH 11/13] Refactor entities/common Extract shadowing / global refs check from make_items_product_specific. It supposed to lower the cognitive complexity of the function. --- ssg/entities/common.py | 51 ++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/ssg/entities/common.py b/ssg/entities/common.py index 27eb9e0a727..164051b46d5 100644 --- a/ssg/entities/common.py +++ b/ssg/entities/common.py @@ -19,6 +19,32 @@ ) +def extract_reference_from_product_specific_label(items_dict, full_label, value, allow_overwrites): + label = full_label.split("@")[0] + + if label in GLOBAL_REFERENCES: + msg = ( + "You cannot use product-qualified for the '{item_u}' reference. " + "Please remove the product-qualifier and merge values with the " + "existing reference if there is any. Original line: {item_q}: {value_q}" + .format(item_u=label, item_q=full_label, value_q=value) + ) + raise ValueError(msg) + + if not allow_overwrites and label in items_dict and value != items_dict[label]: + msg = ( + "There is a product-qualified '{item_q}' item, " + "but also an unqualified '{item_u}' item " + "and those two differ in value - " + "'{value_q}' vs '{value_u}' respectively." + .format(item_q=full_label, item_u=label, + value_q=value, value_u=items_dict[label]) + ) + raise ValueError(msg) + + return label + + def make_items_product_specific(items_dict, product_suffix, allow_overwrites=False): new_items = dict() for full_label, value in items_dict.items(): @@ -26,34 +52,15 @@ def make_items_product_specific(items_dict, product_suffix, allow_overwrites=Fal new_items[full_label] = value continue - label = full_label.split("@")[0] - - # This test should occur before matching product_suffix with the product qualifier + # This procedure should occur before matching product_suffix with the product qualifier # present in the reference, so it catches problems even for products that are not # being built at the moment - if label in GLOBAL_REFERENCES: - msg = ( - "You cannot use product-qualified for the '{item_u}' reference. " - "Please remove the product-qualifier and merge values with the " - "existing reference if there is any. Original line: {item_q}: {value_q}" - .format(item_u=label, item_q=full_label, value_q=value) - ) - raise ValueError(msg) + label = extract_reference_from_product_specific_label(items_dict, full_label, value, + allow_overwrites) if not full_label.endswith(product_suffix): continue - if label in items_dict and not allow_overwrites and value != items_dict[label]: - msg = ( - "There is a product-qualified '{item_q}' item, " - "but also an unqualified '{item_u}' item " - "and those two differ in value - " - "'{value_q}' vs '{value_u}' respectively." - .format(item_q=full_label, item_u=label, - value_q=value, value_u=items_dict[label]) - ) - raise ValueError(msg) - new_items[label] = value return new_items From 24be147746fb8b65c888c8fde220ee69225925bb Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 14:22:07 +0100 Subject: [PATCH 12/13] Fix line length in build_sce.py --- ssg/build_sce.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ssg/build_sce.py b/ssg/build_sce.py index 22a38e486f4..4b21da03dd6 100644 --- a/ssg/build_sce.py +++ b/ssg/build_sce.py @@ -166,8 +166,9 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output): # While we don't _write_ it, we still need to parse SCE # metadata from the templated content. Render it internally. - raw_sce_content = template_builder.get_lang_contents_for_templatable(rule, - langs['sce-bash']) + raw_sce_content = template_builder.get_lang_contents_for_templatable( + rule, langs['sce-bash'] + ) ext = '.sh' filename = rule_id + ext From 85e355354afdfd9b7a871c0b1e32d6ced2d89383 Mon Sep 17 00:00:00 2001 From: Evgeny Kolesnikov Date: Fri, 25 Nov 2022 15:38:00 +0100 Subject: [PATCH 13/13] Remove offending stigid from apple_os/auditing/service_com_apple_auditd_enabled rule The "stigid@ubuntu2004: UBTU-20-010182" entry is for Ubuntu and it violates not-shadowing policy. --- apple_os/auditing/service_com_apple_auditd_enabled/rule.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml b/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml index 5c10b6af9aa..bbb5132b5f0 100644 --- a/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml +++ b/apple_os/auditing/service_com_apple_auditd_enabled/rule.yml @@ -35,7 +35,6 @@ references: nist: AU-3,AU-3(1),AU-8(a),AU-8(b),AU-12(3),AU-14(1) srg: SRG-OS-000037-GPOS-00015,SRG-OS-000038-GPOS-00016,SRG-OS-000039-GPOS-00017,SRG-OS-000040-GPOS-00018,SRG-OS-000041-GPOS-00019,SRG-OS-000042-GPOS-00020,SRG-OS-000042-GPOS-00021,SRG-OS-000055-GPOS-00026,SRG-OS-000254-GPOS-00095,SRG-OS-000255-GPOS-00096,SRG-OS-000303-GPOS-00120,SRG-OS-000337-GPOS-00129,SRG-OS-000358-GPOS-00145,SRG-OS-000359-GPOS-00146 stigid: AOSX-14-001013 - stigid@ubuntu2004: UBTU-20-010182 ocil_clause: 'auditing is not enabled or running'