From 19d5c50402ac4bf2bc9a066a289a4f022d2ce02d Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Mon, 12 Dec 2022 12:43:16 +0100 Subject: [PATCH 1/9] modify the function which resolves and adds new CPE items based on parameters passed to platform up to now the creation of new CPE items was unconditional it means that even if resolved CPE items already existed, they got rewritten whenever the platform was being loaded. This is not desired - each CPE item should get resolved just once. --- ssg/build_cpe.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ssg/build_cpe.py b/ssg/build_cpe.py index 152828dcf48..49df2e518c8 100644 --- a/ssg/build_cpe.py +++ b/ssg/build_cpe.py @@ -93,11 +93,20 @@ def get_cpe(self, cpe_id_or_name): def add_resolved_cpe_items_from_platform(self, platform): for fact_ref in platform.get_fact_refs(): - if fact_ref.arg: - cpe = self.get_cpe(fact_ref.cpe_name) - new_cpe = cpe.create_resolved_cpe_item_for_fact_ref(fact_ref) - self.add_cpe_item(new_cpe) - fact_ref.cpe_name = new_cpe.name + if fact_ref.arg: # the CPE item is parametrized + try: + # if there already exists a CPE item with factref's ID + # we can just use it right away, no new CPE items need to be created + cpe = self.get_cpe_for_fact_ref(fact_ref) + fact_ref.cpe_name = cpe.name + except CPEDoesNotExist: + # if the CPE item with factref's ID does not exist + # it means that we need to create a new CPE item + # which will have parameters in place + cpe = self.get_cpe(fact_ref.cpe_name) + new_cpe = cpe.create_resolved_cpe_item_for_fact_ref(fact_ref) + self.add_cpe_item(new_cpe) + fact_ref.cpe_name = new_cpe.name def get_cpe_for_fact_ref(self, fact_ref): return self.get_cpe(fact_ref.as_id()) From 946afe1ace0459a23a9819773732a27f1db09d13 Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Mon, 12 Dec 2022 12:53:13 +0100 Subject: [PATCH 2/9] add a new function to platforms which collects conditionals from associated CPE items This conditional is later used when creating remediations. --- ssg/build_yaml.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 7830ebd3109..0c3d1de107d 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -1521,8 +1521,8 @@ def from_text(cls, expression, product_cpes): platform.name = id_ platform.original_expression = expression platform.xml_content = platform.get_xml() - platform.bash_conditional = platform.test.to_bash_conditional() - platform.ansible_conditional = platform.test.to_ansible_conditional() + platform.update_conditional_from_cpe_items("bash", product_cpes) + platform.update_conditional_from_cpe_items("ansible", product_cpes) return platform def get_xml(self): @@ -1566,6 +1566,16 @@ def from_yaml(cls, yaml_file, env_yaml=None, product_cpes=None): def get_fact_refs(self): return self.test.get_symbols() + def update_conditional_from_cpe_items(self, language, product_cpes): + self.test.enrich_with_cpe_info(product_cpes) + if language == "bash": + self.bash_conditional = self.test.to_bash_conditional() + elif language == "ansible": + self.ansible_conditional = self.test.to_ansible_conditional() + else: + raise RuntimeError( + "Platform remediations do not support the {0} language".format(language)) + def __eq__(self, other): if not isinstance(other, Platform): return False From ba9bb5e116547097fc96f798543d3fedfa7ee514 Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Mon, 12 Dec 2022 12:55:53 +0100 Subject: [PATCH 3/9] always store the xml representation of platform as string previously, the representation was sometimes stored as string, sometimes as XML ElementTree. Platforms need to be saved and loaded as needed and so this needs to be unified. --- ssg/build_yaml.py | 3 +-- tests/unit/ssg-module/test_build_yaml.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ssg/build_yaml.py b/ssg/build_yaml.py index 0c3d1de107d..375ba02ebe5 100644 --- a/ssg/build_yaml.py +++ b/ssg/build_yaml.py @@ -1542,7 +1542,7 @@ def get_xml(self): return xmlstr def to_xml_element(self): - return self.xml_content + return ET.fromstring(self.xml_content) def get_remediation_conditional(self, language): if language == "bash": @@ -1555,7 +1555,6 @@ def get_remediation_conditional(self, language): @classmethod def from_yaml(cls, yaml_file, env_yaml=None, product_cpes=None): platform = super(Platform, cls).from_yaml(yaml_file, env_yaml) - platform.xml_content = ET.fromstring(platform.xml_content) # If we received a product_cpes, we can restore also the original test object # it can be later used e.g. for comparison if product_cpes: diff --git a/tests/unit/ssg-module/test_build_yaml.py b/tests/unit/ssg-module/test_build_yaml.py index b99f0d410f9..12af3c1ee68 100644 --- a/tests/unit/ssg-module/test_build_yaml.py +++ b/tests/unit/ssg-module/test_build_yaml.py @@ -247,7 +247,7 @@ def test_platform_from_text_simple(product_cpes): "ansible_virtualization_type not in [\"docker\", \"lxc\", \"openvz\", \"podman\", \"container\"]" assert platform.get_remediation_conditional("bash") == \ "[ ! -f /.dockerenv ] && [ ! -f /run/.containerenv ]" - platform_el = ET.fromstring(platform.to_xml_element()) + platform_el = platform.to_xml_element() assert platform_el.tag == "{%s}platform" % cpe_language_namespace assert platform_el.get("id") == "machine" logical_tests = platform_el.findall( @@ -265,7 +265,7 @@ def test_platform_from_text_simple_product_cpe(product_cpes): platform = ssg.build_yaml.Platform.from_text("rhel7-workstation", product_cpes) assert platform.get_remediation_conditional("bash") == "" assert platform.get_remediation_conditional("ansible") == "" - platform_el = ET.fromstring(platform.to_xml_element()) + platform_el = platform.to_xml_element() assert platform_el.tag == "{%s}platform" % cpe_language_namespace assert platform_el.get("id") == "rhel7-workstation" logical_tests = platform_el.findall( @@ -285,7 +285,7 @@ def test_platform_from_text_or(product_cpes): assert platform.get_remediation_conditional("bash") == "( rpm --quiet -q chrony || rpm --quiet -q ntp )" assert platform.get_remediation_conditional("ansible") == \ "( \"chrony\" in ansible_facts.packages or \"ntp\" in ansible_facts.packages )" - platform_el = ET.fromstring(platform.to_xml_element()) + platform_el = platform.to_xml_element() assert platform_el.tag == "{%s}platform" % cpe_language_namespace assert platform_el.get("id") == "chrony_or_ntp" logical_tests = platform_el.findall( @@ -312,7 +312,7 @@ def test_platform_from_text_complex_expression(product_cpes): "systemd and !yum and (ntp or chrony)", product_cpes) assert platform.get_remediation_conditional("bash") == "( rpm --quiet -q systemd && ( rpm --quiet -q chrony || rpm --quiet -q ntp ) && ! ( rpm --quiet -q yum ) )" assert platform.get_remediation_conditional("ansible") == "( \"systemd\" in ansible_facts.packages and ( \"chrony\" in ansible_facts.packages or \"ntp\" in ansible_facts.packages ) and not ( \"yum\" in ansible_facts.packages ) )" - platform_el = ET.fromstring(platform.to_xml_element()) + platform_el = platform.to_xml_element() assert platform_el.tag == "{%s}platform" % cpe_language_namespace assert platform_el.get("id") == "systemd_and_chrony_or_ntp_and_not_yum" logical_tests = platform_el.findall( From 476541b148e3ad6c86b1e08ce8cc0ba2ae1b80d7 Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Wed, 14 Dec 2022 12:43:29 +0100 Subject: [PATCH 4/9] add conditional setter to CPEItem --- ssg/build_cpe.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ssg/build_cpe.py b/ssg/build_cpe.py index 49df2e518c8..4da730050d7 100644 --- a/ssg/build_cpe.py +++ b/ssg/build_cpe.py @@ -226,6 +226,15 @@ def create_resolved_cpe_item_for_fact_ref(self, fact_ref): def is_cpe_name(cpe_id_or_name): return cpe_id_or_name.startswith("cpe:") + def set_conditional(self, language, content): + if language == "ansible": + self.ansible_conditional = content + elif language == "bash": + self.bash_conditional = content + else: + raise RuntimeError( + "The language {0} is not supported as conditional for CPE".format(language)) + class CPEALLogicalTest(Function): From be96b55bdc658fb5edb2b29bc3d5147c0a55970b Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Wed, 14 Dec 2022 13:11:36 +0100 Subject: [PATCH 5/9] change meaning of function build_lang_for_templatable originally, the function did also the file writing but I need to split it so that I can better influence what to do with the result of templating. In case of platforms and CPE items, I do not want the filled template to be written into a file, I want to incorporate it into the CPE item / template and save them to disk as needed. This commit changes tthe function and adapt templating of rules to the new API. Templating of platforms and CPEs will be handled in separate commit. --- ssg/templates.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ssg/templates.py b/ssg/templates.py index 7e2cc63a136..97821113137 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -201,11 +201,10 @@ def write_lang_contents_for_templatable(self, filled_template, lang, templatable def build_lang_for_templatable(self, templatable, lang): """ - Builds templated content of a given Templatable for a selected language, - writing the output to the correct build directories. + Builds templated content of a given Templatable for a selected language + returning the filled template. """ - filled_template = self.get_lang_contents_for_templatable(templatable, lang) - self.write_lang_contents_for_templatable(filled_template, lang, templatable) + return self.get_lang_contents_for_templatable(templatable, lang) def build_cpe(self, cpe): for lang in self.get_resolved_langs_to_generate(cpe): @@ -228,7 +227,8 @@ def build_rule(self, rule): """ for lang in self.get_resolved_langs_to_generate(rule): if lang.name != "sce-bash": - self.build_lang_for_templatable(rule, lang) + filled_template = self.build_lang_for_templatable(rule, lang) + self.write_lang_contents_for_templatable(filled_template, lang, rule) def build_extra_ovals(self): declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml") @@ -242,7 +242,8 @@ def build_extra_ovals(self): "title": oval_def_id, "template": template, }) - self.build_lang_for_templatable(rule, LANGUAGES["oval"]) + filled_template = self.build_lang_for_templatable(rule, LANGUAGES["oval"]) + self.write_lang_contents_for_templatable(filled_template, LANGUAGES["oval"], rule) def build_all_platforms(self): for platform_file in sorted(os.listdir(self.platforms_dir)): From a74bf6f503cb1e842f279642ff845362f5a3982e Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Wed, 14 Dec 2022 13:15:28 +0100 Subject: [PATCH 6/9] save templated remediations into CPE items make sure that as soon as the template is filled, it is inserted into the CPE item at runtime and also the CPE item is stored to disk. --- ssg/templates.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ssg/templates.py b/ssg/templates.py index 97821113137..d20dc4b68f6 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -208,7 +208,14 @@ def build_lang_for_templatable(self, templatable, lang): def build_cpe(self, cpe): for lang in self.get_resolved_langs_to_generate(cpe): - self.build_lang_for_templatable(cpe, lang) + filled_template = self.build_lang_for_templatable(cpe, lang) + if lang.template_type == TemplateType.REMEDIATION: + cpe.set_conditional(lang.name, filled_template) + if lang.template_type == TemplateType.CHECK: + self.write_lang_contents_for_templatable(filled_template, lang, cpe) + self.product_cpes.add_cpe_item(cpe) + cpe_path = os.path.join(self.cpe_items_dir, cpe.id_+".yml") + cpe.dump_yaml(cpe_path) def build_platform(self, platform): """ From 2c6791714fef56b27692967edd571bb3c49a4f08 Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Wed, 14 Dec 2022 13:16:33 +0100 Subject: [PATCH 7/9] ensure that platforms update their conditionals after respective CPE items update their templated conditionals This ensures that changes to conditionals of templated CPE items propagate to platforms. Platforms are also stored to disk because they are loaded in later build stages. --- ssg/templates.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ssg/templates.py b/ssg/templates.py index d20dc4b68f6..00f9b303fb1 100644 --- a/ssg/templates.py +++ b/ssg/templates.py @@ -220,12 +220,21 @@ def build_cpe(self, cpe): def build_platform(self, platform): """ Builds templated content of a given Platform (all CPEs/Symbols) for all available - languages, writing the output to the correct build directories. + languages, writing the output to the correct build directories + and updating the platform it self. """ + langs_affecting_this_platform = set() for fact_ref in platform.test.get_symbols(): cpe = self.product_cpes.get_cpe_for_fact_ref(fact_ref) if cpe.is_templated(): self.build_cpe(cpe) + langs_affecting_this_platform.update( + self.get_resolved_langs_to_generate(cpe)) + for lang in langs_affecting_this_platform: + if lang.template_type == TemplateType.REMEDIATION: + platform.update_conditional_from_cpe_items(lang.name, self.product_cpes) + platform_path = os.path.join(self.platforms_dir, platform.id_+".yml") + platform.dump_yaml(platform_path) def build_rule(self, rule): """ From 7cdf7bd094f5cf5de58ae182a8c0f2f41790b12b Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Mon, 12 Dec 2022 13:02:27 +0100 Subject: [PATCH 8/9] add new parametrized platform os_linux add associated template together with Ansible remediation. Currently, only the OS name can be specified. --- shared/applicability/os_linux.yml | 14 +++++++++++ .../platform_os_linux/ansible.template | 2 ++ .../templates/platform_os_linux/oval.template | 25 +++++++++++++++++++ .../templates/platform_os_linux/template.yml | 3 +++ 4 files changed, 44 insertions(+) create mode 100644 shared/applicability/os_linux.yml create mode 100644 shared/templates/platform_os_linux/ansible.template create mode 100644 shared/templates/platform_os_linux/oval.template create mode 100644 shared/templates/platform_os_linux/template.yml diff --git a/shared/applicability/os_linux.yml b/shared/applicability/os_linux.yml new file mode 100644 index 00000000000..c60f3e33329 --- /dev/null +++ b/shared/applicability/os_linux.yml @@ -0,0 +1,14 @@ +name: "cpe:/o:{arg}" +title: "Operating System is {arg}" +check_id: platform_os_linux_{arg} +template: + name: platform_os_linux +args: + rhel: + os_name: "Red Hat Enterprise Linux" + os_id: 'rhel' + os_id_ansible: "RedHat" + fedora: + os_name: "Fedora" + os_id: 'fedora' + os_id_ansible: "Fedora" diff --git a/shared/templates/platform_os_linux/ansible.template b/shared/templates/platform_os_linux/ansible.template new file mode 100644 index 00000000000..265f2792d63 --- /dev/null +++ b/shared/templates/platform_os_linux/ansible.template @@ -0,0 +1,2 @@ +{{%- set ansible_os_release_name_cond = 'ansible_distribution == "' + OS_ID_ANSIBLE + '"' -%}} + {{{ ansible_os_release_name_cond.strip() }}} diff --git a/shared/templates/platform_os_linux/oval.template b/shared/templates/platform_os_linux/oval.template new file mode 100644 index 00000000000..4723f60620b --- /dev/null +++ b/shared/templates/platform_os_linux/oval.template @@ -0,0 +1,25 @@ + + + {{{ oval_metadata("The installed operating system is " + OS_NAME, affected_platforms=["multi_platform_all"]) }}} + + + + + + + + + + + + /etc/os-release + ^ID=["']?(\w+)["']?$ + 1 + + + + {{{ OS_ID }}} + + + diff --git a/shared/templates/platform_os_linux/template.yml b/shared/templates/platform_os_linux/template.yml new file mode 100644 index 00000000000..0e9fd8e65a3 --- /dev/null +++ b/shared/templates/platform_os_linux/template.yml @@ -0,0 +1,3 @@ +supported_languages: + - ansible + - oval From 3469d5e8ab6e268cb6e6b0816c9cec17c6b7e445 Mon Sep 17 00:00:00 2001 From: Vojtech Polasek Date: Fri, 16 Dec 2022 10:14:08 +0100 Subject: [PATCH 9/9] make the os_linux unversioned (so far) result of rebase against master which brings support for versions --- shared/applicability/os_linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/applicability/os_linux.yml b/shared/applicability/os_linux.yml index c60f3e33329..819a29fd58c 100644 --- a/shared/applicability/os_linux.yml +++ b/shared/applicability/os_linux.yml @@ -1,6 +1,7 @@ name: "cpe:/o:{arg}" title: "Operating System is {arg}" check_id: platform_os_linux_{arg} +versioned: false template: name: platform_os_linux args: