ComplianceAsCode · jan-cerny · Jan 17, 2023 · Jan 2, 2023 · Jan 2, 2023 · Jan 2, 2023
diff --git a/.github/workflows/gate.yaml b/.github/workflows/gate.yaml
@@ -83,6 +83,8 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Build
+        env:
+          ADDITIONAL_CMAKE_OPTIONS: "-DSSG_SCE_ENABLED:BOOL=ON"
         run: |-
           ./build_product \
               alinux2 \

diff --git a/build-scripts/build_sce.py b/build-scripts/build_sce.py
@@ -73,6 +73,6 @@ def parse_args():
         args.build_config_yaml, args.product_yaml)
     empty = "/sce/empty/placeholder"
     template_builder = ssg.templates.Builder(
-        env_yaml, empty, args.templates_dir, empty, empty)
+        env_yaml, empty, args.templates_dir, empty, empty, empty, None)
     ssg.build_sce.checks(env_yaml, args.product_yaml, args.scedirs,
                          template_builder, args.output)
diff --git a/build-scripts/compose_ds.py b/build-scripts/compose_ds.py
@@ -6,9 +6,10 @@
 import time
 import xml.etree.ElementTree as ET
 
+from ssg.build_sce import collect_sce_checks
 from ssg.constants import (
-    cat_namespace, datastream_namespace, oval_namespace, XCCDF12_NS,
-    xlink_namespace)
+    cat_namespace, datastream_namespace, oval_namespace, sce_namespace,
+    XCCDF12_NS, xlink_namespace)
 import ssg.xml
 
 try:
@@ -29,6 +30,48 @@ def mangle_path(path):
     return path
 
 
+# From the list generated by collect_sce_checks, extract the path to the check content,
+# and embed the script into the datastream
+def embed_sce_checks_in_datastream(datastreamtree, checklists, sce_files, refdir):
+    for file in sce_files:
+        path = os.path.join(refdir, file)
+        mangled_path = mangle_path(file)
+
+        with open(path, 'rt', encoding='utf8') as fd:
+            sce_script_content = fd.read()
+
+        component_id = "scap_{}_ecomp_{}".format(ID_NS, mangled_path)
+        component = ET.SubElement(
+            datastreamtree, '{%s}extended-component' % datastream_namespace,
+            attrib={
+                'id': component_id,
+                'timestamp': get_timestamp(path)
+            })
+        # Append the file content
+        script_data = ET.SubElement(component, '{%s}script' % sce_namespace)
+        script_data.text = sce_script_content
+
+        # Create a component reference to map the checklist to the extended component
+        component_ref_id = "scap_{}_cref_{}".format(ID_NS, mangled_path)
+        component_ref = ET.SubElement(
+            checklists, '{%s}component-ref' % datastream_namespace,
+            attrib={
+                'id': component_ref_id,
+                ('{%s}href' % xlink_namespace): '#' + component_id
+            })
+
+        # Add the component reference to the catalog of XCCDF checklists
+        checklists_component_ref = checklists.find(
+            "{%s}component-ref" % datastream_namespace)
+        catalog = checklists_component_ref.find('{%s}catalog' % cat_namespace)
+        uri = ET.SubElement(
+            catalog, '{%s}uri' % cat_namespace,
+            attrib={
+                'name': file,
+                'uri': '#' + component_ref_id
+            })
+
+
 def move_patches_up_to_date_to_source_data_stream_component(datastreamtree):
     ds_checklists = datastreamtree.find(
         ".//{%s}checklists" % datastream_namespace)
@@ -130,6 +173,7 @@ def parse_args():
     parser.add_argument("--ocil", help="OCIL file name")
     parser.add_argument("--cpe-dict", help="CPE dictionary file name")
     parser.add_argument("--cpe-oval", help="CPE OVAL file name")
+    parser.add_argument("--enable-sce", action='store_true', help="Enable building sce data")
     parser.add_argument(
         "--output-12", help="Output SCAP 1.2 source data stream file name")
     parser.add_argument(
@@ -180,7 +224,7 @@ def create_catalog(component_ref, dependencies):
 
 def compose_ds(
         xccdf_file_name, oval_file_name, ocil_file_name,
-        cpe_dict_file_name, cpe_oval_file_name):
+        cpe_dict_file_name, cpe_oval_file_name, sce_enabled):
     ds_collection = ET.Element(
         "{%s}data-stream-collection" % datastream_namespace)
     name = "from_xccdf_" + os.path.basename(xccdf_file_name)
@@ -202,6 +246,10 @@ def compose_ds(
     add_component(ds_collection, checks, oval_file_name)
     add_component(ds_collection, checks, ocil_file_name)
     add_component(ds_collection, checks, cpe_oval_file_name)
+    if sce_enabled:
+        sce_check_files = collect_sce_checks(ds_collection)
+        refdir = os.path.dirname(oval_file_name)
+        embed_sce_checks_in_datastream(ds_collection, checklists, sce_check_files, refdir)
     return ET.ElementTree(ds_collection)
 
 
@@ -210,6 +258,7 @@ def upgrade_ds_to_scap_13(ds):
     dsc_el.set("schematron-version", "1.3")
     ds_el = ds.find("{%s}data-stream" % datastream_namespace)
     ds_el.set("scap-version", '1.3')
+
     # Move reference to remote OVAL content to a source data stream component
     move_patches_up_to_date_to_source_data_stream_component(ds)
     return ds
@@ -219,7 +268,7 @@ def upgrade_ds_to_scap_13(ds):
     args = parse_args()
     ssg.xml.register_namespaces()
     ds = compose_ds(
-        args.xccdf, args.oval, args.ocil, args.cpe_dict, args.cpe_oval)
+        args.xccdf, args.oval, args.ocil, args.cpe_dict, args.cpe_oval, args.enable_sce)
     if args.output_12:
         ds.write(args.output_12)
     ds_13 = upgrade_ds_to_scap_13(ds)

diff --git a/cmake/SSGCommon.cmake b/cmake/SSGCommon.cmake
@@ -98,15 +98,15 @@ macro(ssg_build_xccdf_oval_ocil PRODUCT)
         add_custom_command(
             OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/profiles"
             COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/profiles"
-            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_all.py" --resolved-base "${CMAKE_CURRENT_BINARY_DIR}" --controls-dir "${CMAKE_SOURCE_DIR}/controls" --build-config-yaml "${CMAKE_BINARY_DIR}/build_config.yml" --product-yaml "${CMAKE_CURRENT_SOURCE_DIR}/product.yml" --sce-metadata "${CMAKE_CURRENT_BINARY_DIR}/checks_from_templates/sce/metadata.json"
+            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_all.py" --resolved-base "${CMAKE_CURRENT_BINARY_DIR}" --controls-dir "${CMAKE_SOURCE_DIR}/controls" --build-config-yaml "${CMAKE_BINARY_DIR}/build_config.yml" --product-yaml "${CMAKE_CURRENT_SOURCE_DIR}/product.yml" --sce-metadata "${CMAKE_CURRENT_BINARY_DIR}/checks/sce/metadata.json"
             DEPENDS generate-internal-${PRODUCT}-sce-metadata.json
             COMMENT "[${PRODUCT}-content] compiling everything"
         )
     else()
         add_custom_command(
             OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/profiles"
             COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/profiles"
-            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_all.py" --resolved-base "${CMAKE_CURRENT_BINARY_DIR}" --controls-dir "${CMAKE_SOURCE_DIR}/controls" --build-config-yaml "${CMAKE_BINARY_DIR}/build_config.yml" --product-yaml "${CMAKE_CURRENT_SOURCE_DIR}/product.yml" --sce-metadata "${CMAKE_CURRENT_BINARY_DIR}/checks_from_templates/sce/metadata.json" --stig-references "${STIG_REFERENCE_FILE}"
+            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_all.py" --resolved-base "${CMAKE_CURRENT_BINARY_DIR}" --controls-dir "${CMAKE_SOURCE_DIR}/controls" --build-config-yaml "${CMAKE_BINARY_DIR}/build_config.yml" --product-yaml "${CMAKE_CURRENT_SOURCE_DIR}/product.yml" --sce-metadata "${CMAKE_CURRENT_BINARY_DIR}/checks/sce/metadata.json" --stig-references "${STIG_REFERENCE_FILE}"
             DEPENDS generate-internal-${PRODUCT}-sce-metadata.json
             COMMENT "[${PRODUCT}-content] compiling everything"
         )
@@ -278,7 +278,7 @@ endmacro()
 # (without needing a separate XML or XSLT linking step) and also place
 # <complex-check /> elements as necessary.
 macro(ssg_build_sce PRODUCT)
-    set(BUILD_CHECKS_DIR "${CMAKE_CURRENT_BINARY_DIR}/checks_from_templates")
+    set(BUILD_CHECKS_DIR "${CMAKE_CURRENT_BINARY_DIR}/checks")
     # Unlike build_oval_unlinked, here we're ignoring the existing checks from
     # templates and other places and we're merely appending/templating the
     # content from the rules directories. That's why we ignore BUILD_CHECKS_DIR
@@ -311,7 +311,7 @@ macro(ssg_build_sce PRODUCT)
     endif()
     add_custom_target(
         generate-internal-${PRODUCT}-sce-metadata.json
-        DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/checks_from_templates/sce/metadata.json"
+        DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/checks/sce/metadata.json"
     )
 endmacro()
 
@@ -410,11 +410,17 @@ endmacro()
 # evaluation using e.g., OpenSCAP) by combining XCCDF, OVAL, SCE, and OCIL
 # content. This relies heavily on the OpenSCAP executable here.
 macro(ssg_build_sds PRODUCT)
+    if(SSG_SCE_ENABLED)
+        set(COMPOSE_EXTRA_ARGS "--enable-sce")
+    else()
+        set(COMPOSE_EXTRA_ARGS "")
+    endif()
+
     if(SSG_BUILD_SCAP_12_DS)
         add_custom_command(
             OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds-1.2.xml"
             OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
-            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compose_ds.py" --xccdf "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" --oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" --ocil "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml" --cpe-dict "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" --cpe-oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" --output-12 "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds-1.2.xml" --output-13 "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
+	    COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compose_ds.py" --xccdf "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" --oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" --ocil "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml" --cpe-dict "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" --cpe-oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" --output-12 "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds-1.2.xml" --output-13 "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" ${COMPOSE_EXTRA_ARGS}
             COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds-1.2.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds-1.2.xml"
             COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
             DEPENDS generate-ssg-${PRODUCT}-xccdf.xml
@@ -431,7 +437,7 @@ macro(ssg_build_sds PRODUCT)
     else()
         add_custom_command(
             OUTPUT "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
-            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compose_ds.py" --xccdf "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" --oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" --ocil "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml" --cpe-dict "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" --cpe-oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" --output-13 "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
+            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compose_ds.py" --xccdf "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-xccdf.xml" --oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-oval.xml" --ocil "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ocil.xml" --cpe-dict "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-dictionary.xml" --cpe-oval "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-cpe-oval.xml" --output-13 "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" ${COMPOSE_EXTRA_ARGS}
             COMMAND "${XMLLINT_EXECUTABLE}" --nsclean --format --output "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
             DEPENDS generate-ssg-${PRODUCT}-xccdf.xml
             DEPENDS generate-ssg-${PRODUCT}-oval.xml
@@ -514,6 +520,13 @@ macro(ssg_build_sds PRODUCT)
                 COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${CMAKE_SOURCE_DIR}/tests/missing_refs.sh" "${PYTHON_EXECUTABLE}" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml" ${REFERENCES_CHECK_PROFILE_LIST}
         )
         set_tests_properties("missing-references-ssg-${PRODUCT}-ds.xml" PROPERTIES LABELS quick)
+
+    endif()
+    if("${PRODUCT}" MATCHES "ubuntu2004" AND SSG_SCE_ENABLED)
+        add_test(
+            NAME "ds-sce-${PRODUCT}"
+            COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/tests/test_ds_sce.py" "${CMAKE_BINARY_DIR}" "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
+        )
     endif()
 endmacro()
 
@@ -758,11 +771,6 @@ macro(ssg_build_product PRODUCT)
             DESTINATION "${SSG_CONTENT_INSTALL_DIR}")
     endif()
 
-    if (SSG_SCE_ENABLED)
-        install(DIRECTORY "${CMAKE_BINARY_DIR}/${PRODUCT}/checks/sce/"
-            DESTINATION "${SSG_CONTENT_INSTALL_DIR}/${PRODUCT}/checks/sce")
-    endif()
-
     install(FILES "${CMAKE_BINARY_DIR}/ssg-${PRODUCT}-ds.xml"
         DESTINATION "${SSG_CONTENT_INSTALL_DIR}")
 

diff --git a/ssg/build_sce.py b/ssg/build_sce.py
@@ -7,7 +7,10 @@
 import sys
 
 from .build_yaml import Rule, DocumentationNotComplete
-from .constants import MULTI_PLATFORM_LIST
+from .constants import (
+    MULTI_PLATFORM_LIST, OSCAP_VALUE, datastream_namespace,
+    xlink_namespace, XCCDF12_NS, SCE_SYSTEM
+)
 from .jinja import process_file_with_macros
 from .rule_yaml import parse_prodtype
 from .rules import get_rule_dir_id, get_rule_dir_sces, find_rule_dirs_in_paths
@@ -32,22 +35,34 @@ def load_sce_and_metadata_parsed(raw_content):
     metadata = dict()
     sce_content = []
 
+    keywords = ['platform', 'check-import', 'check-export', 'complex-check']
     for line in raw_content.split("\n"):
         found_metadata = False
-        keywords = ['platform', 'check-import', 'check-export', 'complex-check']
         for keyword in keywords:
-            if line.startswith('# ' + keyword + ' = '):
-                # Strip off the initial comment marker
-                _, value = line[2:].split('=', maxsplit=1)
-                values = value.strip()
-                if ',' in values:
-                    values.split(',')
-                metadata[keyword] = values
-                found_metadata = True
-                break
+            if not line.startswith('# ' + keyword + ' = '):
+                continue
+
+            found_metadata = True
+
+            # Strip off the initial comment marker
+            _, value = line[2:].split('=', maxsplit=1)
+            metadata[keyword] = value.strip()
+
         if not found_metadata:
             sce_content.append(line)
 
+    if 'check-export' in metadata:
+        # Special case for the variables exposed to the SCE script: prepend
+        # the OSCAP_VALUE prefix to reference the variable
+        new_variables = []
+        for value in metadata['check-export'].split(','):
+            k, v = value.split('=', maxsplit=1)
+            new_variables.append(k+'='+OSCAP_VALUE+v)
+        metadata['check-export'] = new_variables
+
+    if 'platform' in metadata:
+        metadata['platform'] = metadata['platform'].split(',')
+
     return "\n".join(sce_content), metadata
 
 
@@ -219,3 +234,29 @@ def checks(env_yaml, yaml_path, sce_dirs, template_builder, output):
     json.dump(already_loaded, open(metadata_path, 'w'))
 
     return already_loaded
+
+
+# Retrieve the SCE checks and return a list of path to each check script.
+def collect_sce_checks(datastreamtree):
+    checklists = datastreamtree.find(
+        ".//{%s}checklists" % datastream_namespace)
+    checklists_component_ref = checklists.find(
+        "{%s}component-ref" % datastream_namespace)
+    # The component ID is the component-ref href without leading '#'
+    checklist_component_id = checklists_component_ref.get('{%s}href' % xlink_namespace)[1:]
+
+    checks_xpath = str.format(
+        ".//{{{ds_ns}}}component[@id='{cid}']/"
+        "{{{xccdf_ns}}}Benchmark//"
+        "{{{xccdf_ns}}}Rule/"
+        "{{{xccdf_ns}}}check[@system='{sce_sys}']/"
+        "{{{xccdf_ns}}}check-content-ref",
+        ds_ns=datastream_namespace,
+        xccdf_ns=XCCDF12_NS,
+        cid=checklist_component_id,
+        sce_sys=SCE_SYSTEM
+    )
+
+    checks = datastreamtree.findall(checks_xpath)
+    # Extract the file paths of the SCE checks
+    return [check.get('href') for check in checks]
diff --git a/ssg/constants.py b/ssg/constants.py
@@ -75,6 +75,7 @@
 xhtml_namespace = "http://www.w3.org/1999/xhtml"
 xsi_namespace = "http://www.w3.org/2001/XMLSchema-instance"
 cat_namespace = "urn:oasis:names:tc:entity:xmlns:xml:catalog"
+sce_namespace = "http://open-scap.org/page/SCE_xccdf_stream"
 ocil_cs = "http://scap.nist.gov/schema/ocil/2"
 xccdf_header = xml_version + "<xccdf>"
 xccdf_footer = "</xccdf>"
@@ -140,6 +141,7 @@
     "cpe-dict": "http://cpe.mitre.org/dictionary/2.0",
     "cat": cat_namespace,
     "cpe-lang": "http://cpe.mitre.org/language/2.0",
+    "sce": sce_namespace,
 }
 
 FIX_TYPE_TO_SYSTEM = {

diff --git a/ssg/templates.py b/ssg/templates.py
@@ -29,7 +29,7 @@
     "kubernetes": TemplatingLang("kubernetes", ".yml",  TemplateType.REMEDIATION, "kubernetes"),
     "oval": TemplatingLang("oval", ".xml",              TemplateType.CHECK,       "oval"),
     "puppet": TemplatingLang("puppet", ".pp",           TemplateType.REMEDIATION, "puppet"),
-    "sce-bash": TemplatingLang("sce-bash", ".sh",       TemplateType.REMEDIATION, "sce")
+    "sce-bash": TemplatingLang("sce-bash", ".sh",       TemplateType.CHECK,       "sce")
 }
 
 PREPROCESSING_FILE_NAME = "template.py"
@@ -128,7 +128,8 @@ def __init__(self, env_yaml, resolved_rules_dir, templates_dir,
         self._init_lang_output_dirs()
         self._init_and_load_templates()
         self.product_cpes = ProductCPEs()
-        self.product_cpes.load_cpes_from_directory_tree(cpe_items_dir, self.env_yaml)
+        if cpe_items_dir is not None:
+            self.product_cpes.load_cpes_from_directory_tree(cpe_items_dir, self.env_yaml)
 
     def _init_and_load_templates(self):
         for item in sorted(os.listdir(self.templates_dir)):