Implement distributed product properties without applying them

build-scripts/compile_product.py

@@ -11,6 +11,10 @@ def create_parser():
         "e.g.: ~/scap-security-guide/products/rhel7/product.yml "
         "needed for autodetection of profile root"
     )
+    parser.add_argument(
+        "--product-properties",
+        help="The directory with additional product properties yamls."
+    )
     parser.add_argument(
         "--compiled-product-yaml", required=True,
         help="Where to save the compiled product yaml."
@@ -23,6 +27,8 @@ def main():
     args = parser.parse_args()
 
     product = ssg.products.Product(args.product_yaml)
+    if args.product_properties:
+        product.read_properties_from_directory(args.product_properties)
     product.write(args.compiled_product_yaml)
 
 

cmake/SSGCommon.cmake

@@ -96,7 +96,7 @@ macro(ssg_build_compiled_artifacts PRODUCT)
 
     add_custom_command(
         OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/product.yml"
-        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_product.py" --product-yaml "${CMAKE_SOURCE_DIR}/products/${PRODUCT}/product.yml" --compiled-product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml"
+        COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_product.py" --product-yaml "${CMAKE_SOURCE_DIR}/products/${PRODUCT}/product.yml" --product-properties "${CMAKE_SOURCE_DIR}/product_properties" --compiled-product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml"
         COMMENT "[${PRODUCT}-content] compiling product yaml"
     )
 

docs/flowcharts/flowchart_products.md

@@ -11,6 +11,7 @@ flowchart TD
     subgraph products
     60[products] --> |identified by| 61[product_name]
         61[product_name] --> |defined at| 62[product.yml]
+        61[product_properties] --> |defined at| 62[product.yml, product_properties/]
         61[product_name] --> |contains| 63[overlays]
         61[product_name] --> |contains| 64[profiles]
             64[profiles] --> |written at| 65[profile_name.profile]

docs/manual/developer/03_creating_content.md

@@ -68,6 +68,10 @@ build files/configuration, etc.
 <td><p><code>utils</code></p></td>
 <td><p>Miscellaneous scripts used for development but not used by the build system.</p></td>
 </tr>
+<tr class="even">
+<td><p><code>product_properties</code></p></td>
+<td><p>Directory with its own README and with drop-in files that can define product properties across more products at once using jinja macros.</p></td>
+</tr>
 </tbody>
 </table>
 

docs/manual/developer/06_contributing_with_content.md

@@ -418,15 +418,18 @@ that begin with underscores are not meant to be used in descriptions.
 You can also check documentation for all macros in the `Jinja Macros Reference`
 section accessible from the table of contents.
 
-To parametrize rules and remediations as well as Jinja macros, you can
-use product-specific variables defined in `product.yml` in product root
-directory. Moreover, you can define **implied properties** which are
-variables inferred from them. For example, you can define a condition
-that checks if the system uses `yum` or `dnf` as a package manager and
-based on that populate a variable containing correct path to the
-configuration file. The inferring logic is implemented in
-`_get_implied_properties` in `ssg/yaml.py`. Constants and mappings used
-in implied properties should be defined in `ssg/constants.py`.
+To parametrize rules and remediations as well as Jinja macros,
+use product-specific variables defined either
+in `product.yml` in product root directory,
+or in files in the `product_properties` project directory.
+Use this functionality to associate product properties with product versions,
+so you can use only product properties in the content.
+In other words, use this functionality to avoid referencing product versions in macros used in checks or remediations.
+Instead, use properties that directly relate to configurations being checked or set, and that help to reveal the intention of the check or remediation code.
+
+As Jinja2 conditionals are prone to errors, products can be protected by product stability tests.
+If a product sample is present in `tests/data/product_stability/`, it is compared to the actual compiled product,
+and if there is a difference that is not only cosmetic, a product stability test will fail.
 
 Rules are unselected by default - even if the scanner reads rule
 definitions, they are effectively ignored during the scan or

product_properties/README.md

@@ -0,0 +1,12 @@
+Product properties
+==================
+
+YAML files contained here are processed in lexicographic order, and they allow to define product properties in a efficient way.
+Processing of those files can use `jinja2`, and macros or conditionals have access to product properties defined previously and in the `product.yml`.
+
+Properties in a file are expressed in two mappings - obligatory `default` and optional `overrides`.
+Properties defined in a mapping nested below `default` can be overriden in a mapping nested below `overrides`.
+A property can be set only in one file, so the default-override pattern implemented by Jinja macros can be accomplished, but it has to take place in one file.
+Properties specified in the `product.yml` can't be overriden by this mechanism, and attempting to do so will result in an error.
+
+Conventionally, use the filename numerical prefix e.g. `10-` to ensure that some symbols are available before they are used in a definition of other symbols.

ssg/environment.py

@@ -6,7 +6,10 @@
 from .yaml import open_raw
 
 
-def open_environment(build_config_yaml_path, product_yaml_path):
+def open_environment(build_config_yaml_path, product_yaml_path, product_properties_path=None):
     contents = open_raw(build_config_yaml_path)
-    contents.update(load_product_yaml(product_yaml_path))
+    product = load_product_yaml(product_yaml_path)
+    if product_properties_path:
+        product.read_properties_from_directory(product_properties_path)
+    contents.update(product)
     return contents

ssg/products.py

@@ -26,7 +26,7 @@
                         XCCDF_PLATFORM_TO_PACKAGE,
                         SSG_REF_URIS)
 from .utils import merge_dicts, required_key
-from .yaml import open_raw, ordered_dump
+from .yaml import open_raw, ordered_dump, open_and_expand
 
 
 def _validate_product_oval_feed_url(contents):
@@ -111,50 +111,99 @@ def product_yaml_path(ssg_root, product):
 
 class Product(object):
     def __init__(self, filename):
-        self.primary_data = dict()
+        self._primary_data = dict()
+        self._acquired_data = dict()
         self._load_from_filename(filename)
-        if "basic_properties_derived" not in self.primary_data:
+        if "basic_properties_derived" not in self._primary_data:
             self._derive_basic_properties(filename)
 
+    @property
+    def _data_as_dict(self):
+        data = dict()
+        data.update(self._acquired_data)
+        data.update(self._primary_data)
+        return data
+
     def write(self, filename):
         with open(filename, "w") as f:
-            ordered_dump(self.primary_data, f)
+            ordered_dump(self._data_as_dict, f)
 
     def __getitem__(self, key):
-        return self.primary_data[key]
+        return self._data_as_dict[key]
 
     def __contains__(self, key):
-        return key in self.primary_data
+        return key in self._data_as_dict
 
     def __iter__(self):
-        return iter(self.primary_data.items())
+        return iter(self._data_as_dict.items())
 
     def __len__(self):
-        return len(self.primary_data)
+        return len(self._data_as_dict)
 
     def get(self, key, default=None):
-        return self.primary_data.get(key, default)
+        return self._data_as_dict.get(key, default)
 
     def _load_from_filename(self, filename):
-        self.primary_data = open_raw(filename)
+        self._primary_data = open_raw(filename)
 
     def _derive_basic_properties(self, filename):
-        _validate_product_oval_feed_url(self.primary_data)
+        _validate_product_oval_feed_url(self._primary_data)
 
         # The product directory is necessary to get absolute paths to benchmark, profile and
         # cpe directories, which are all relative to the product directory
-        self.primary_data["product_dir"] = os.path.dirname(filename)
+        self._primary_data["product_dir"] = os.path.dirname(filename)
 
-        platform_package_overrides = self.primary_data.get("platform_package_overrides", {})
+        platform_package_overrides = self._primary_data.get("platform_package_overrides", {})
         # Merge common platform package mappings, while keeping product specific mappings
-        self.primary_data["platform_package_overrides"] = merge_dicts(
+        self._primary_data["platform_package_overrides"] = merge_dicts(
                 XCCDF_PLATFORM_TO_PACKAGE, platform_package_overrides)
-        self.primary_data.update(_get_implied_properties(self.primary_data))
-
-        reference_uris = self.primary_data.get("reference_uris", {})
-        self.primary_data["reference_uris"] = merge_dicts(SSG_REF_URIS, reference_uris)
-
-        self.primary_data["basic_properties_derived"] = True
+        self._primary_data.update(_get_implied_properties(self._primary_data))
+
+        reference_uris = self._primary_data.get("reference_uris", {})
+        self._primary_data["reference_uris"] = merge_dicts(SSG_REF_URIS, reference_uris)
+
+        self._primary_data["basic_properties_derived"] = True
+
+    def expand_by_acquired_data(self, property_dict):
+        for specified_key in property_dict:
+            if specified_key in self:
+                msg = (
+                    "The property {name} is already defined, "
+                    "you can't define it once more elsewhere."
+                    .format(name=specified_key))
+                raise ValueError(msg)
+        self._acquired_data.update(property_dict)
+
+    @staticmethod
+    def transform_default_and_overrides_mappings_to_mapping(mappings):
+        result = dict()
+        if not isinstance(mappings, dict):
+            msg = (
+                "Expected a mapping, got {type}."
+                .format(type=str(type(mappings))))
+            raise ValueError(msg)
+
+        mapping = mappings.pop("default")
+        if mapping:
+            result.update(mapping)
+        mapping = mappings.pop("overrides", dict())
+        if mapping:
+            result.update(mapping)
+        if len(mappings):
+            msg = (
+                "The dictionary contains unwanted keys: {keys}"
+                .format(keys=list(mappings.keys())))
+            raise ValueError(msg)
+        return result
+
+    def read_properties_from_directory(self, path):
+        filenames = glob(path + "/*.yml")
+        for f in sorted(filenames):
+            substitutions_dict = dict()
+            substitutions_dict.update(self)
+            new_defs = open_and_expand(f, substitutions_dict)
+            new_symbols = self.transform_default_and_overrides_mappings_to_mapping(new_defs)
+            self.expand_by_acquired_data(new_symbols)
 
 
 def load_product_yaml(product_yaml_path):

ssg/yaml.py

@@ -48,23 +48,34 @@ def _save_rename(result, stem, prefix):
     result["{0}_{1}".format(prefix, stem)] = stem
 
 
+def _get_yaml_contents_without_documentation_complete(parsed_yaml, substitutions_dict):
+    """
+    If the YAML is a mapping, then handle the documentation_complete accordingly,
+    and take that key-value out.
+    Otherwise, if YAML is empty, or it is a list, pass it on.
+    """
+    if isinstance(parsed_yaml, dict):
+        documentation_incomplete_content_and_not_debug_build = (
+            parsed_yaml.pop("documentation_complete", "true") == "false"
+            and substitutions_dict.get("cmake_build_type") != "Debug")
+        if documentation_incomplete_content_and_not_debug_build:
+            raise DocumentationNotComplete("documentation not complete and not a debug build")
+    return parsed_yaml
+
+
 def _open_yaml(stream, original_file=None, substitutions_dict={}):
     """
     Open given file-like object and parse it as YAML.
 
     Optionally, pass the path to the original_file for better error handling
     when the file contents are passed.
 
-    Return None if it contains "documentation_complete" key set to "false".
+    Raise an exception if it contains "documentation_complete" key set to "false".
     """
     try:
         yaml_contents = yaml.load(stream, Loader=yaml_SafeLoader)
 
-        if yaml_contents.pop("documentation_complete", "true") == "false" and \
-                substitutions_dict.get("cmake_build_type") != "Debug":
-            raise DocumentationNotComplete("documentation not complete and not a debug build")
-
-        return yaml_contents
+        return _get_yaml_contents_without_documentation_complete(yaml_contents, substitutions_dict)
     except DocumentationNotComplete as e:
         raise e
     except Exception as e:

tests/test_parse_affected.py

@@ -15,6 +15,7 @@
 import ssg.yaml
 import ssg.build_yaml
 import ssg.rule_yaml
+import ssg.products
 
 
 def main():
@@ -34,16 +35,20 @@ def main():
 
     known_dirs = set()
     for product in ssg.constants.product_directories:
-        product_dir = os.path.join(ssg_root, "products", product)
-        product_yaml_path = os.path.join(product_dir, "product.yml")
-        product_yaml = ssg.yaml.open_raw(product_yaml_path)
+        product_yaml_path = ssg.products.product_yaml_path(ssg_root, product)
+        product = ssg.products.Product(product_yaml_path)
 
-        env_yaml = ssg.environment.open_environment(ssg_build_config_yaml, product_yaml_path)
+        product_properties_path = os.path.join(ssg_root, "product_properties")
+        env_yaml = ssg.environment.open_environment(
+                ssg_build_config_yaml, product_yaml_path, product_properties_path)
+        env_yaml.update(product)
         ssg.jinja.add_python_functions(env_yaml)
 
-        guide_dir = os.path.join(product_dir, product_yaml['benchmark_root'])
-        additional_content_directories = product_yaml.get("additional_content_directories", [])
-        add_content_dirs = [os.path.abspath(os.path.join(product_dir, rd)) for rd in additional_content_directories]
+        guide_dir = os.path.join(product["product_dir"], product['benchmark_root'])
+        additional_content_directories = product.get("additional_content_directories", [])
+        add_content_dirs = [
+                os.path.abspath(os.path.join(product["product_dir"], rd))
+                for rd in additional_content_directories]
 
         for cur_dir in [guide_dir] + add_content_dirs:
             if cur_dir not in known_dirs:

tests/unit/ssg-module/data/properties/00-default.yml

@@ -0,0 +1,9 @@
+
+default:
+  property_one: one
+
+{{%- if product == "rhel7" %}}
+  rhel_version: "seven"
+{{%- else %}}
+  rhel_version: "not_seven"
+{{% endif %}}

tests/unit/ssg-module/data/properties/10-property_two.yml

@@ -0,0 +1,8 @@
+default:
+  property_two: one
+
+overrides:
+{{% if property_one == "one" %}}
+  property_two: two
+{{% endif %}}
+