diff --git a/build-scripts/compile_product.py b/build-scripts/compile_product.py index 9b8da4a70eb..d12f83244c8 100644 --- a/build-scripts/compile_product.py +++ b/build-scripts/compile_product.py @@ -11,6 +11,10 @@ def create_parser(): "e.g.: ~/scap-security-guide/products/rhel7/product.yml " "needed for autodetection of profile root" ) + parser.add_argument( + "--product-properties", + help="The directory with additional product properties yamls." + ) parser.add_argument( "--compiled-product-yaml", required=True, help="Where to save the compiled product yaml." @@ -23,6 +27,8 @@ def main(): args = parser.parse_args() product = ssg.products.Product(args.product_yaml) + if args.product_properties: + product.read_properties_from_directory(args.product_properties) product.write(args.compiled_product_yaml) diff --git a/cmake/SSGCommon.cmake b/cmake/SSGCommon.cmake index f52e5425229..52f88413d7f 100644 --- a/cmake/SSGCommon.cmake +++ b/cmake/SSGCommon.cmake @@ -96,7 +96,7 @@ macro(ssg_build_compiled_artifacts PRODUCT) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/product.yml" - COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_product.py" --product-yaml "${CMAKE_SOURCE_DIR}/products/${PRODUCT}/product.yml" --compiled-product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" + COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${SSG_BUILD_SCRIPTS}/compile_product.py" --product-yaml "${CMAKE_SOURCE_DIR}/products/${PRODUCT}/product.yml" --product-properties "${CMAKE_SOURCE_DIR}/product_properties" --compiled-product-yaml "${CMAKE_CURRENT_BINARY_DIR}/product.yml" COMMENT "[${PRODUCT}-content] compiling product yaml" ) diff --git a/docs/flowcharts/flowchart_products.md b/docs/flowcharts/flowchart_products.md index 477a17dbaed..69cea5cc345 100644 --- a/docs/flowcharts/flowchart_products.md +++ b/docs/flowcharts/flowchart_products.md @@ -11,6 +11,7 @@ flowchart TD subgraph products 60[products] --> |identified by| 61[product_name] 61[product_name] --> |defined at| 62[product.yml] + 61[product_properties] --> |defined at| 62[product.yml, product_properties/] 61[product_name] --> |contains| 63[overlays] 61[product_name] --> |contains| 64[profiles] 64[profiles] --> |written at| 65[profile_name.profile] diff --git a/docs/manual/developer/03_creating_content.md b/docs/manual/developer/03_creating_content.md index 113eba3b7bc..26e5f6a264f 100644 --- a/docs/manual/developer/03_creating_content.md +++ b/docs/manual/developer/03_creating_content.md @@ -68,6 +68,10 @@ build files/configuration, etc.

utils

Miscellaneous scripts used for development but not used by the build system.

+ +

product_properties

+

Directory with its own README and with drop-in files that can define product properties across more products at once using jinja macros.

+ diff --git a/docs/manual/developer/06_contributing_with_content.md b/docs/manual/developer/06_contributing_with_content.md index 65bbf62175c..f7c8182bad6 100644 --- a/docs/manual/developer/06_contributing_with_content.md +++ b/docs/manual/developer/06_contributing_with_content.md @@ -418,15 +418,18 @@ that begin with underscores are not meant to be used in descriptions. You can also check documentation for all macros in the `Jinja Macros Reference` section accessible from the table of contents. -To parametrize rules and remediations as well as Jinja macros, you can -use product-specific variables defined in `product.yml` in product root -directory. Moreover, you can define **implied properties** which are -variables inferred from them. For example, you can define a condition -that checks if the system uses `yum` or `dnf` as a package manager and -based on that populate a variable containing correct path to the -configuration file. The inferring logic is implemented in -`_get_implied_properties` in `ssg/yaml.py`. Constants and mappings used -in implied properties should be defined in `ssg/constants.py`. +To parametrize rules and remediations as well as Jinja macros, +use product-specific variables defined either +in `product.yml` in product root directory, +or in files in the `product_properties` project directory. +Use this functionality to associate product properties with product versions, +so you can use only product properties in the content. +In other words, use this functionality to avoid referencing product versions in macros used in checks or remediations. +Instead, use properties that directly relate to configurations being checked or set, and that help to reveal the intention of the check or remediation code. + +As Jinja2 conditionals are prone to errors, products can be protected by product stability tests. +If a product sample is present in `tests/data/product_stability/`, it is compared to the actual compiled product, +and if there is a difference that is not only cosmetic, a product stability test will fail. Rules are unselected by default - even if the scanner reads rule definitions, they are effectively ignored during the scan or diff --git a/product_properties/README.md b/product_properties/README.md new file mode 100644 index 00000000000..c5694ff0815 --- /dev/null +++ b/product_properties/README.md @@ -0,0 +1,12 @@ +Product properties +================== + +YAML files contained here are processed in lexicographic order, and they allow to define product properties in a efficient way. +Processing of those files can use `jinja2`, and macros or conditionals have access to product properties defined previously and in the `product.yml`. + +Properties in a file are expressed in two mappings - obligatory `default` and optional `overrides`. +Properties defined in a mapping nested below `default` can be overriden in a mapping nested below `overrides`. +A property can be set only in one file, so the default-override pattern implemented by Jinja macros can be accomplished, but it has to take place in one file. +Properties specified in the `product.yml` can't be overriden by this mechanism, and attempting to do so will result in an error. + +Conventionally, use the filename numerical prefix e.g. `10-` to ensure that some symbols are available before they are used in a definition of other symbols. diff --git a/ssg/environment.py b/ssg/environment.py index 36e78a4142b..52a29e03465 100644 --- a/ssg/environment.py +++ b/ssg/environment.py @@ -6,7 +6,10 @@ from .yaml import open_raw -def open_environment(build_config_yaml_path, product_yaml_path): +def open_environment(build_config_yaml_path, product_yaml_path, product_properties_path=None): contents = open_raw(build_config_yaml_path) - contents.update(load_product_yaml(product_yaml_path)) + product = load_product_yaml(product_yaml_path) + if product_properties_path: + product.read_properties_from_directory(product_properties_path) + contents.update(product) return contents diff --git a/ssg/products.py b/ssg/products.py index 0ec01d08064..0d711739b41 100644 --- a/ssg/products.py +++ b/ssg/products.py @@ -26,7 +26,7 @@ XCCDF_PLATFORM_TO_PACKAGE, SSG_REF_URIS) from .utils import merge_dicts, required_key -from .yaml import open_raw, ordered_dump +from .yaml import open_raw, ordered_dump, open_and_expand def _validate_product_oval_feed_url(contents): @@ -111,50 +111,99 @@ def product_yaml_path(ssg_root, product): class Product(object): def __init__(self, filename): - self.primary_data = dict() + self._primary_data = dict() + self._acquired_data = dict() self._load_from_filename(filename) - if "basic_properties_derived" not in self.primary_data: + if "basic_properties_derived" not in self._primary_data: self._derive_basic_properties(filename) + @property + def _data_as_dict(self): + data = dict() + data.update(self._acquired_data) + data.update(self._primary_data) + return data + def write(self, filename): with open(filename, "w") as f: - ordered_dump(self.primary_data, f) + ordered_dump(self._data_as_dict, f) def __getitem__(self, key): - return self.primary_data[key] + return self._data_as_dict[key] def __contains__(self, key): - return key in self.primary_data + return key in self._data_as_dict def __iter__(self): - return iter(self.primary_data.items()) + return iter(self._data_as_dict.items()) def __len__(self): - return len(self.primary_data) + return len(self._data_as_dict) def get(self, key, default=None): - return self.primary_data.get(key, default) + return self._data_as_dict.get(key, default) def _load_from_filename(self, filename): - self.primary_data = open_raw(filename) + self._primary_data = open_raw(filename) def _derive_basic_properties(self, filename): - _validate_product_oval_feed_url(self.primary_data) + _validate_product_oval_feed_url(self._primary_data) # The product directory is necessary to get absolute paths to benchmark, profile and # cpe directories, which are all relative to the product directory - self.primary_data["product_dir"] = os.path.dirname(filename) + self._primary_data["product_dir"] = os.path.dirname(filename) - platform_package_overrides = self.primary_data.get("platform_package_overrides", {}) + platform_package_overrides = self._primary_data.get("platform_package_overrides", {}) # Merge common platform package mappings, while keeping product specific mappings - self.primary_data["platform_package_overrides"] = merge_dicts( + self._primary_data["platform_package_overrides"] = merge_dicts( XCCDF_PLATFORM_TO_PACKAGE, platform_package_overrides) - self.primary_data.update(_get_implied_properties(self.primary_data)) - - reference_uris = self.primary_data.get("reference_uris", {}) - self.primary_data["reference_uris"] = merge_dicts(SSG_REF_URIS, reference_uris) - - self.primary_data["basic_properties_derived"] = True + self._primary_data.update(_get_implied_properties(self._primary_data)) + + reference_uris = self._primary_data.get("reference_uris", {}) + self._primary_data["reference_uris"] = merge_dicts(SSG_REF_URIS, reference_uris) + + self._primary_data["basic_properties_derived"] = True + + def expand_by_acquired_data(self, property_dict): + for specified_key in property_dict: + if specified_key in self: + msg = ( + "The property {name} is already defined, " + "you can't define it once more elsewhere." + .format(name=specified_key)) + raise ValueError(msg) + self._acquired_data.update(property_dict) + + @staticmethod + def transform_default_and_overrides_mappings_to_mapping(mappings): + result = dict() + if not isinstance(mappings, dict): + msg = ( + "Expected a mapping, got {type}." + .format(type=str(type(mappings)))) + raise ValueError(msg) + + mapping = mappings.pop("default") + if mapping: + result.update(mapping) + mapping = mappings.pop("overrides", dict()) + if mapping: + result.update(mapping) + if len(mappings): + msg = ( + "The dictionary contains unwanted keys: {keys}" + .format(keys=list(mappings.keys()))) + raise ValueError(msg) + return result + + def read_properties_from_directory(self, path): + filenames = glob(path + "/*.yml") + for f in sorted(filenames): + substitutions_dict = dict() + substitutions_dict.update(self) + new_defs = open_and_expand(f, substitutions_dict) + new_symbols = self.transform_default_and_overrides_mappings_to_mapping(new_defs) + self.expand_by_acquired_data(new_symbols) def load_product_yaml(product_yaml_path): diff --git a/ssg/yaml.py b/ssg/yaml.py index 356eb3b7a4d..f85eac30c5c 100644 --- a/ssg/yaml.py +++ b/ssg/yaml.py @@ -48,6 +48,21 @@ def _save_rename(result, stem, prefix): result["{0}_{1}".format(prefix, stem)] = stem +def _get_yaml_contents_without_documentation_complete(parsed_yaml, substitutions_dict): + """ + If the YAML is a mapping, then handle the documentation_complete accordingly, + and take that key-value out. + Otherwise, if YAML is empty, or it is a list, pass it on. + """ + if isinstance(parsed_yaml, dict): + documentation_incomplete_content_and_not_debug_build = ( + parsed_yaml.pop("documentation_complete", "true") == "false" + and substitutions_dict.get("cmake_build_type") != "Debug") + if documentation_incomplete_content_and_not_debug_build: + raise DocumentationNotComplete("documentation not complete and not a debug build") + return parsed_yaml + + def _open_yaml(stream, original_file=None, substitutions_dict={}): """ Open given file-like object and parse it as YAML. @@ -55,16 +70,12 @@ def _open_yaml(stream, original_file=None, substitutions_dict={}): Optionally, pass the path to the original_file for better error handling when the file contents are passed. - Return None if it contains "documentation_complete" key set to "false". + Raise an exception if it contains "documentation_complete" key set to "false". """ try: yaml_contents = yaml.load(stream, Loader=yaml_SafeLoader) - if yaml_contents.pop("documentation_complete", "true") == "false" and \ - substitutions_dict.get("cmake_build_type") != "Debug": - raise DocumentationNotComplete("documentation not complete and not a debug build") - - return yaml_contents + return _get_yaml_contents_without_documentation_complete(yaml_contents, substitutions_dict) except DocumentationNotComplete as e: raise e except Exception as e: diff --git a/tests/test_parse_affected.py b/tests/test_parse_affected.py index 53690df5ce1..8f803a0c31f 100755 --- a/tests/test_parse_affected.py +++ b/tests/test_parse_affected.py @@ -15,6 +15,7 @@ import ssg.yaml import ssg.build_yaml import ssg.rule_yaml +import ssg.products def main(): @@ -34,16 +35,20 @@ def main(): known_dirs = set() for product in ssg.constants.product_directories: - product_dir = os.path.join(ssg_root, "products", product) - product_yaml_path = os.path.join(product_dir, "product.yml") - product_yaml = ssg.yaml.open_raw(product_yaml_path) + product_yaml_path = ssg.products.product_yaml_path(ssg_root, product) + product = ssg.products.Product(product_yaml_path) - env_yaml = ssg.environment.open_environment(ssg_build_config_yaml, product_yaml_path) + product_properties_path = os.path.join(ssg_root, "product_properties") + env_yaml = ssg.environment.open_environment( + ssg_build_config_yaml, product_yaml_path, product_properties_path) + env_yaml.update(product) ssg.jinja.add_python_functions(env_yaml) - guide_dir = os.path.join(product_dir, product_yaml['benchmark_root']) - additional_content_directories = product_yaml.get("additional_content_directories", []) - add_content_dirs = [os.path.abspath(os.path.join(product_dir, rd)) for rd in additional_content_directories] + guide_dir = os.path.join(product["product_dir"], product['benchmark_root']) + additional_content_directories = product.get("additional_content_directories", []) + add_content_dirs = [ + os.path.abspath(os.path.join(product["product_dir"], rd)) + for rd in additional_content_directories] for cur_dir in [guide_dir] + add_content_dirs: if cur_dir not in known_dirs: diff --git a/tests/unit/ssg-module/data/properties/00-default.yml b/tests/unit/ssg-module/data/properties/00-default.yml new file mode 100644 index 00000000000..74ce8f670cd --- /dev/null +++ b/tests/unit/ssg-module/data/properties/00-default.yml @@ -0,0 +1,9 @@ + +default: + property_one: one + +{{%- if product == "rhel7" %}} + rhel_version: "seven" +{{%- else %}} + rhel_version: "not_seven" +{{% endif %}} diff --git a/tests/unit/ssg-module/data/properties/10-property_two.yml b/tests/unit/ssg-module/data/properties/10-property_two.yml new file mode 100644 index 00000000000..ddb41ec2576 --- /dev/null +++ b/tests/unit/ssg-module/data/properties/10-property_two.yml @@ -0,0 +1,8 @@ +default: + property_two: one + +overrides: +{{% if property_one == "one" %}} + property_two: two +{{% endif %}} + diff --git a/tests/unit/ssg-module/test_controls.py b/tests/unit/ssg-module/test_controls.py index d289d0a7b30..59594e43657 100644 --- a/tests/unit/ssg-module/test_controls.py +++ b/tests/unit/ssg-module/test_controls.py @@ -2,6 +2,8 @@ import logging import os +import pytest + import ssg.controls import ssg.build_yaml from ssg.environment import open_environment @@ -13,10 +15,15 @@ profiles_dir = os.path.join(data_dir, "profiles_dir") -def _load_test(profile): +@pytest.fixture +def env_yaml(): product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") - env_yaml = open_environment(build_config_yaml, product_yaml) + return open_environment( + build_config_yaml, product_yaml, os.path.join(ssg_root, "product_properties")) + + +def _load_test(env_yaml, profile): controls_manager = ssg.controls.ControlsManager(controls_dir, env_yaml) controls_manager.load() c_r1 = controls_manager.get_control(profile, "R1") @@ -60,14 +67,11 @@ def _load_test(profile): assert c_r5.status_justification == "Mitigate with third-party software." -def test_controls_load(): - _load_test("abcd") +def test_controls_load(env_yaml): + _load_test(env_yaml, "abcd") -def test_controls_levels(): - product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") - build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") - env_yaml = open_environment(build_config_yaml, product_yaml) +def test_controls_levels(env_yaml): controls_manager = ssg.controls.ControlsManager(controls_dir, env_yaml) controls_manager.load() @@ -185,11 +189,7 @@ def test_controls_levels(): assert "configure_crypto_policy" in s7_high[0].selections -def test_controls_load_product(): - product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") - build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") - env_yaml = open_environment(build_config_yaml, product_yaml) - +def test_controls_load_product(env_yaml): controls_manager = ssg.controls.ControlsManager(controls_dir, env_yaml) controls_manager.load() @@ -207,20 +207,21 @@ def test_controls_load_product(): assert c_r1.variables["var_accounts_tmout"] == "10_min" -def test_profile_resolution_inline(): +def test_profile_resolution_inline(env_yaml): profile_resolution( - ssg.build_yaml.ProfileWithInlinePolicies, "abcd-low-inline") + env_yaml, ssg.build_yaml.ProfileWithInlinePolicies, "abcd-low-inline") -def test_profile_resolution_extends_inline(): +def test_profile_resolution_extends_inline(env_yaml): profile_resolution_extends( + env_yaml, ssg.build_yaml.ProfileWithInlinePolicies, "abcd-low-inline", "abcd-high-inline") -def test_profile_resolution_all_inline(): +def test_profile_resolution_all_inline(env_yaml): profile_resolution_all( - ssg.build_yaml.ProfileWithInlinePolicies, "abcd-all-inline") + env_yaml, ssg.build_yaml.ProfileWithInlinePolicies, "abcd-all-inline") class DictContainingAnyRule(dict): @@ -233,11 +234,7 @@ def __contains__(self, rid): return True -def profile_resolution(cls, profile_low): - product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") - build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") - env_yaml = open_environment(build_config_yaml, product_yaml) - +def profile_resolution(env_yaml, cls, profile_low): controls_manager = ssg.controls.ControlsManager(controls_dir, env_yaml) controls_manager.load() low_profile_path = os.path.join(profiles_dir, profile_low + ".profile") @@ -261,11 +258,7 @@ def profile_resolution(cls, profile_low): assert "security_patches_up_to_date" in selected -def profile_resolution_extends(cls, profile_low, profile_high): - product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") - build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") - env_yaml = open_environment(build_config_yaml, product_yaml) - +def profile_resolution_extends(env_yaml, cls, profile_low, profile_high): # tests ABCD High profile which is defined as an extension of ABCD Low controls_manager = ssg.controls.ControlsManager(controls_dir, env_yaml) controls_manager.load() @@ -299,11 +292,7 @@ def profile_resolution_extends(cls, profile_low, profile_high): assert high_profile.variables["var_password_pam_ocredit"] == "2" -def profile_resolution_all(cls, profile_all): - product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") - build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") - env_yaml = open_environment(build_config_yaml, product_yaml) - +def profile_resolution_all(env_yaml, cls, profile_all): controls_manager = ssg.controls.ControlsManager(controls_dir, env_yaml) controls_manager.load() profile_path = os.path.join(profiles_dir, profile_all + ".profile") @@ -336,13 +325,13 @@ def profile_resolution_all(cls, profile_all): assert "security_patches_up_to_date" in selected -def test_load_control_from_folder(): - _load_test("qrst") +def test_load_control_from_folder(env_yaml): + _load_test(env_yaml, "qrst") -def test_load_control_from_folder_and_file(): - _load_test("jklm") +def test_load_control_from_folder_and_file(env_yaml): + _load_test(env_yaml, "jklm") -def test_load_control_from_specific_folder_and_file(): - _load_test("nopq") +def test_load_control_from_specific_folder_and_file(env_yaml): + _load_test(env_yaml, "nopq") diff --git a/tests/unit/ssg-module/test_products.py b/tests/unit/ssg-module/test_products.py index 7cdb716d181..d61a06d9859 100644 --- a/tests/unit/ssg-module/test_products.py +++ b/tests/unit/ssg-module/test_products.py @@ -3,6 +3,7 @@ import pytest import ssg.products +import ssg.yaml @pytest.fixture @@ -11,8 +12,36 @@ def ssg_root(): @pytest.fixture -def testing_product_yaml_path(): - return os.path.abspath(os.path.join(os.path.dirname(__file__), "data", "product.yml")) +def testing_datadir(): + return os.path.abspath(os.path.join(os.path.dirname(__file__), "data")) + + +@pytest.fixture +def testing_product_yaml_path(testing_datadir): + return os.path.abspath(os.path.join(testing_datadir, "product.yml")) + + +@pytest.fixture +def testing_product(testing_product_yaml_path): + return ssg.products.Product(testing_product_yaml_path) + + +@pytest.fixture +def product_with_updated_properties(testing_product, testing_datadir): + properties_dir = os.path.join(testing_datadir, "properties") + testing_product.read_properties_from_directory(properties_dir) + return testing_product + + +def test_default_and_overrides_mappings_to_mapping(): + converter = ssg.products.Product.transform_default_and_overrides_mappings_to_mapping + assert converter(dict(default=[])) == dict() + assert converter(dict(default=dict(one=1))) == dict(one=1) + assert converter(dict(default=dict(one=2), overrides=dict(one=1))) == dict(one=1) + with pytest.raises(ValueError): + converter([dict(one=2), 5]) + with pytest.raises(KeyError): + converter(dict(deflaut=dict(one=2))) def test_get_all(ssg_root): @@ -28,16 +57,14 @@ def test_get_all(ssg_root): assert "firefox" not in products.linux -def test_product_yaml(testing_product_yaml_path): - product = ssg.products.Product(testing_product_yaml_path) - assert "product" in product - assert product["product"] == "rhel7" - assert product["pkg_system"] == "rpm" - assert product["product_dir"].endswith("data") - assert product.get("X", "x") == "x" +def test_product_yaml(testing_product): + assert "product" in testing_product + assert testing_product["pkg_system"] == "rpm" + assert testing_product["product_dir"].endswith("data") + assert testing_product.get("X", "x") == "x" copied_product = dict() - copied_product.update(product) + copied_product.update(testing_product) assert copied_product["pkg_system"] == "rpm" @@ -51,8 +78,49 @@ def product_filename_py3(tmp_path): return tmp_path / "tmp_product.yml" -def test_product_yaml_write(testing_product_yaml_path, product_filename_py2): - product = ssg.products.Product(testing_product_yaml_path) - product.write(product_filename_py2) +def test_product_yaml_write(testing_product, product_filename_py2): + testing_product.write(product_filename_py2) second_product = ssg.products.Product(product_filename_py2) - assert product["product_dir"] == second_product["product_dir"] + assert testing_product["product_dir"] == second_product["product_dir"] + + +def test_product_updates_with_dict(testing_product): + assert "property_one" not in testing_product + properties = dict(property_one="one") + testing_product.expand_by_acquired_data(properties) + assert testing_product["property_one"] == "one" + + +def test_product_updates_with_files(product_with_updated_properties): + product = product_with_updated_properties + assert product["property_one"] == "one" + assert product["product"] == "rhel7" + assert product["rhel_version"] == "seven" + + +def test_updates_have_access_to_previously_defined_properties(product_with_updated_properties): + product = product_with_updated_properties + assert product["property_two"] == "two" + + +def test_product_properties_set_only_in_one_place(product_with_updated_properties): + product = product_with_updated_properties + existing_data = dict(pkg_manager=product["pkg_manager"]) + with pytest.raises(ValueError): + product.expand_by_acquired_data(existing_data) + + existing_data = dict(property_one=1) + with pytest.raises(ValueError): + product.expand_by_acquired_data(existing_data) + + new_data = dict(new_one=1) + product.expand_by_acquired_data(new_data) + with pytest.raises(ValueError): + product.expand_by_acquired_data(new_data) + + +def test_product_updating_twice_doesnt_work(product_with_updated_properties, testing_datadir): + testing_product = product_with_updated_properties + properties_dir = os.path.join(testing_datadir, "properties") + with pytest.raises(ValueError): + testing_product.read_properties_from_directory(properties_dir) diff --git a/tests/unit/ssg-module/test_templates.py b/tests/unit/ssg-module/test_templates.py index a24bda79cad..2dbde4a8b32 100644 --- a/tests/unit/ssg-module/test_templates.py +++ b/tests/unit/ssg-module/test_templates.py @@ -18,7 +18,7 @@ build_config_yaml = os.path.join(ssg_root, "build", "build_config.yml") product_yaml = os.path.join(ssg_root, "products", "rhel8", "product.yml") -env_yaml = open_environment(build_config_yaml, product_yaml) +env_yaml = open_environment(build_config_yaml, product_yaml, os.path.join(ssg_root, "product_properties")) def test_render_extra_ovals(): diff --git a/utils/autoprodtyper.py b/utils/autoprodtyper.py index 143e9b1a951..4603413522d 100755 --- a/utils/autoprodtyper.py +++ b/utils/autoprodtyper.py @@ -64,7 +64,8 @@ def main(): product_base = os.path.join(SSG_ROOT, "products", args.product) product_yaml = os.path.join(product_base, "product.yml") - env_yaml = ssg.environment.open_environment(args.build_config_yaml, product_yaml) + env_yaml = ssg.environment.open_environment( + args.build_config_yaml, product_yaml, os.path.join(SSG_ROOT, "product_properties")) profiles_root = os.path.join(product_base, "profiles") if args.profiles_root: diff --git a/utils/build_stig_control.py b/utils/build_stig_control.py index d62f63366b8..5900ccf1910 100755 --- a/utils/build_stig_control.py +++ b/utils/build_stig_control.py @@ -85,7 +85,8 @@ def get_implemented_stigs(args): product_dir = os.path.join(args.root, "products", args.product) product_yaml_path = os.path.join(product_dir, "product.yml") - env_yaml = ssg.environment.open_environment(args.build_config_yaml, str(product_yaml_path)) + env_yaml = ssg.environment.open_environment( + args.build_config_yaml, product_yaml_path, os.path.join(args._root, "product_properties")) known_rules = dict() for rule in platform_rules: diff --git a/utils/controlrefcheck.py b/utils/controlrefcheck.py index d5063ccafc6..f955b022300 100755 --- a/utils/controlrefcheck.py +++ b/utils/controlrefcheck.py @@ -73,7 +73,8 @@ def get_rule_object(all_rules, args, control_rule, env_yaml) -> ssg.build_yaml.R def get_controls_env(args): product_base = os.path.join(SSG_ROOT, "products", args.product) product_yaml = os.path.join(product_base, "product.yml") - env_yaml = ssg.environment.open_environment(args.build_config_yaml, product_yaml) + env_yaml = ssg.environment.open_environment( + args.build_config_yaml, product_yaml, os.path.join(SSG_ROOT, "product_properties")) controls_manager = ssg.controls.ControlsManager(args.controls, env_yaml) controls_manager.load() return controls_manager, env_yaml diff --git a/utils/create_scap_delta_tailoring.py b/utils/create_scap_delta_tailoring.py index f270a435fb5..5c5a7d37ebc 100755 --- a/utils/create_scap_delta_tailoring.py +++ b/utils/create_scap_delta_tailoring.py @@ -109,7 +109,8 @@ def get_implemented_stigs(product, root_path, build_config_yaml_path, product_dir = os.path.join(root_path, "products", product) product_yaml_path = os.path.join(product_dir, "product.yml") - env_yaml = ssg.environment.open_environment(build_config_yaml_path, str(product_yaml_path)) + env_yaml = ssg.environment.open_environment( + build_config_yaml_path, product_yaml_path, os.path.join(root_path, "product_properties")) known_rules = dict() for rule in platform_rules: diff --git a/utils/create_srg_export.py b/utils/create_srg_export.py index 9d8cf6cba91..f1de9e92f66 100755 --- a/utils/create_srg_export.py +++ b/utils/create_srg_export.py @@ -369,10 +369,11 @@ def handle_output(output: str, results: list, format_type: str, product: str) -> print(f'Wrote output to {output}') -def get_env_yaml(root: str, product: str, build_config_yaml: str) -> dict: - product_dir = os.path.join(root, "products", product) +def get_env_yaml(root: str, product_path: str, build_config_yaml: str) -> dict: + product_dir = os.path.join(root, "products", product_path) product_yaml_path = os.path.join(product_dir, "product.yml") - env_yaml = ssg.environment.open_environment(build_config_yaml, str(product_yaml_path)) + env_yaml = ssg.environment.open_environment( + build_config_yaml, product_yaml_path, os.path.join(root, "product_properties")) return env_yaml @@ -383,8 +384,8 @@ def main() -> None: srgs = ssg.build_stig.parse_srgs(args.manual) product_dir = os.path.join(args.root, "products", args.product) - product_yaml_path = os.path.join(product_dir, "product.yml") - env_yaml = ssg.environment.open_environment(args.build_config_yaml, str(product_yaml_path)) + env_yaml = get_env_yaml( + args.root, args.product, args.build_config_yaml) policy = get_policy(args, env_yaml) rule_json = get_rule_json(args.json) diff --git a/utils/fix_rules.py b/utils/fix_rules.py index 4d3ce7a7445..763089ea73f 100755 --- a/utils/fix_rules.py +++ b/utils/fix_rules.py @@ -153,6 +153,8 @@ def find_rules_generator(args, func): if product not in product_yamls: product_path = ssg.products.product_yaml_path(args.root, product) product_yaml = ssg.products.load_product_yaml(product_path) + properties_directory = os.path.join(args.root, "product_properties") + product_yaml.read_properties_from_directory(properties_directory) product_yamls[product] = product_yaml local_env_yaml = dict(cmake_build_type='Debug') diff --git a/utils/refchecker.py b/utils/refchecker.py index 9896e98fb84..4ae789d02a3 100755 --- a/utils/refchecker.py +++ b/utils/refchecker.py @@ -111,7 +111,8 @@ def main(): product_base = os.path.join(SSG_ROOT, "products", args.product) product_yaml_path = os.path.join(product_base, "product.yml") product_yaml = ssg.products.Product(product_yaml_path) - env_yaml = ssg.environment.open_environment(args.build_config_yaml, product_yaml_path) + env_yaml = ssg.environment.open_environment( + args.build_config_yaml, product_yaml_path, os.path.join(SSG_ROOT, "product_properties")) controls_manager = None if os.path.exists(args.controls): diff --git a/utils/rule_dir_json.py b/utils/rule_dir_json.py index 76afc3b2fd8..749bb9c2998 100755 --- a/utils/rule_dir_json.py +++ b/utils/rule_dir_json.py @@ -50,6 +50,7 @@ def walk_products(root, all_products): product_dir = os.path.join(root, "products", product) product_yaml_path = os.path.join(product_dir, "product.yml") product_yaml = ssg.products.load_product_yaml(product_yaml_path) + product_yaml.read_properties_from_directory(os.path.join(root, "product_properties")) product_yamls[product] = product_yaml guide_dir = os.path.join(product_dir, product_yaml['benchmark_root']) diff --git a/utils/template_renderer.py b/utils/template_renderer.py index ecb2d94a922..01e73f779f8 100644 --- a/utils/template_renderer.py +++ b/utils/template_renderer.py @@ -58,7 +58,7 @@ def __init__(self, product, build_dir, verbose=False): self.verbose = verbose def get_env_yaml(self, build_dir): - product_yaml = os.path.join(self.project_directory, "products", self.product, "product.yml") + product_yaml = os.path.join(build_dir, self.product, "product.yml") build_config_yaml = os.path.join(build_dir, "build_config.yml") if not (os.path.exists(product_yaml) and os.path.exists(build_config_yaml)): msg = (