scripts: edt: Add support for include property filtering

Add the ability to filter which properties get imported when we do an
include.  We add a new YAML form for this:

include:
  - name: other.yaml
    property-blocklist:
      - prop-to-block

or

include:
  - name: other.yaml
    property-allowlist:
      - prop-to-allow

These lists can intermix simple file names with maps, like:

include:
  - foo.yaml
  - name: bar.yaml
    property-allowlist:
      - prop-to-allow

And you can filter from child bindings like this:

include:
  - name: bar.yaml
    child-binding:
      property-allowlist:
        - child-prop-to-allow

Signed-off-by: Kumar Gala <[email protected]>
Signed-off-by: Martí Bolívar <[email protected]>

dts/binding-template.yaml

@@ -60,7 +60,50 @@ compatible: "manufacturer,device"
 # When including multiple files, any overlapping 'required' keys on properties
 # in the included files are ORed together. This makes sure that a
 # 'required: true' is always respected.
-include: other.yaml # or [other1.yaml, other2.yaml]
+#
+# When 'include:' is a list, its elements can be either filenames as
+# strings, or maps. Each map element must have a 'name' key which is
+# the filename to include, and may have 'property-allowlist' and
+# 'property-blocklist' keys that filter which properties are included.
+# You can freely intermix strings and maps in a single 'include:' list.
+# You cannot have a single map element with both 'property-allowlist' and
+# 'property-blocklist' keys. A map element with neither 'property-allowlist'
+# nor 'property-blocklist' is valid; no additional filtering is done.
+include: other.yaml
+# To include multiple files:
+#
+# include:
+#   - other1.yaml
+#   - other2.yaml
+#
+# To filter the included properties:
+#
+# include:
+#   - name: other1.yaml
+#     property-allowlist:
+#       - i-want-this-one
+#       - and-this-one
+#   - other2.yaml
+#     property-blocklist:
+#       - do-not-include-this-one
+#       - or-this-one
+#
+# You can intermix these types of includes:
+#
+# include:
+#   - other1.yaml
+#   - other2.yaml
+#     property-blocklist:
+#       - do-not-include-this-one
+#       - or-this-one
+#
+# And you can filter from a child binding like this:
+#
+# include:
+#   - name: bar.yaml
+#     child-binding:
+#       property-allowlist:
+#         - child-prop-to-allow
 
 # If the node describes a bus, then the bus type should be given, like below
 bus: <string describing bus type, e.g. "i2c">

scripts/dts/python-devicetree/src/devicetree/edtlib.py

@@ -68,6 +68,7 @@
 #   variables. See the existing @properties for a template.
 
 from collections import OrderedDict, defaultdict
+from copy import deepcopy
 import logging
 import os
 import re
@@ -1607,26 +1608,51 @@ def _merge_includes(self, raw, binding_path):
             return raw
 
         include = raw.pop("include")
-        fnames = []
-        if isinstance(include, str):
-            fnames.append(include)
-        elif isinstance(include, list):
-            if not all(isinstance(elem, str) for elem in include):
-                _err(f"all elements in 'include:' in {binding_path} "
-                     "should be strings")
-            fnames += include
-        else:
-            _err(f"'include:' in {binding_path} "
-                 "should be a string or a list of strings")
 
         # First, merge the included files together. If more than one included
         # file has a 'required:' for a particular property, OR the values
         # together, so that 'required: true' wins.
 
         merged = {}
-        for fname in fnames:
-            _merge_props(merged, self._load_raw(fname), None, binding_path,
-                         check_required=False)
+
+        if isinstance(include, str):
+            # Simple scalar string case
+            _merge_props(merged, self._load_raw(include), None, binding_path,
+                         False)
+        elif isinstance(include, list):
+            # List of strings and maps. These types may be intermixed.
+            for elem in include:
+                if isinstance(elem, str):
+                    _merge_props(merged, self._load_raw(elem), None,
+                                 binding_path, False)
+                elif isinstance(elem, dict):
+                    name = elem.pop('name', None)
+                    allowlist = elem.pop('property-allowlist', None)
+                    blocklist = elem.pop('property-blocklist', None)
+                    child_filter = elem.pop('child-binding', None)
+
+                    if elem:
+                        # We've popped out all the valid keys.
+                        _err(f"'include:' in {binding_path} should not have "
+                             f"these unexpected contents: {elem}")
+
+                    _check_include_dict(name, allowlist, blocklist,
+                                        child_filter, binding_path)
+
+                    contents = self._load_raw(name)
+
+                    _filter_properties(contents, allowlist, blocklist,
+                                       child_filter, binding_path)
+                    _merge_props(merged, contents, None, binding_path, False)
+                else:
+                    _err(f"all elements in 'include:' in {binding_path} "
+                         "should be either strings or maps with a 'name' key "
+                         "and optional 'property-allowlist' or "
+                         f"'property-blocklist' keys, but got: {elem}")
+        else:
+            # Invalid item.
+            _err(f"'include:' in {binding_path} "
+                 f"should be a string or list, but has type {type(include)}")
 
         # Next, merge the merged included files into 'raw'. Error out if
         # 'raw' has 'required: false' while the merged included files have
@@ -1952,6 +1978,89 @@ def _binding_inc_error(msg):
     raise yaml.constructor.ConstructorError(None, None, "error: " + msg)
 
 
+def _check_include_dict(name, allowlist, blocklist, child_filter,
+                        binding_path):
+    # Check that an 'include:' named 'name' with property-allowlist
+    # 'allowlist', property-blocklist 'blocklist', and
+    # child-binding filter 'child_filter' has valid structure.
+
+    if name is None:
+        _err(f"'include:' element in {binding_path} "
+             "should have a 'name' key")
+
+    if allowlist is not None and blocklist is not None:
+        _err(f"'include:' of file '{name}' in {binding_path} "
+             "should not specify both 'property-allowlist:' "
+             "and 'property-blocklist:'")
+
+    while child_filter is not None:
+        child_copy = deepcopy(child_filter)
+        child_allowlist = child_copy.pop('property-allowlist', None)
+        child_blocklist = child_copy.pop('property-blocklist', None)
+        next_child_filter = child_copy.pop('child-binding', None)
+
+        if child_copy:
+            # We've popped out all the valid keys.
+            _err(f"'include:' of file '{name}' in {binding_path} "
+                 "should not have these unexpected contents in a "
+                 f"'child-binding': {child_copy}")
+
+        if child_allowlist is not None and child_blocklist is not None:
+            _err(f"'include:' of file '{name}' in {binding_path} "
+                 "should not specify both 'property-allowlist:' and "
+                 "'property-blocklist:' in a 'child-binding:'")
+
+        child_filter = next_child_filter
+
+
+def _filter_properties(raw, allowlist, blocklist, child_filter,
+                       binding_path):
+    # Destructively modifies 'raw["properties"]' and
+    # 'raw["child-binding"]', if they exist, according to
+    # 'allowlist', 'blocklist', and 'child_filter'.
+
+    props = raw.get('properties')
+    _filter_properties_helper(props, allowlist, blocklist, binding_path)
+
+    child_binding = raw.get('child-binding')
+    while child_filter is not None and child_binding is not None:
+        _filter_properties_helper(child_binding.get('properties'),
+                                  child_filter.get('property-allowlist'),
+                                  child_filter.get('property-blocklist'),
+                                  binding_path)
+        child_filter = child_filter.get('child-binding')
+        child_binding = child_binding.get('child-binding')
+
+
+def _filter_properties_helper(props, allowlist, blocklist, binding_path):
+    if props is None or (allowlist is None and blocklist is None):
+        return
+
+    _check_prop_filter('property-allowlist', allowlist, binding_path)
+    _check_prop_filter('property-blocklist', blocklist, binding_path)
+
+    if allowlist is not None:
+        allowset = set(allowlist)
+        to_del = [prop for prop in props if prop not in allowset]
+    else:
+        blockset = set(blocklist)
+        to_del = [prop for prop in props if prop in blockset]
+
+    for prop in to_del:
+        del props[prop]
+
+
+def _check_prop_filter(name, value, binding_path):
+    # Ensure an include: ... property-allowlist or property-blocklist
+    # is a list.
+
+    if value is None:
+        return
+
+    if not isinstance(value, list):
+        _err(f"'{name}' value {value} in {binding_path} should be a list")
+
+
 def _merge_props(to_dict, from_dict, parent, binding_path, check_required):
     # Recursively merges 'from_dict' into 'to_dict', to implement 'include:'.
     #

scripts/dts/python-devicetree/tests/test-bindings-include/README.rst

@@ -0,0 +1 @@
+This directory contains bindings used to test the 'include:' feature.

scripts/dts/python-devicetree/tests/test-bindings-include/allow-and-blocklist-child.yaml

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: |
+  An include must not give both an allowlist and a blocklist in a
+  child binding. This binding should cause an error.
+compatible: allow-and-blocklist-child
+include:
+  - name: include.yaml
+    child-binding:
+      property-blocklist: [x]
+      property-allowlist: [y]

scripts/dts/python-devicetree/tests/test-bindings-include/allow-and-blocklist.yaml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: |
+  An include must not give both an allowlist and a blocklist.
+  This binding should cause an error.
+compatible: allow-and-blocklist
+include:
+  - name: include.yaml
+    property-blocklist: [x]
+    property-allowlist: [y]

scripts/dts/python-devicetree/tests/test-bindings-include/allow-not-list.yaml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: |
+  A property-allowlist, if given, must be a list. This binding should
+  cause an error.
+compatible: allow-not-list
+include:
+  - name: include.yaml
+    property-allowlist:
+      foo:

scripts/dts/python-devicetree/tests/test-bindings-include/allowlist.yaml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: Valid property-allowlist.
+compatible: allowlist
+include:
+  - name: include.yaml
+    property-allowlist: [x]

scripts/dts/python-devicetree/tests/test-bindings-include/block-not-list.yaml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: |
+  A property-blocklist, if given, must be a list. This binding should
+  cause an error.
+compatible: block-not-list
+include:
+  - name: include.yaml
+    property-blocklist:
+      foo:

scripts/dts/python-devicetree/tests/test-bindings-include/blocklist.yaml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: Valid property-blocklist.
+compatible: blocklist
+include:
+  - name: include.yaml
+    property-blocklist: [x]

scripts/dts/python-devicetree/tests/test-bindings-include/empty-allowlist.yaml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: An empty property-allowlist is valid.
+compatible: empty-allowlist
+include:
+  - name: include.yaml
+    property-allowlist: []

scripts/dts/python-devicetree/tests/test-bindings-include/empty-blocklist.yaml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: An empty property-blocklist is valid.
+compatible: empty-blocklist
+include:
+  - name: include.yaml
+    property-blocklist: []

scripts/dts/python-devicetree/tests/test-bindings-include/filter-child-bindings.yaml

@@ -0,0 +1,11 @@
+description: Test binding for filtering 'child-binding' properties
+
+include:
+  - name: include.yaml
+    property-allowlist: [x]
+    child-binding:
+      property-blocklist: [child-prop-1]
+      child-binding:
+        property-allowlist: [grandchild-prop-1]
+
+compatible: filter-child-bindings

scripts/dts/python-devicetree/tests/test-bindings-include/include-2.yaml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: Second file for testing "intermixed" includes.
+compatible: include-2
+properties:
+  a:
+    type: int

scripts/dts/python-devicetree/tests/test-bindings-include/include-invalid-keys.yaml

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: |
+  Invalid include element: invalid keys are present.
+compatible: include-invalid-keys
+include:
+  - name: include.yaml
+    property-allowlist: [x]
+    bad-key-1: 3
+    bad-key-2: 3

scripts/dts/python-devicetree/tests/test-bindings-include/include-invalid-type.yaml

@@ -0,0 +1,5 @@
+description: |
+  Invalid include: wrong top level type.
+compatible: include-invalid-type
+include:
+  a-map-is-not-allowed-here: 3

scripts/dts/python-devicetree/tests/test-bindings-include/include-no-list.yaml

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: A map element with just a name is valid, and has no filters.
+compatible: include-no-list
+include:
+  - name: include.yaml

scripts/dts/python-devicetree/tests/test-bindings-include/include-no-name.yaml

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: |
+  Invalid include element: no name key is present.
+compatible: include-no-name
+include:
+  - property-allowlist: [x]

scripts/dts/python-devicetree/tests/test-bindings-include/include.yaml

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: Test file for including other bindings
+compatible: include
+properties:
+  x:
+    type: int
+  y:
+    type: int
+  z:
+    type: int
+child-binding:
+  properties:
+    child-prop-1:
+      type: int
+    child-prop-2:
+      type: int
+
+  child-binding:
+    properties:
+      grandchild-prop-1:
+        type: int
+      grandchild-prop-2:
+        type: int

scripts/dts/python-devicetree/tests/test-bindings-include/intermixed.yaml

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+description: Including intermixed file names and maps is valid.
+compatible: intermixed
+include:
+  - name: include.yaml
+    property-allowlist: [x]
+  - include-2.yaml