Implement semantic markup support for Ansible documentation.

lib/ansible/cli/doc.py

@@ -362,12 +362,23 @@ class DocCLI(CLI, RoleMixin):
     _ITALIC = re.compile(r"\bI\(([^)]+)\)")
     _BOLD = re.compile(r"\bB\(([^)]+)\)")
     _MODULE = re.compile(r"\bM\(([^)]+)\)")
+    _PLUGIN = re.compile(r"\bP\(([^#)]+)#([a-z]+)\)")
     _LINK = re.compile(r"\bL\(([^)]+), *([^)]+)\)")
     _URL = re.compile(r"\bU\(([^)]+)\)")
     _REF = re.compile(r"\bR\(([^)]+), *([^)]+)\)")
     _CONST = re.compile(r"\bC\(([^)]+)\)")
+    _SEM_PARAMETER_STRING = r"\(((?:[^\\)]+|\\.)+)\)"
+    _SEM_OPTION_NAME = re.compile(r"\bO" + _SEM_PARAMETER_STRING)
+    _SEM_OPTION_VALUE = re.compile(r"\bV" + _SEM_PARAMETER_STRING)
+    _SEM_ENV_VARIABLE = re.compile(r"\bE" + _SEM_PARAMETER_STRING)
+    _SEM_RET_VALUE = re.compile(r"\bRV" + _SEM_PARAMETER_STRING)
     _RULER = re.compile(r"\bHORIZONTALLINE\b")
 
+    # helper for unescaping
+    _UNESCAPE = re.compile(r"\\(.)")
+    _FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$')
+    _IGNORE_MARKER = 'ignore:'
+
     # rst specific
     _RST_NOTE = re.compile(r".. note::")
     _RST_SEEALSO = re.compile(r".. seealso::")
@@ -379,6 +390,33 @@ def __init__(self, args):
         super(DocCLI, self).__init__(args)
         self.plugin_list = set()
 
+    @staticmethod
+    def _tty_ify_sem_simle(matcher):
+        text = DocCLI._UNESCAPE.sub(r'\1', matcher.group(1))
+        return f"`{text}'"
+
+    @staticmethod
+    def _tty_ify_sem_complex(matcher):
+        text = DocCLI._UNESCAPE.sub(r'\1', matcher.group(1))
+        value = None
+        if '=' in text:
+            text, value = text.split('=', 1)
+        m = DocCLI._FQCN_TYPE_PREFIX_RE.match(text)
+        if m:
+            plugin_fqcn = m.group(1)
+            plugin_type = m.group(2)
+            text = m.group(3)
+        elif text.startswith(DocCLI._IGNORE_MARKER):
+            text = text[len(DocCLI._IGNORE_MARKER):]
+            plugin_fqcn = plugin_type = ''
+        else:
+            plugin_fqcn = plugin_type = ''
+        if value is not None:
+            text = f"{text}={value}"
+        if plugin_fqcn and plugin_type:
+            return f"`{text}' (of {plugin_type} {plugin_fqcn})"
+        return f"`{text}'"
+
     @classmethod
     def find_plugins(cls, path, internal, plugin_type, coll_filter=None):
         display.deprecated("find_plugins method as it is incomplete/incorrect. use ansible.plugins.list functions instead.", version='2.17')
@@ -393,8 +431,13 @@ def tty_ify(cls, text):
         t = cls._MODULE.sub("[" + r"\1" + "]", t)       # M(word) => [word]
         t = cls._URL.sub(r"\1", t)                      # U(word) => word
         t = cls._LINK.sub(r"\1 <\2>", t)                # L(word, url) => word <url>
+        t = cls._PLUGIN.sub("[" + r"\1" + "]", t)       # P(word#type) => [word]
         t = cls._REF.sub(r"\1", t)            # R(word, sphinx-ref) => word
         t = cls._CONST.sub(r"`\1'", t)        # C(word) => `word'
+        t = cls._SEM_OPTION_NAME.sub(cls._tty_ify_sem_complex, t)  # O(expr)
+        t = cls._SEM_OPTION_VALUE.sub(cls._tty_ify_sem_simle, t)  # V(expr)
+        t = cls._SEM_ENV_VARIABLE.sub(cls._tty_ify_sem_simle, t)  # E(expr)
+        t = cls._SEM_RET_VALUE.sub(cls._tty_ify_sem_complex, t)  # RV(expr)
         t = cls._RULER.sub("\n{0}\n".format("-" * 13), t)   # HORIZONTALLINE => -------
 
         # remove rst

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py

@@ -65,6 +65,7 @@ def setup_collection_loader():
 
 from ansible import __version__ as ansible_version
 from ansible.executor.module_common import REPLACER_WINDOWS, NEW_STYLE_PYTHON_MODULE_RE
+from ansible.module_utils.common.collections import is_iterable
 from ansible.module_utils.common.parameters import DEFAULT_TYPE_VALIDATORS
 from ansible.module_utils.compat.version import StrictVersion, LooseVersion
 from ansible.module_utils.basic import to_bytes
@@ -76,7 +77,15 @@ def setup_collection_loader():
 
 from .module_args import AnsibleModuleImportError, AnsibleModuleNotInitialized, get_argument_spec
 
-from .schema import ansible_module_kwargs_schema, doc_schema, return_schema
+from .schema import (
+    ansible_module_kwargs_schema,
+    doc_schema,
+    return_schema,
+    _SEM_OPTION_NAME,
+    _SEM_RET_VALUE,
+    _check_sem_quoting,
+    _parse_prefix,
+)
 
 from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, parse_yaml, parse_isodate
 
@@ -1068,6 +1077,8 @@ def _validate_docs(self):
                     'invalid-documentation',
                 )
 
+            self._validate_all_semantic_markup(doc, returns)
+
             if not self.collection:
                 existing_doc = self._check_for_new_args(doc)
                 self._check_version_added(doc, existing_doc)
@@ -1193,6 +1204,112 @@ def _validate_docs(self):
 
         return doc_info, doc
 
+    def _check_sem_option(self, directive, content):
+        try:
+            content = _check_sem_quoting(directive, content)
+            plugin_fqcn, plugin_type, option_link, option, value = _parse_prefix(directive, content)
+        except Exception:
+            # Validation errors have already been covered in the schema check
+            return
+        if plugin_fqcn is not None:
+            return
+        if tuple(option_link) not in self._all_options:
+            self.reporter.error(
+                path=self.object_path,
+                code='invalid-documentation-markup',
+                msg='Directive "%s" contains a non-existing option "%s"' % (directive, option)
+            )
+
+    def _check_sem_return_value(self, directive, content):
+        try:
+            content = _check_sem_quoting(directive, content)
+            plugin_fqcn, plugin_type, rv_link, rv, value = _parse_prefix(directive, content)
+        except Exception:
+            # Validation errors have already been covered in the schema check
+            return
+        if plugin_fqcn is not None:
+            return
+        if tuple(rv_link) not in self._all_return_values:
+            self.reporter.error(
+                path=self.object_path,
+                code='invalid-documentation-markup',
+                msg='Directive "%s" contains a non-existing return value "%s"' % (directive, rv)
+            )
+
+    def _validate_semantic_markup(self, object):
+        # Make sure we operate on strings
+        if is_iterable(object):
+            for entry in object:
+                self._validate_semantic_markup(entry)
+            return
+        if not isinstance(object, string_types):
+            return
+
+        for m in _SEM_OPTION_NAME.finditer(object):
+            self._check_sem_option(m.group(0), m.group(1))
+        for m in _SEM_RET_VALUE.finditer(object):
+            self._check_sem_return_value(m.group(0), m.group(1))
+
+    def _validate_semantic_markup_collect(self, destination, sub_key, data, all_paths):
+        if not isinstance(data, dict):
+            return
+        for key, value in data.items():
+            if not isinstance(value, dict):
+                continue
+            keys = {key}
+            if is_iterable(value.get('aliases')):
+                keys.update(value['aliases'])
+            new_paths = [path + [key] for path in all_paths for key in keys]
+            destination.update([tuple(path) for path in new_paths])
+            self._validate_semantic_markup_collect(destination, sub_key, value.get(sub_key), new_paths)
+
+    def _validate_semantic_markup_options(self, options):
+        if not isinstance(options, dict):
+            return
+        for key, value in options.items():
+            self._validate_semantic_markup(value.get('description'))
+            self._validate_semantic_markup_options(value.get('suboptions'))
+
+    def _validate_semantic_markup_return_values(self, return_vars):
+        if not isinstance(return_vars, dict):
+            return
+        for key, value in return_vars.items():
+            self._validate_semantic_markup(value.get('description'))
+            self._validate_semantic_markup(value.get('returned'))
+            self._validate_semantic_markup_return_values(value.get('contains'))
+
+    def _validate_all_semantic_markup(self, docs, return_docs):
+        if not isinstance(docs, dict):
+            docs = {}
+        if not isinstance(return_docs, dict):
+            return_docs = {}
+
+        self._all_options = set()
+        self._all_return_values = set()
+        self._validate_semantic_markup_collect(self._all_options, 'suboptions', docs.get('options'), [[]])
+        self._validate_semantic_markup_collect(self._all_return_values, 'contains', return_docs, [[]])
+
+        for string_keys in ('short_description', 'description', 'notes', 'requirements', 'todo'):
+            self._validate_semantic_markup(docs.get(string_keys))
+
+        if is_iterable(docs.get('seealso')):
+            for entry in docs.get('seealso'):
+                if isinstance(entry, dict):
+                    self._validate_semantic_markup(entry.get('description'))
+
+        if isinstance(docs.get('attributes'), dict):
+            for entry in docs.get('attributes').values():
+                if isinstance(entry, dict):
+                    for key in ('description', 'details'):
+                        self._validate_semantic_markup(entry.get(key))
+
+        if isinstance(docs.get('deprecated'), dict):
+            for key in ('why', 'alternative'):
+                self._validate_semantic_markup(docs.get('deprecated').get(key))
+
+        self._validate_semantic_markup_options(docs.get('options'))
+        self._validate_semantic_markup_return_values(return_docs)
+
     def _check_version_added(self, doc, existing_doc):
         version_added_raw = doc.get('version_added')
         try:

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py

@@ -12,6 +12,7 @@
 from urllib.parse import urlparse
 
 from voluptuous import ALLOW_EXTRA, PREVENT_EXTRA, All, Any, Invalid, Length, Required, Schema, Self, ValueInvalid, Exclusive
+from ansible.constants import DOCUMENTABLE_PLUGINS
 from ansible.module_utils.six import string_types
 from ansible.module_utils.common.collections import is_iterable
 from ansible.module_utils.parsing.convert_bool import boolean
@@ -81,17 +82,48 @@ def date(error_code=None):
 
 
 _MODULE = re.compile(r"\bM\(([^)]+)\)")
+_PLUGIN = re.compile(r"\bP\(([^)]+)\)")
 _LINK = re.compile(r"\bL\(([^)]+)\)")
 _URL = re.compile(r"\bU\(([^)]+)\)")
 _REF = re.compile(r"\bR\(([^)]+)\)")
 
+_SEM_PARAMETER_STRING = r"\(((?:[^\\)]+|\\.)+)\)"
+_SEM_OPTION_NAME = re.compile(r"\bO" + _SEM_PARAMETER_STRING)
+_SEM_OPTION_VALUE = re.compile(r"\bV" + _SEM_PARAMETER_STRING)
+_SEM_ENV_VARIABLE = re.compile(r"\bE" + _SEM_PARAMETER_STRING)
+_SEM_RET_VALUE = re.compile(r"\bRV" + _SEM_PARAMETER_STRING)
+
+_UNESCAPE = re.compile(r"\\(.)")
+_CONTENT_LINK_SPLITTER_RE = re.compile(r'(?:\[[^\]]*\])?\.')
+_CONTENT_LINK_END_STUB_RE = re.compile(r'\[[^\]]*\]$')
+_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$')
+_IGNORE_MARKER = 'ignore:'
+_IGNORE_STRING = '(ignore)'
+
+_VALID_PLUGIN_TYPES = set(DOCUMENTABLE_PLUGINS)
+
 
 def _check_module_link(directive, content):
     if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(content):
         raise _add_ansible_error_code(
             Invalid('Directive "%s" must contain a FQCN' % directive), 'invalid-documentation-markup')
 
 
+def _check_plugin_link(directive, content):
+    if '#' not in content:
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" must contain a "#"' % directive), 'invalid-documentation-markup')
+    plugin_fqcn, plugin_type = content.split('#', 1)
+    if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(plugin_fqcn):
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" must contain a FQCN; found "%s"' % (directive, plugin_fqcn)),
+            'invalid-documentation-markup')
+    if plugin_type not in _VALID_PLUGIN_TYPES:
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" must contain a valid plugin type; found "%s"' % (directive, plugin_type)),
+            'invalid-documentation-markup')
+
+
 def _check_link(directive, content):
     if ',' not in content:
         raise _add_ansible_error_code(
@@ -118,19 +150,81 @@ def _check_ref(directive, content):
             Invalid('Directive "%s" must contain a comma' % directive), 'invalid-documentation-markup')
 
 
+def _check_sem_quoting(directive, content):
+    for m in _UNESCAPE.finditer(content):
+        if m.group(1) not in ('\\', ')'):
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" contains unnecessarily quoted "%s"' % (directive, m.group(1))),
+                'invalid-documentation-markup')
+    return _UNESCAPE.sub(r'\1', content)
+
+
+def _parse_prefix(directive, content):
+    value = None
+    if '=' in content:
+        content, value = content.split('=', 1)
+    m = _FQCN_TYPE_PREFIX_RE.match(content)
+    if m:
+        plugin_fqcn = m.group(1)
+        plugin_type = m.group(2)
+        content = m.group(3)
+        if not FULLY_QUALIFIED_COLLECTION_RESOURCE_RE.match(plugin_fqcn):
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" must contain a FQCN; found "%s"' % (directive, plugin_fqcn)),
+                'invalid-documentation-markup')
+        if plugin_type not in _VALID_PLUGIN_TYPES:
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" must contain a valid plugin type; found "%s"' % (directive, plugin_type)),
+                'invalid-documentation-markup')
+    elif content.startswith(_IGNORE_MARKER):
+        content = content[len(_IGNORE_MARKER):]
+        plugin_fqcn = plugin_type = _IGNORE_STRING
+    else:
+        plugin_fqcn = plugin_type = None
+    if ':' in content or '#' in content:
+        raise _add_ansible_error_code(
+            Invalid('Directive "%s" contains wrongly specified FQCN/plugin type' % directive),
+            'invalid-documentation-markup')
+    content_link = _CONTENT_LINK_SPLITTER_RE.split(content)
+    for i, part in enumerate(content_link):
+        if i == len(content_link) - 1:
+            part = _CONTENT_LINK_END_STUB_RE.sub('', part)
+            content_link[i] = part
+        if '.' in part or '[' in part or ']' in part:
+            raise _add_ansible_error_code(
+                Invalid('Directive "%s" contains invalid name "%s"' % (directive, content)),
+                'invalid-documentation-markup')
+    return plugin_fqcn, plugin_type, content_link, content, value
+
+
+def _check_sem_option_return_value(directive, content):
+    content = _check_sem_quoting(directive, content)
+    _parse_prefix(directive, content)
+
+
 def doc_string(v):
     """Match a documentation string."""
     if not isinstance(v, string_types):
         raise _add_ansible_error_code(
             Invalid('Must be a string'), 'invalid-documentation')
     for m in _MODULE.finditer(v):
         _check_module_link(m.group(0), m.group(1))
+    for m in _PLUGIN.finditer(v):
+        _check_plugin_link(m.group(0), m.group(1))
     for m in _LINK.finditer(v):
         _check_link(m.group(0), m.group(1))
     for m in _URL.finditer(v):
         _check_url(m.group(0), m.group(1))
     for m in _REF.finditer(v):
         _check_ref(m.group(0), m.group(1))
+    for m in _SEM_OPTION_NAME.finditer(v):
+        _check_sem_option_return_value(m.group(0), m.group(1))
+    for m in _SEM_OPTION_VALUE.finditer(v):
+        _check_sem_quoting(m.group(0), m.group(1))
+    for m in _SEM_ENV_VARIABLE.finditer(v):
+        _check_sem_quoting(m.group(0), m.group(1))
+    for m in _SEM_RET_VALUE.finditer(v):
+        _check_sem_option_return_value(m.group(0), m.group(1))
     return v
 
 
@@ -172,6 +266,11 @@ def sequence_of_sequences(min=None, max=None):
                 Required('module'): Any(*string_types),
                 'description': doc_string,
             },
+            {
+                Required('plugin'): Any(*string_types),
+                Required('plugin_type'): Any(*DOCUMENTABLE_PLUGINS),
+                'description': doc_string,
+            },
             {
                 Required('ref'): Any(*string_types),
                 Required('description'): doc_string,