support swagger resource provider with multiple readme files

src/aaz_dev/swagger/model/specs/_resource_provider.py

@@ -7,22 +7,22 @@
 
 import yaml
 
-from swagger.utils.tools import swagger_resource_path_to_resource_id
+from swagger.utils.tools import swagger_resource_path_to_resource_id, resolve_path_to_uri
 from ._resource import Resource, ResourceVersion
 from ._utils import map_path_2_repo
-
+from utils.readme_helper import parse_readme_file
 logger = logging.getLogger('backend')
 
 
 class OpenAPIResourceProvider:
 
-    def __init__(self, name, folder_path, readme_path, swagger_module):
+    def __init__(self, name, folder_path, readme_paths, swagger_module):
         self.name = name
         self.folder_path = folder_path
-        self._readme_path = readme_path
+        self._readme_paths = readme_paths
         self.swagger_module = swagger_module
 
-        if readme_path is None:
+        if not readme_paths:
             logger.warning(f"MissReadmeFile: {self} : {map_path_2_repo(folder_path)}")
         self._tags = None
         self._resource_map = None
@@ -74,6 +74,12 @@ def get_resource_map_by_tag(self, tag):
                     resource_map[resource.id][resource.version] = resource
         return resource_map
 
+    def load_readme_config(self, readme_file):
+        for readme_path in self._readme_paths:
+            if resolve_path_to_uri(readme_path) == readme_file:
+                return parse_readme_file(readme_path)['config']
+        return None
+
     @property
     def tags(self):
         if self._tags is None:
@@ -82,44 +88,45 @@ def tags(self):
 
     def _parse_readme_input_file_tags(self):
         tags = {}
-        if self._readme_path is None:
+        if not self._readme_paths:
             return tags
 
-        with open(self._readme_path, 'r', encoding='utf-8') as f:
-            readme = f.read()
-
-        re_yaml = re.compile(
-            r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n',
-            flags=re.MULTILINE)
-        for piece in re_yaml.finditer(readme):
-            flags = piece[1]
-            yaml_body = piece[3]
-            if 'input-file' not in yaml_body:
-                continue
+        for readme_path in self._readme_paths:
+            with open(readme_path, 'r', encoding='utf-8') as f:
+                readme = f.read()
+
+            re_yaml = re.compile(
+                r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n',
+                flags=re.MULTILINE)
+            for piece in re_yaml.finditer(readme):
+                flags = piece[1]
+                yaml_body = piece[3]
+                if 'input-file' not in yaml_body:
+                    continue
 
-            try:
-                body = yaml.safe_load(yaml_body)
-            except yaml.YAMLError as err:
-                logger.error(f'ParseYamlFailed: {self} : {self._readme_path} {flags}: {err}')
-                continue
-
-            files = []
-            for file_path in body['input-file']:
-                file_path = file_path.replace('$(this-folder)/', '')
-                file_path = os.path.join(os.path.dirname(self._readme_path), *file_path.split('/'))
-                if not os.path.isfile(file_path):
-                    logger.warning(f'FileNotExist: {self} : {file_path}')
+                try:
+                    body = yaml.safe_load(yaml_body)
+                except yaml.YAMLError as err:
+                    logger.error(f'ParseYamlFailed: {self} : {readme_path} {flags}: {err}')
                     continue
-                files.append(file_path)
-
-            if len(files):
-                tag = piece[2]
-                if tag is None:
-                    tag = ''
-                tag = OpenAPIResourceProviderTag(tag.strip(), self)
-                if tag not in tags:
-                    tags[tag] = set()
-                tags[tag] = tags[tag].union(files)
+
+                files = []
+                for file_path in body['input-file']:
+                    file_path = file_path.replace('$(this-folder)/', '')
+                    file_path = os.path.join(os.path.dirname(readme_path), *file_path.split('/'))
+                    if not os.path.isfile(file_path):
+                        logger.warning(f'FileNotExist: {self} : {file_path}')
+                        continue
+                    files.append(file_path)
+
+                if len(files):
+                    tag = piece[2]
+                    if tag is None:
+                        tag = ''
+                    tag = OpenAPIResourceProviderTag(tag.strip(), self)
+                    if tag not in tags:
+                        tags[tag] = set()
+                    tags[tag] = tags[tag].union(files)
 
         tags = [*tags.items()]
         tags.sort(key=lambda item: item[0].date, reverse=True)

src/aaz_dev/swagger/model/specs/_swagger_module.py

@@ -1,7 +1,8 @@
 import os
+import glob
 
 from utils.plane import PlaneEnum
-from utils.config import Config
+from swagger.utils.tools import resolve_path_to_uri
 from ._resource_provider import OpenAPIResourceProvider, TypeSpecResourceProvider
 from ._typespec_helper import TypeSpecHelper
 
@@ -36,17 +37,6 @@ def names(self):
             return [self.name]
         else:
             return [*self._parent.names, self.name]
-
-    @staticmethod
-    def resolve_path_to_uri(path):
-        relative_path = os.path.relpath(path, start=Config.get_swagger_root()).replace(os.sep, '/')
-        if relative_path.startswith('../'):
-            raise ValueError(f"Invalid path: {path}")
-        if relative_path.startswith('./'):
-            relative_path = relative_path[2:]
-        if relative_path.startswith('/'):
-            relative_path = relative_path[1:]
-        return relative_path
 
 
 class MgmtPlaneModule(SwaggerModule):
@@ -77,8 +67,8 @@ def _get_openapi_resource_providers(self):
             if os.path.isdir(path):
                 name_parts = name.split('.')
                 if len(name_parts) >= 2:
-                    readme_path = _search_readme_md_path(path, search_parent=True)
-                    rp.append(OpenAPIResourceProvider(name, path, readme_path, swagger_module=self))
+                    readme_paths = [*_search_readme_md_paths(path, search_parent=True)]
+                    rp.append(OpenAPIResourceProvider(name, path, readme_paths, swagger_module=self))
                 elif name.lower() != 'common':
                     # azsadmin module only
                     sub_module = MgmtPlaneModule(plane=self.plane, name=name, folder_path=path, parent=self)
@@ -91,7 +81,7 @@ def _get_typespec_resource_providers(self):
             return rp
 
         for namespace, ts_path, cfg_path in TypeSpecHelper.find_mgmt_plane_entry_files(self.folder_path):
-            entry_file = self.resolve_path_to_uri(ts_path)
+            entry_file = resolve_path_to_uri(ts_path)
             if namespace in rp:
                 rp[namespace].entry_files.append(entry_file)
             else:
@@ -133,8 +123,8 @@ def _get_openapi_resource_providers(self):
                     continue
                 name_parts = name.split('.')
                 if len(name_parts) >= 2:
-                    readme_path = _search_readme_md_path(path, search_parent=True)
-                    rp.append(OpenAPIResourceProvider(name, path, readme_path, swagger_module=self))
+                    readme_paths = [*_search_readme_md_paths(path, search_parent=True)]
+                    rp.append(OpenAPIResourceProvider(name, path, readme_paths, swagger_module=self))
                 elif name.lower() != 'common':
                     sub_module = DataPlaneModule(plane=self.plane, name=name, folder_path=path, parent=self)
                     rp.extend(sub_module.get_resource_providers())
@@ -146,7 +136,7 @@ def _get_typespec_resource_providers(self):
             return rp
 
         for namespace, ts_path, cfg_path in TypeSpecHelper.find_data_plane_entry_files(self.folder_path):
-            entry_file = self.resolve_path_to_uri(ts_path)
+            entry_file = resolve_path_to_uri(ts_path)
             if namespace in rp:
                 rp[namespace].entry_files.append(entry_file)
             else:
@@ -155,21 +145,13 @@ def _get_typespec_resource_providers(self):
         return [*rp.values()]
 
 
-def _search_readme_md_path(path, search_parent=False):
+def _search_readme_md_paths(path, search_parent=False):
+    # Check parent directory first if requested
     if search_parent:
-        readme_path = os.path.join(os.path.dirname(path), 'readme.md')
-        if os.path.exists(readme_path):
-            return readme_path
-
-    readme_path = os.path.join(path, 'readme.md')
-    if os.path.exists(readme_path):
-        return readme_path
-
-    # find in sub directory
-    for name in os.listdir(path):
-        sub_path = os.path.join(path, name)
-        if os.path.isdir(sub_path):
-            readme_path = _search_readme_md_path(sub_path)
-            if readme_path is not None:
-                return readme_path
-    return None
+        parent_readme = os.path.join(os.path.dirname(path), 'readme.md')
+        if os.path.isfile(parent_readme):
+            yield parent_readme
+
+    # Use glob to recursively find all readme.md files
+    pattern = os.path.join(path, '**', 'readme.md')
+    yield from glob.iglob(pattern, recursive=True)

src/aaz_dev/swagger/model/specs/_swagger_specs.py

@@ -12,12 +12,12 @@ def __init__(self, folder_path):
         self._folder_path = folder_path
 
     @property
-    def _spec_folder_path(self):
+    def spec_folder_path(self):
         return os.path.join(self._folder_path, 'specification')
 
     def get_mgmt_plane_modules(self, plane):
         modules = []
-        for name in os.listdir(self._spec_folder_path):
+        for name in os.listdir(self.spec_folder_path):
             module = self.get_mgmt_plane_module(name, plane=plane)
             if module:
                 modules.append(module)
@@ -30,7 +30,7 @@ def get_mgmt_plane_module(self, *names, plane):
         if not PlaneEnum.is_valid_swagger_module(plane=plane, module_name=name):
             return None
 
-        path = os.path.join(self._spec_folder_path, name)
+        path = os.path.join(self.spec_folder_path, name)
         if not os.path.isdir(path):
             return None
         if os.path.isdir(os.path.join(path, 'resource-manager')) or TypeSpecHelper.find_mgmt_plane_entry_files(path):
@@ -45,7 +45,7 @@ def get_mgmt_plane_module(self, *names, plane):
 
     def get_data_plane_modules(self, plane):
         modules = []
-        for name in os.listdir(self._spec_folder_path):
+        for name in os.listdir(self.spec_folder_path):
             module = self.get_data_plane_module(name, plane=plane)
             if module:
                 modules.append(module)
@@ -58,7 +58,7 @@ def get_data_plane_module(self, *names, plane):
         if not PlaneEnum.is_valid_swagger_module(plane=plane, module_name=name):
             return None
 
-        path = os.path.join(self._spec_folder_path, name)
+        path = os.path.join(self.spec_folder_path, name)
         if os.path.isdir(os.path.join(path, 'data-plane')) or TypeSpecHelper.find_data_plane_entry_files(path):
             module = DataPlaneModule(plane=plane, name=name, folder_path=path)
             for name in names[1:]:

src/aaz_dev/swagger/utils/tools.py

@@ -4,6 +4,8 @@
 # license information.
 # -----------------------------------------------------------------------------
 import re
+import os
+from utils.config import Config
 
 URL_PARAMETER_PLACEHOLDER = "{}"
 
@@ -39,3 +41,14 @@ def swagger_resource_path_to_resource_id(path):
         idx += 1
     path_parts[0] = "/".join(url_parts).lower()
     return "?".join(path_parts)
+
+
+def resolve_path_to_uri(path):
+    relative_path = os.path.relpath(path, start=Config.get_swagger_root()).replace(os.sep, '/')
+    if relative_path.startswith('../'):
+        raise ValueError(f"Invalid path: {path}")
+    if relative_path.startswith('./'):
+        relative_path = relative_path[2:]
+    if relative_path.startswith('/'):
+        relative_path = relative_path[1:]
+    return relative_path

src/aaz_dev/utils/readme_helper.py

@@ -0,0 +1,50 @@
+import yaml
+
+def _update_config(config, yaml_content):
+    for key, value in yaml_content.items():
+        if key not in config:
+            config[key] = value
+            continue
+        if isinstance(value, dict):
+            _update_config(config[key], value)
+        elif isinstance(value, list):
+            config[key].extend(value)
+        else:
+            config[key] = value
+
+def parse_readme_file(readme_path: str):
+    """Parse the readme file title and combine basic config in the yaml section."""
+    readme_config = {}
+    with open(readme_path, 'r', encoding='utf-8') as f:
+        content = f.readlines()
+    # content.append("```") # append a fake yaml section to make sure the last yaml section is ended
+    readme_title = None
+    in_yaml_section = False
+    yaml_content = []
+    for line in content:
+        if not readme_title and line.strip().startswith("# ") and not in_yaml_section:
+            readme_title = line.strip()[2:].strip()
+        if line.strip().startswith("```") and 'yaml' in line:
+            condition = line.split('yaml')[1].strip()
+            # Do not parse the yaml section if it has the condition
+            if not condition:
+                in_yaml_section = True
+        elif in_yaml_section:
+            if line.strip().startswith("```"):
+                try:
+                    yaml_config = yaml.load("\n".join(yaml_content), Loader=yaml.FullLoader)
+                except Exception as e:
+                    raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_path}")
+                _update_config(readme_config, yaml_config)
+                in_yaml_section = False
+                yaml_content = []
+            else:
+                if line.strip():
+                    yaml_content.append(line)
+                else:
+                    yaml_content.append("")
+
+    return {
+        "title": readme_title,
+        "config": readme_config
+    }