From fa549b7153f06bada519d753a195dfadee4c3968 Mon Sep 17 00:00:00 2001 From: kai ru Date: Fri, 27 Dec 2024 11:01:26 +0800 Subject: [PATCH 1/2] swagger rp support multiple readme files --- .../swagger/model/specs/_resource_provider.py | 85 ++++++++++--------- .../swagger/model/specs/_swagger_module.py | 52 ++++-------- .../swagger/model/specs/_swagger_specs.py | 10 +-- src/aaz_dev/swagger/utils/tools.py | 13 +++ src/aaz_dev/utils/readme_helper.py | 50 +++++++++++ 5 files changed, 131 insertions(+), 79 deletions(-) create mode 100644 src/aaz_dev/utils/readme_helper.py diff --git a/src/aaz_dev/swagger/model/specs/_resource_provider.py b/src/aaz_dev/swagger/model/specs/_resource_provider.py index 20e85578..a2b91dfd 100644 --- a/src/aaz_dev/swagger/model/specs/_resource_provider.py +++ b/src/aaz_dev/swagger/model/specs/_resource_provider.py @@ -7,22 +7,22 @@ import yaml -from swagger.utils.tools import swagger_resource_path_to_resource_id +from swagger.utils.tools import swagger_resource_path_to_resource_id, resolve_path_to_uri from ._resource import Resource, ResourceVersion from ._utils import map_path_2_repo - +from utils.readme_helper import parse_readme_file logger = logging.getLogger('backend') class OpenAPIResourceProvider: - def __init__(self, name, folder_path, readme_path, swagger_module): + def __init__(self, name, folder_path, readme_paths, swagger_module): self.name = name self.folder_path = folder_path - self._readme_path = readme_path + self._readme_paths = readme_paths self.swagger_module = swagger_module - if readme_path is None: + if not readme_paths: logger.warning(f"MissReadmeFile: {self} : {map_path_2_repo(folder_path)}") self._tags = None self._resource_map = None @@ -74,6 +74,12 @@ def get_resource_map_by_tag(self, tag): resource_map[resource.id][resource.version] = resource return resource_map + def load_readme_config(self, readme_file): + for readme_path in self._readme_paths: + if resolve_path_to_uri(readme_path) == readme_file: + return parse_readme_file(readme_path)['config'] + return None + @property def tags(self): if self._tags is None: @@ -82,44 +88,45 @@ def tags(self): def _parse_readme_input_file_tags(self): tags = {} - if self._readme_path is None: + if not self._readme_paths: return tags - with open(self._readme_path, 'r', encoding='utf-8') as f: - readme = f.read() - - re_yaml = re.compile( - r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n', - flags=re.MULTILINE) - for piece in re_yaml.finditer(readme): - flags = piece[1] - yaml_body = piece[3] - if 'input-file' not in yaml_body: - continue + for readme_path in self._readme_paths: + with open(readme_path, 'r', encoding='utf-8') as f: + readme = f.read() + + re_yaml = re.compile( + r'```\s*yaml\s*(.*\$\(\s*tag\s*\)\s*==\s*[\'"]\s*(.*)\s*[\'"].*)?\n((((?!```).)*\n)*)```\s*\n', + flags=re.MULTILINE) + for piece in re_yaml.finditer(readme): + flags = piece[1] + yaml_body = piece[3] + if 'input-file' not in yaml_body: + continue - try: - body = yaml.safe_load(yaml_body) - except yaml.YAMLError as err: - logger.error(f'ParseYamlFailed: {self} : {self._readme_path} {flags}: {err}') - continue - - files = [] - for file_path in body['input-file']: - file_path = file_path.replace('$(this-folder)/', '') - file_path = os.path.join(os.path.dirname(self._readme_path), *file_path.split('/')) - if not os.path.isfile(file_path): - logger.warning(f'FileNotExist: {self} : {file_path}') + try: + body = yaml.safe_load(yaml_body) + except yaml.YAMLError as err: + logger.error(f'ParseYamlFailed: {self} : {readme_path} {flags}: {err}') continue - files.append(file_path) - - if len(files): - tag = piece[2] - if tag is None: - tag = '' - tag = OpenAPIResourceProviderTag(tag.strip(), self) - if tag not in tags: - tags[tag] = set() - tags[tag] = tags[tag].union(files) + + files = [] + for file_path in body['input-file']: + file_path = file_path.replace('$(this-folder)/', '') + file_path = os.path.join(os.path.dirname(readme_path), *file_path.split('/')) + if not os.path.isfile(file_path): + logger.warning(f'FileNotExist: {self} : {file_path}') + continue + files.append(file_path) + + if len(files): + tag = piece[2] + if tag is None: + tag = '' + tag = OpenAPIResourceProviderTag(tag.strip(), self) + if tag not in tags: + tags[tag] = set() + tags[tag] = tags[tag].union(files) tags = [*tags.items()] tags.sort(key=lambda item: item[0].date, reverse=True) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_module.py b/src/aaz_dev/swagger/model/specs/_swagger_module.py index 66007747..76ff8e8b 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_module.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_module.py @@ -1,7 +1,8 @@ import os +import glob from utils.plane import PlaneEnum -from utils.config import Config +from swagger.utils.tools import resolve_path_to_uri from ._resource_provider import OpenAPIResourceProvider, TypeSpecResourceProvider from ._typespec_helper import TypeSpecHelper @@ -36,17 +37,6 @@ def names(self): return [self.name] else: return [*self._parent.names, self.name] - - @staticmethod - def resolve_path_to_uri(path): - relative_path = os.path.relpath(path, start=Config.get_swagger_root()).replace(os.sep, '/') - if relative_path.startswith('../'): - raise ValueError(f"Invalid path: {path}") - if relative_path.startswith('./'): - relative_path = relative_path[2:] - if relative_path.startswith('/'): - relative_path = relative_path[1:] - return relative_path class MgmtPlaneModule(SwaggerModule): @@ -77,8 +67,8 @@ def _get_openapi_resource_providers(self): if os.path.isdir(path): name_parts = name.split('.') if len(name_parts) >= 2: - readme_path = _search_readme_md_path(path, search_parent=True) - rp.append(OpenAPIResourceProvider(name, path, readme_path, swagger_module=self)) + readme_paths = [*_search_readme_md_paths(path, search_parent=True)] + rp.append(OpenAPIResourceProvider(name, path, readme_paths, swagger_module=self)) elif name.lower() != 'common': # azsadmin module only sub_module = MgmtPlaneModule(plane=self.plane, name=name, folder_path=path, parent=self) @@ -91,7 +81,7 @@ def _get_typespec_resource_providers(self): return rp for namespace, ts_path, cfg_path in TypeSpecHelper.find_mgmt_plane_entry_files(self.folder_path): - entry_file = self.resolve_path_to_uri(ts_path) + entry_file = resolve_path_to_uri(ts_path) if namespace in rp: rp[namespace].entry_files.append(entry_file) else: @@ -133,8 +123,8 @@ def _get_openapi_resource_providers(self): continue name_parts = name.split('.') if len(name_parts) >= 2: - readme_path = _search_readme_md_path(path, search_parent=True) - rp.append(OpenAPIResourceProvider(name, path, readme_path, swagger_module=self)) + readme_paths = [*_search_readme_md_paths(path, search_parent=True)] + rp.append(OpenAPIResourceProvider(name, path, readme_paths, swagger_module=self)) elif name.lower() != 'common': sub_module = DataPlaneModule(plane=self.plane, name=name, folder_path=path, parent=self) rp.extend(sub_module.get_resource_providers()) @@ -146,7 +136,7 @@ def _get_typespec_resource_providers(self): return rp for namespace, ts_path, cfg_path in TypeSpecHelper.find_data_plane_entry_files(self.folder_path): - entry_file = self.resolve_path_to_uri(ts_path) + entry_file = resolve_path_to_uri(ts_path) if namespace in rp: rp[namespace].entry_files.append(entry_file) else: @@ -155,21 +145,13 @@ def _get_typespec_resource_providers(self): return [*rp.values()] -def _search_readme_md_path(path, search_parent=False): +def _search_readme_md_paths(path, search_parent=False): + # Check parent directory first if requested if search_parent: - readme_path = os.path.join(os.path.dirname(path), 'readme.md') - if os.path.exists(readme_path): - return readme_path - - readme_path = os.path.join(path, 'readme.md') - if os.path.exists(readme_path): - return readme_path - - # find in sub directory - for name in os.listdir(path): - sub_path = os.path.join(path, name) - if os.path.isdir(sub_path): - readme_path = _search_readme_md_path(sub_path) - if readme_path is not None: - return readme_path - return None + parent_readme = os.path.join(os.path.dirname(path), 'readme.md') + if os.path.isfile(parent_readme): + yield parent_readme + + # Use glob to recursively find all readme.md files + pattern = os.path.join(path, '**', 'readme.md') + yield from glob.iglob(pattern, recursive=True) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_specs.py b/src/aaz_dev/swagger/model/specs/_swagger_specs.py index 57492f06..688d084e 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_specs.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_specs.py @@ -12,12 +12,12 @@ def __init__(self, folder_path): self._folder_path = folder_path @property - def _spec_folder_path(self): + def spec_folder_path(self): return os.path.join(self._folder_path, 'specification') def get_mgmt_plane_modules(self, plane): modules = [] - for name in os.listdir(self._spec_folder_path): + for name in os.listdir(self.spec_folder_path): module = self.get_mgmt_plane_module(name, plane=plane) if module: modules.append(module) @@ -30,7 +30,7 @@ def get_mgmt_plane_module(self, *names, plane): if not PlaneEnum.is_valid_swagger_module(plane=plane, module_name=name): return None - path = os.path.join(self._spec_folder_path, name) + path = os.path.join(self.spec_folder_path, name) if not os.path.isdir(path): return None if os.path.isdir(os.path.join(path, 'resource-manager')) or TypeSpecHelper.find_mgmt_plane_entry_files(path): @@ -45,7 +45,7 @@ def get_mgmt_plane_module(self, *names, plane): def get_data_plane_modules(self, plane): modules = [] - for name in os.listdir(self._spec_folder_path): + for name in os.listdir(self.spec_folder_path): module = self.get_data_plane_module(name, plane=plane) if module: modules.append(module) @@ -58,7 +58,7 @@ def get_data_plane_module(self, *names, plane): if not PlaneEnum.is_valid_swagger_module(plane=plane, module_name=name): return None - path = os.path.join(self._spec_folder_path, name) + path = os.path.join(self.spec_folder_path, name) if os.path.isdir(os.path.join(path, 'data-plane')) or TypeSpecHelper.find_data_plane_entry_files(path): module = DataPlaneModule(plane=plane, name=name, folder_path=path) for name in names[1:]: diff --git a/src/aaz_dev/swagger/utils/tools.py b/src/aaz_dev/swagger/utils/tools.py index 62f4c439..f2610ae8 100644 --- a/src/aaz_dev/swagger/utils/tools.py +++ b/src/aaz_dev/swagger/utils/tools.py @@ -4,6 +4,8 @@ # license information. # ----------------------------------------------------------------------------- import re +import os +from utils.config import Config URL_PARAMETER_PLACEHOLDER = "{}" @@ -39,3 +41,14 @@ def swagger_resource_path_to_resource_id(path): idx += 1 path_parts[0] = "/".join(url_parts).lower() return "?".join(path_parts) + + +def resolve_path_to_uri(path): + relative_path = os.path.relpath(path, start=Config.get_swagger_root()).replace(os.sep, '/') + if relative_path.startswith('../'): + raise ValueError(f"Invalid path: {path}") + if relative_path.startswith('./'): + relative_path = relative_path[2:] + if relative_path.startswith('/'): + relative_path = relative_path[1:] + return relative_path diff --git a/src/aaz_dev/utils/readme_helper.py b/src/aaz_dev/utils/readme_helper.py new file mode 100644 index 00000000..e9d24933 --- /dev/null +++ b/src/aaz_dev/utils/readme_helper.py @@ -0,0 +1,50 @@ +import yaml + +def _update_config(config, yaml_content): + for key, value in yaml_content.items(): + if key not in config: + config[key] = value + continue + if isinstance(value, dict): + _update_config(config[key], value) + elif isinstance(value, list): + config[key].extend(value) + else: + config[key] = value + +def parse_readme_file(readme_path: str): + """Parse the readme file title and combine basic config in the yaml section.""" + readme_config = {} + with open(readme_path, 'r', encoding='utf-8') as f: + content = f.readlines() + # content.append("```") # append a fake yaml section to make sure the last yaml section is ended + readme_title = None + in_yaml_section = False + yaml_content = [] + for line in content: + if not readme_title and line.strip().startswith("# ") and not in_yaml_section: + readme_title = line.strip()[2:].strip() + if line.strip().startswith("```") and 'yaml' in line: + condition = line.split('yaml')[1].strip() + # Do not parse the yaml section if it has the condition + if not condition: + in_yaml_section = True + elif in_yaml_section: + if line.strip().startswith("```"): + try: + yaml_config = yaml.load("\n".join(yaml_content), Loader=yaml.FullLoader) + except Exception as e: + raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_path}") + _update_config(readme_config, yaml_config) + in_yaml_section = False + yaml_content = [] + else: + if line.strip(): + yaml_content.append(line) + else: + yaml_content.append("") + + return { + "title": readme_title, + "config": readme_config + } From 556564ac33781c13b65b6533053f6ae33ff0469c Mon Sep 17 00:00:00 2001 From: kai ru <69238381+kairu-ms@users.noreply.github.com> Date: Fri, 27 Dec 2024 11:09:10 +0800 Subject: [PATCH 2/2] Update src/aaz_dev/utils/readme_helper.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/aaz_dev/utils/readme_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/utils/readme_helper.py b/src/aaz_dev/utils/readme_helper.py index e9d24933..dd8a2887 100644 --- a/src/aaz_dev/utils/readme_helper.py +++ b/src/aaz_dev/utils/readme_helper.py @@ -32,7 +32,7 @@ def parse_readme_file(readme_path: str): elif in_yaml_section: if line.strip().startswith("```"): try: - yaml_config = yaml.load("\n".join(yaml_content), Loader=yaml.FullLoader) + yaml_config = yaml.safe_load("\n".join(yaml_content)) except Exception as e: raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_path}") _update_config(readme_config, yaml_config)