From 9243c995ee25067b43a94ed14fabede7947fe883 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 25 Oct 2023 16:11:50 +0800 Subject: [PATCH 01/37] feat: initial commit --- src/aaz_dev/command/api/editor.py | 18 ++++++++++ .../command/controller/workspace_manager.py | 13 +++++++ .../swagger/controller/command_generator.py | 4 +++ .../swagger/model/schema/example_item.py | 36 +++++++++++++++++++ src/aaz_dev/swagger/model/schema/fields.py | 16 --------- src/aaz_dev/swagger/model/schema/operation.py | 12 +++++-- src/aaz_dev/swagger/model/schema/path_item.py | 16 +++++++++ src/aaz_dev/swagger/model/schema/response.py | 3 +- src/aaz_dev/swagger/model/schema/swagger.py | 7 ++++ .../swagger/model/specs/_swagger_loader.py | 6 ++++ .../workspace/WSEditorCommandContent.tsx | 31 +++++++++++++++- 11 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 src/aaz_dev/swagger/model/schema/example_item.py diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index cf84b160..f4b89282 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -3,6 +3,7 @@ from flask import Blueprint, jsonify, request, url_for from command.controller.workspace_manager import WorkspaceManager +from swagger.model.specs import SwaggerLoader from utils import exceptions from utils.config import Config @@ -178,6 +179,23 @@ def editor_workspace_command_tree_node_rename(name, node_names): return jsonify(result) +@bp.route("/Workspaces//CommandTree/Nodes//Leaves//GenerateExamples", + methods=["POST"]) +def editor_workspace_generate_example(name, node_names, leaf_name): + if node_names[0] != WorkspaceManager.COMMAND_TREE_ROOT_NAME: + raise exceptions.ResourceNotFind("Command not exist.") + + manager = WorkspaceManager(name) + manager.load() + + node_names = node_names[1:] + leaf = manager.find_command_tree_leaf(*node_names, leaf_name) + if not leaf: + raise exceptions.ResourceNotFind("Command not exist.") + + manager.load_examples_by_swagger(leaf.resources[0]) + + @bp.route("/Workspaces//CommandTree/Nodes//Leaves/", methods=("GET", "PATCH")) def editor_workspace_command(name, node_names, leaf_name): diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index b4bfd74e..bd3335ae 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -522,6 +522,19 @@ def generate_unique_name(self, *node_names, name): new_name = f"{name}-untitled{idx}" return new_name + def load_examples_by_swagger(self, resource): + root = self.find_command_tree_node() + assert root + + # convert cmd resource to swagger resource + swagger_resource = self.swagger_specs.get_module_manager( + plane=self.ws.plane, + mod_names=self.ws.mod_names + ).get_resource_in_version(resource["id"], resource["version"], resource.rp_name) + + # load swagger resource + self.swagger_command_generator.load_examples(swagger_resource) + def add_new_resources_by_swagger(self, mod_names, version, resources): root_node = self.find_command_tree_node() assert root_node diff --git a/src/aaz_dev/swagger/controller/command_generator.py b/src/aaz_dev/swagger/controller/command_generator.py index 89c2574a..6b816a6f 100644 --- a/src/aaz_dev/swagger/controller/command_generator.py +++ b/src/aaz_dev/swagger/controller/command_generator.py @@ -34,6 +34,10 @@ def load_resources(self, resources): self.loader.load_file(resource.file_path) self.loader.link_swaggers() + def load_examples(self, resource): + self.loader.load_file(resource.file_path) + self.loader.link_examples() + def create_draft_command_group(self, resource, update_by=None, methods=('get', 'delete', 'put', 'post', 'head', 'patch'), diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py new file mode 100644 index 00000000..9ea0128f --- /dev/null +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -0,0 +1,36 @@ +from schematics.models import Model +from schematics.types import DictType, ModelType + +from .reference import Linkable, ReferenceField + + +class ExampleItem(Model, Linkable): + ref = ReferenceField() + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ref_instance = None + + def link(self, swagger_loader, *traces): + if self.is_linked(): + return + + super().link(swagger_loader, *traces) + + self.ref_instance, instance_traces = swagger_loader.load_ref(self.ref, *self.traces, "ref") + + +class XmsExamplesField(DictType): + """ + Describes the format for specifying examples for request and response of an operation in an OpenAPI definition. + It is a dictionary of different variations of the examples for a given operation. + + https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/x-ms-examples.md + """ + def __init__(self, **kwargs): + super().__init__( + field=ModelType(ExampleItem), + serialized_name="x-ms-examples", + deserialize_from="x-ms-examples", + **kwargs + ) diff --git a/src/aaz_dev/swagger/model/schema/fields.py b/src/aaz_dev/swagger/model/schema/fields.py index c35f41f2..1096637c 100644 --- a/src/aaz_dev/swagger/model/schema/fields.py +++ b/src/aaz_dev/swagger/model/schema/fields.py @@ -207,22 +207,6 @@ def __init__(self, **kwargs): ) -class XmsExamplesField(DictType): - """ - Describes the format for specifying examples for request and response of an operation in an OpenAPI definition. It is a dictionary of different variations of the examples for a given operation. - - https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/x-ms-examples.md - """ - - def __init__(self, **kwargs): - super().__init__( - field=BaseType(), - serialized_name='x-ms-examples', - deserialize_from='x-ms-examples', - **kwargs - ) - - class XmsErrorResponseField(BooleanType): """ Indicates whether the response status code should be treated as an error response or not. diff --git a/src/aaz_dev/swagger/model/schema/operation.py b/src/aaz_dev/swagger/model/schema/operation.py index a3ae3432..0247dddc 100644 --- a/src/aaz_dev/swagger/model/schema/operation.py +++ b/src/aaz_dev/swagger/model/schema/operation.py @@ -5,8 +5,9 @@ CMDHttpRequestQuery, CMDHttpRequestHeader, CMDHttpRequestJsonBody, CMDRequestJson, CMDHttpOperationLongRunning from swagger.utils import exceptions from swagger.utils.tools import swagger_resource_path_to_resource_id_template +from .example_item import XmsExamplesField from .external_documentation import ExternalDocumentation -from .fields import MimeField, XmsRequestIdField, XmsExamplesField, SecurityRequirementField, XPublishField, \ +from .fields import MimeField, XmsRequestIdField, SecurityRequirementField, XPublishField, \ XSfCodeGenField, XmsClientNameField from .parameter import ParameterField, PathParameter, QueryParameter, HeaderParameter, BodyParameter,\ FormDataParameter, ParameterBase @@ -51,7 +52,7 @@ class Operation(Model, Linkable): x_ms_odata = XmsODataField() # TODO: # indicates the operation includes one or more OData query parameters. x_ms_request_id = XmsRequestIdField() - x_ms_examples = XmsExamplesField() # TODO: + x_ms_examples = XmsExamplesField() # specific properties _x_publish = XPublishField() # only used in Maps Data Plane @@ -131,6 +132,13 @@ def link(self, swagger_loader, *traces): *self.traces, "x_ms_long_running_operation_options", "final_state_schema" ) + def link_examples(self, swagger_loader, *traces): + super().link(swagger_loader, *traces) + + if self.x_ms_examples is not None: + for key, example in self.x_ms_examples.items(): + example.link(swagger_loader, *self.traces, "x_ms_examples", key) + def to_cmd(self, builder, parent_parameters, **kwargs): cmd_op = CMDHttpOperation() if self.x_ms_long_running_operation: diff --git a/src/aaz_dev/swagger/model/schema/path_item.py b/src/aaz_dev/swagger/model/schema/path_item.py index 028bd35b..8575a3cc 100644 --- a/src/aaz_dev/swagger/model/schema/path_item.py +++ b/src/aaz_dev/swagger/model/schema/path_item.py @@ -52,6 +52,22 @@ def link(self, swagger_loader, *traces): if self.patch is not None: self.patch.link(swagger_loader, *self.traces, 'patch') + def link_examples(self, swagger_loader, *traces): + super().link(swagger_loader, *traces) + + if self.get is not None: + self.get.link_examples(swagger_loader, *self.traces, "get") + if self.put is not None: + self.put.link_examples(swagger_loader, *self.traces, "put") + if self.post is not None: + self.post.link_examples(swagger_loader, *self.traces, "post") + if self.delete is not None: + self.delete.link_examples(swagger_loader, *self.traces, "delete") + if self.head is not None: + self.head.link_examples(swagger_loader, *self.traces, "head") + if self.patch is not None: + self.patch.link_examples(swagger_loader, *self.traces, "patch") + def to_cmd(self, builder, **kwargs): op = getattr(self, builder.method, None) if op is None: diff --git a/src/aaz_dev/swagger/model/schema/response.py b/src/aaz_dev/swagger/model/schema/response.py index cf6460e7..b6cae2c4 100644 --- a/src/aaz_dev/swagger/model/schema/response.py +++ b/src/aaz_dev/swagger/model/schema/response.py @@ -7,7 +7,8 @@ from swagger.model.schema.fields import MutabilityEnum from swagger.utils import exceptions from utils.error_format import AAZErrorFormatEnum -from .fields import XmsExamplesField, XmsErrorResponseField, XNullableField +from .example_item import XmsExamplesField +from .fields import XmsErrorResponseField, XNullableField from .header import Header from .reference import Linkable from .schema import Schema, ReferenceSchema, schema_and_reference_schema_claim_function diff --git a/src/aaz_dev/swagger/model/schema/swagger.py b/src/aaz_dev/swagger/model/schema/swagger.py index d0e60415..520ea42c 100644 --- a/src/aaz_dev/swagger/model/schema/swagger.py +++ b/src/aaz_dev/swagger/model/schema/swagger.py @@ -85,3 +85,10 @@ def link(self, swagger_loader, *traces): if self.x_ms_parameterized_host is not None: self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') + + def link_examples(self, swagger_loader, *traces): + super().link(swagger_loader, *traces) + + if self.paths is not None: + for key, path in self.paths.items(): + path.link_examples(swagger_loader, *self.traces, "paths", key) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_loader.py b/src/aaz_dev/swagger/model/specs/_swagger_loader.py index 1213f26c..f7b3a9c1 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_loader.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_loader.py @@ -62,6 +62,12 @@ def link_swaggers(self): swagger.link(self, file_path) self._linked_idx += 1 + def link_examples(self): + while self._linked_idx < len(self.loaded_swaggers): + file_path, swagger = [*self.loaded_swaggers.items()][self._linked_idx] + swagger.link_examples(self, file_path) + self._linked_idx += 1 + def get_loaded(self, *traces): return self._loaded.get(traces, None) diff --git a/src/web/src/views/workspace/WSEditorCommandContent.tsx b/src/web/src/views/workspace/WSEditorCommandContent.tsx index 01e18bf1..35259462 100644 --- a/src/web/src/views/workspace/WSEditorCommandContent.tsx +++ b/src/web/src/views/workspace/WSEditorCommandContent.tsx @@ -1001,6 +1001,35 @@ class ExampleDialog extends React.Component { + let { workspaceUrl, command } = this.props; + + const leafUrl = `${workspaceUrl}/CommandTree/Nodes/aaz/` + command.names.slice(0, -1).join('/') + '/Leaves/' + command.names[command.names.length - 1] + '/GenerateExamples'; + + this.setState({ + updating: true, + }) + axios.post(leafUrl, { + }).then(res => { + const cmd = DecodeResponseCommand(res.data); + this.setState({ + updating: false, + }) + this.props.onClose(cmd); + }).catch(err => { + console.error(err.response); + if (err.response?.data?.message) { + const data = err.response!.data!; + this.setState({ + invalidText: `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}`, + }) + } + this.setState({ + updating: false, + }) + }); + } + handleClose = () => { this.setState({ invalidText: undefined @@ -1134,7 +1163,7 @@ class ExampleDialog extends React.ComponentDelete } - {isAdd && } + {isAdd && } } From 6b12d025964a95af6642a49a9ed27ffc41907335 Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 31 Oct 2023 14:18:42 +0800 Subject: [PATCH 02/37] feat: separately load by operation id --- src/aaz_dev/command/api/editor.py | 22 +++++++- .../command/controller/workspace_manager.py | 56 ++++++++++++++----- .../swagger/controller/command_generator.py | 4 -- .../swagger/controller/example_generator.py | 42 ++++++++++++++ .../swagger/model/schema/example_item.py | 14 +++++ src/aaz_dev/swagger/model/schema/path_item.py | 14 ++--- src/aaz_dev/swagger/model/schema/swagger.py | 7 ++- .../swagger/model/specs/_swagger_loader.py | 10 ++-- 8 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 src/aaz_dev/swagger/controller/example_generator.py diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index f4b89282..9d6a8413 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -193,7 +193,27 @@ def editor_workspace_generate_example(name, node_names, leaf_name): if not leaf: raise exceptions.ResourceNotFind("Command not exist.") - manager.load_examples_by_swagger(leaf.resources[0]) + cfg_editor = manager.load_cfg_editor_by_command(leaf) + command = cfg_editor.find_command(*leaf.names) + + if command: + examples = manager.gen_examples_by_swagger(leaf, command) + else: + raise exceptions.ResourceNotFind("Command not exist") + + result = command.to_primitive() + # manager.save() + + del result['name'] + result.update({ + 'names': leaf.names, + 'help': leaf.help.to_primitive(), + 'stage': leaf.stage, + }) + if examples: + result['examples'] = [e.to_primitive() for e in leaf.examples] + + return jsonify(result) @bp.route("/Workspaces//CommandTree/Nodes//Leaves/", diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index bd3335ae..454fa5ac 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -6,6 +6,7 @@ from command.model.editor import CMDEditorWorkspace, CMDCommandTreeNode, CMDCommandTreeLeaf from swagger.controller.command_generator import CommandGenerator +from swagger.controller.example_generator import ExampleGenerator from swagger.controller.specs_manager import SwaggerSpecsManager from swagger.utils.exceptions import InvalidSwaggerValueError from utils import exceptions @@ -80,6 +81,7 @@ def __init__(self, name, folder=None, aaz_manager=None, swagger_manager=None): self._aaz_specs = aaz_manager self._swagger_specs = swagger_manager self._swagger_command_generator = None + self._swagger_example_generator = None @property def is_in_memory(self): @@ -103,6 +105,13 @@ def swagger_command_generator(self): self._swagger_command_generator = CommandGenerator() return self._swagger_command_generator + @property + def swagger_example_generator(self): + if not self._swagger_example_generator: + self._swagger_example_generator = ExampleGenerator() + + return self._swagger_example_generator + def load(self): assert not self.is_in_memory # TODO: handle exception @@ -456,6 +465,40 @@ def update_command_tree_leaf_examples(self, *leaf_names, examples): leaf.examples.append(example) return leaf + def gen_examples_by_swagger(self, leaf, command): + return self.generate_examples_by_swagger(command.resources[0], + command.operations[0].operation_id, + ' '.join(leaf.names)) + + # leaf.examples = swagger_examples + # for example in examples: + # if not isinstance(example, CMDCommandExample): + # example = CMDCommandExample(example) + # try: + # example.validate() + # except Exception as err: + # # if not example.get('name', None) or not isinstance(example['name'], str): + # raise exceptions.InvalidAPIUsage( + # f"Invalid example data: {err}") + # leaf.examples.append(example) + + def generate_examples_by_swagger(self, resource, operation_id, cmd_name): + root = self.find_command_tree_node() + assert root + + # convert cmd resource to swagger resource + swagger_resource = self.swagger_specs.get_module_manager( + plane=self.ws.plane, + mod_names=self.ws.mod_names + ).get_resource_in_version(resource["id"], resource["version"], resource.rp_name) + + # load swagger resource + self.swagger_example_generator.load_examples(swagger_resource, operation_id) + + examples = self.swagger_example_generator.create_draft_examples(swagger_resource, operation_id, cmd_name) + + return examples + def rename_command_tree_node(self, *node_names, new_node_names): new_name = ' '.join(new_node_names) if not new_name: @@ -522,19 +565,6 @@ def generate_unique_name(self, *node_names, name): new_name = f"{name}-untitled{idx}" return new_name - def load_examples_by_swagger(self, resource): - root = self.find_command_tree_node() - assert root - - # convert cmd resource to swagger resource - swagger_resource = self.swagger_specs.get_module_manager( - plane=self.ws.plane, - mod_names=self.ws.mod_names - ).get_resource_in_version(resource["id"], resource["version"], resource.rp_name) - - # load swagger resource - self.swagger_command_generator.load_examples(swagger_resource) - def add_new_resources_by_swagger(self, mod_names, version, resources): root_node = self.find_command_tree_node() assert root_node diff --git a/src/aaz_dev/swagger/controller/command_generator.py b/src/aaz_dev/swagger/controller/command_generator.py index 6b816a6f..89c2574a 100644 --- a/src/aaz_dev/swagger/controller/command_generator.py +++ b/src/aaz_dev/swagger/controller/command_generator.py @@ -34,10 +34,6 @@ def load_resources(self, resources): self.loader.load_file(resource.file_path) self.loader.link_swaggers() - def load_examples(self, resource): - self.loader.load_file(resource.file_path) - self.loader.link_examples() - def create_draft_command_group(self, resource, update_by=None, methods=('get', 'delete', 'put', 'post', 'head', 'patch'), diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py new file mode 100644 index 00000000..176e2019 --- /dev/null +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -0,0 +1,42 @@ +from command.model.configuration import CMDCommandExample +from swagger.model.schema.example_item import XmsExamplesField +from swagger.model.schema.path_item import PathItem +from swagger.model.specs import SwaggerLoader + + +class ExampleGenerator: + def __init__(self): + self.loader = SwaggerLoader() + + def load_examples(self, resource, operation_id): + self.loader.load_file(resource.file_path) + self.loader.link_examples(resource.file_path, resource.path, operation_id) + + def create_draft_examples(self, resource, operation_id, cmd_name): + swagger = self.loader.get_loaded(resource.file_path) + assert swagger is not None + + path_item = swagger.paths.get(resource.path, None) + if path_item is None: + path_item = swagger.x_ms_paths.get(resource.path, None) + assert isinstance(path_item, PathItem) + + linked_examples = XmsExamplesField() + if path_item.get is not None and operation_id == path_item.get.operation_id: + linked_examples = path_item.get.x_ms_examples + elif path_item.delete is not None and operation_id == path_item.delete.operation_id: + linked_examples = path_item.delete.x_ms_examples + elif path_item.put is not None and operation_id == path_item.put.operation_id: + linked_examples = path_item.put.x_ms_examples + elif path_item.post is not None and operation_id == path_item.post.operation_id: + linked_examples = path_item.post.x_ms_examples + elif path_item.head is not None and operation_id == path_item.head.operation_id: + linked_examples = path_item.head.x_ms_examples + + cmd_examples = [] + for name, example_item in linked_examples.items(): + example = example_item.to_cmd(cmd_name) + example.name = name + cmd_examples.append(example) + + return cmd_examples diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index 9ea0128f..ced6f888 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -1,5 +1,6 @@ from schematics.models import Model from schematics.types import DictType, ModelType +from command.model.configuration import CMDCommandExample from .reference import Linkable, ReferenceField @@ -19,6 +20,19 @@ def link(self, swagger_loader, *traces): self.ref_instance, instance_traces = swagger_loader.load_ref(self.ref, *self.traces, "ref") + def to_cmd(self, cmd_name, **kwargs): + assert self.ref_instance + + params = [] + for k, v in self.ref_instance.get('parameters', {}).items(): + example_key = '--' + k + example_value = v + params.extend([example_key, example_value]) + + return CMDCommandExample({ + 'commands': [cmd_name + ' ' + ' '.join(params)] + }) + class XmsExamplesField(DictType): """ diff --git a/src/aaz_dev/swagger/model/schema/path_item.py b/src/aaz_dev/swagger/model/schema/path_item.py index 8575a3cc..4023b59d 100644 --- a/src/aaz_dev/swagger/model/schema/path_item.py +++ b/src/aaz_dev/swagger/model/schema/path_item.py @@ -52,20 +52,20 @@ def link(self, swagger_loader, *traces): if self.patch is not None: self.patch.link(swagger_loader, *self.traces, 'patch') - def link_examples(self, swagger_loader, *traces): + def link_examples(self, swagger_loader, operation_id, *traces): super().link(swagger_loader, *traces) - if self.get is not None: + if self.get is not None and self.get.operation_id == operation_id: self.get.link_examples(swagger_loader, *self.traces, "get") - if self.put is not None: + if self.put is not None and self.put.operation_id == operation_id: self.put.link_examples(swagger_loader, *self.traces, "put") - if self.post is not None: + if self.post is not None and self.post.operation_id == operation_id: self.post.link_examples(swagger_loader, *self.traces, "post") - if self.delete is not None: + if self.delete is not None and self.delete.operation_id == operation_id: self.delete.link_examples(swagger_loader, *self.traces, "delete") - if self.head is not None: + if self.head is not None and self.head.operation_id == operation_id: self.head.link_examples(swagger_loader, *self.traces, "head") - if self.patch is not None: + if self.patch is not None and self.patch.operation_id == operation_id: self.patch.link_examples(swagger_loader, *self.traces, "patch") def to_cmd(self, builder, **kwargs): diff --git a/src/aaz_dev/swagger/model/schema/swagger.py b/src/aaz_dev/swagger/model/schema/swagger.py index 520ea42c..5553f186 100644 --- a/src/aaz_dev/swagger/model/schema/swagger.py +++ b/src/aaz_dev/swagger/model/schema/swagger.py @@ -86,9 +86,10 @@ def link(self, swagger_loader, *traces): if self.x_ms_parameterized_host is not None: self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') - def link_examples(self, swagger_loader, *traces): + def link_examples(self, swagger_loader, path, operation_id, *traces): super().link(swagger_loader, *traces) if self.paths is not None: - for key, path in self.paths.items(): - path.link_examples(swagger_loader, *self.traces, "paths", key) + for key, path_item in self.paths.items(): + if key == path: + path_item.link_examples(swagger_loader, operation_id, *self.traces, "paths", key) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_loader.py b/src/aaz_dev/swagger/model/specs/_swagger_loader.py index f7b3a9c1..77a73f45 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_loader.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_loader.py @@ -62,11 +62,11 @@ def link_swaggers(self): swagger.link(self, file_path) self._linked_idx += 1 - def link_examples(self): - while self._linked_idx < len(self.loaded_swaggers): - file_path, swagger = [*self.loaded_swaggers.items()][self._linked_idx] - swagger.link_examples(self, file_path) - self._linked_idx += 1 + def link_examples(self, file_path, path, operation_id): + swagger = self.loaded_swaggers[file_path] + assert swagger + + swagger.link_examples(self, path, operation_id, file_path) def get_loaded(self, *traces): return self._loaded.get(traces, None) From 9be1a264119d916610b8bb4a806792a7be2335e3 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 8 Nov 2023 13:50:06 +0800 Subject: [PATCH 03/37] feat: process list/update command --- src/aaz_dev/command/api/editor.py | 16 +--- .../command/controller/workspace_manager.py | 67 ++++++++++------ .../swagger/controller/example_generator.py | 80 ++++++++++++------- .../swagger/model/schema/example_item.py | 23 +++--- src/aaz_dev/swagger/model/schema/path_item.py | 14 ++-- src/aaz_dev/swagger/model/schema/swagger.py | 4 +- .../swagger/model/specs/_swagger_loader.py | 4 +- 7 files changed, 121 insertions(+), 87 deletions(-) diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index 9d6a8413..4dd83281 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -197,21 +197,11 @@ def editor_workspace_generate_example(name, node_names, leaf_name): command = cfg_editor.find_command(*leaf.names) if command: - examples = manager.gen_examples_by_swagger(leaf, command) + examples = manager.generate_examples_by_swagger(leaf, command) else: - raise exceptions.ResourceNotFind("Command not exist") - - result = command.to_primitive() - # manager.save() + raise exceptions.ResourceNotFind("Command not exist.") - del result['name'] - result.update({ - 'names': leaf.names, - 'help': leaf.help.to_primitive(), - 'stage': leaf.stage, - }) - if examples: - result['examples'] = [e.to_primitive() for e in leaf.examples] + result = [example.to_primitive() for example in examples] return jsonify(result) diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index 454fa5ac..0c0deb86 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -13,7 +13,7 @@ from utils.config import Config from .specs_manager import AAZSpecsManager from .workspace_cfg_editor import WorkspaceCfgEditor -from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand +from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand, CMDHttpOperation logger = logging.getLogger('backend') @@ -465,37 +465,52 @@ def update_command_tree_leaf_examples(self, *leaf_names, examples): leaf.examples.append(example) return leaf - def gen_examples_by_swagger(self, leaf, command): - return self.generate_examples_by_swagger(command.resources[0], - command.operations[0].operation_id, - ' '.join(leaf.names)) - - # leaf.examples = swagger_examples - # for example in examples: - # if not isinstance(example, CMDCommandExample): - # example = CMDCommandExample(example) - # try: - # example.validate() - # except Exception as err: - # # if not example.get('name', None) or not isinstance(example['name'], str): - # raise exceptions.InvalidAPIUsage( - # f"Invalid example data: {err}") - # leaf.examples.append(example) - - def generate_examples_by_swagger(self, resource, operation_id, cmd_name): + def generate_examples_by_swagger(self, leaf, command): + def is_ready_to_skip(operations): + if not operations: + return True + + if len(operations) == 1 and isinstance(operations[0], CMDHttpOperation): + return False + + # skip when there is more than one operation and not all operations are get requests + for op in operations: + if not isinstance(op, CMDHttpOperation) or op.http.request.method != "get": + return True + + return False + + if is_ready_to_skip(command.operations): + return [] + + return self.generate_operations_examples_by_swagger( + command.resources, + [op.operation_id for op in command.operations], + " ".join(leaf.names), + command.arg_groups + ) + + def generate_operations_examples_by_swagger(self, resources, operation_ids, cmd_name, arg_groups): root = self.find_command_tree_node() assert root # convert cmd resource to swagger resource - swagger_resource = self.swagger_specs.get_module_manager( - plane=self.ws.plane, - mod_names=self.ws.mod_names - ).get_resource_in_version(resource["id"], resource["version"], resource.rp_name) + swagger_resources = [] + for resource in resources: + swagger_resources.append(self.swagger_specs.get_module_manager( + plane=self.ws.plane, + mod_names=self.ws.mod_names + ).get_resource_in_version(resource["id"], resource["version"], resource.rp_name)) # load swagger resource - self.swagger_example_generator.load_examples(swagger_resource, operation_id) - - examples = self.swagger_example_generator.create_draft_examples(swagger_resource, operation_id, cmd_name) + self.swagger_example_generator.load_examples(swagger_resources, operation_ids) + + examples = self.swagger_example_generator.create_draft_examples( + swagger_resources, + operation_ids, + cmd_name, + arg_groups + ) return examples diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index 176e2019..1eade354 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -8,35 +8,59 @@ class ExampleGenerator: def __init__(self): self.loader = SwaggerLoader() - def load_examples(self, resource, operation_id): - self.loader.load_file(resource.file_path) - self.loader.link_examples(resource.file_path, resource.path, operation_id) - - def create_draft_examples(self, resource, operation_id, cmd_name): - swagger = self.loader.get_loaded(resource.file_path) - assert swagger is not None - - path_item = swagger.paths.get(resource.path, None) - if path_item is None: - path_item = swagger.x_ms_paths.get(resource.path, None) - assert isinstance(path_item, PathItem) - - linked_examples = XmsExamplesField() - if path_item.get is not None and operation_id == path_item.get.operation_id: - linked_examples = path_item.get.x_ms_examples - elif path_item.delete is not None and operation_id == path_item.delete.operation_id: - linked_examples = path_item.delete.x_ms_examples - elif path_item.put is not None and operation_id == path_item.put.operation_id: - linked_examples = path_item.put.x_ms_examples - elif path_item.post is not None and operation_id == path_item.post.operation_id: - linked_examples = path_item.post.x_ms_examples - elif path_item.head is not None and operation_id == path_item.head.operation_id: - linked_examples = path_item.head.x_ms_examples + def load_examples(self, resources, operation_ids): + for resource in resources: + self.loader.load_file(resource.file_path) + self.loader.link_examples(resource.file_path, resource.path, operation_ids) + def create_draft_examples(self, resources, operation_ids, cmd_name, arg_groups): cmd_examples = [] - for name, example_item in linked_examples.items(): - example = example_item.to_cmd(cmd_name) - example.name = name - cmd_examples.append(example) + arg_name_map = get_arg_name_map(arg_groups) + + for resource in resources: + swagger = self.loader.get_loaded(resource.file_path) + if not swagger: + continue + + path_item = swagger.paths.get(resource.path, None) + if path_item is None: + path_item = swagger.x_ms_paths.get(resource.path, None) + if not isinstance(path_item, PathItem): + continue + + linked_examples = XmsExamplesField() + if path_item.get is not None and path_item.get.operation_id in operation_ids: + linked_examples = path_item.get.x_ms_examples + elif path_item.delete is not None and path_item.delete.operation_id in operation_ids: + linked_examples = path_item.delete.x_ms_examples + elif path_item.put is not None and path_item.put.operation_id in operation_ids: + linked_examples = path_item.put.x_ms_examples + elif path_item.post is not None and path_item.post.operation_id in operation_ids: + linked_examples = path_item.post.x_ms_examples + elif path_item.head is not None and path_item.head.operation_id in operation_ids: + linked_examples = path_item.head.x_ms_examples + + for name, example_item in linked_examples.items(): + example = example_item.to_cmd(arg_name_map, cmd_name) + example.name = name + cmd_examples.append(example) return cmd_examples + + +def get_arg_name_map(arg_groups): + IGNORE_ARGS = ["subscriptionId"] + + arg_name_map = {} + for group in arg_groups: + for arg in group.args: + swagger_name = arg.var.split(".", maxsplit=1)[-1] + if swagger_name in IGNORE_ARGS: + continue + + arg_name = arg.options[-1] + arg_name = "-" + arg_name if len(arg_name) == 1 else "--" + arg_name + + arg_name_map[swagger_name] = arg_name + + return arg_name_map diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index ced6f888..eaaa11db 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -1,3 +1,5 @@ +import json + from schematics.models import Model from schematics.types import DictType, ModelType from command.model.configuration import CMDCommandExample @@ -20,18 +22,21 @@ def link(self, swagger_loader, *traces): self.ref_instance, instance_traces = swagger_loader.load_ref(self.ref, *self.traces, "ref") - def to_cmd(self, cmd_name, **kwargs): + def to_cmd(self, args_name_map, cmd_name, **kwargs): assert self.ref_instance params = [] - for k, v in self.ref_instance.get('parameters', {}).items(): - example_key = '--' + k - example_value = v - params.extend([example_key, example_value]) - - return CMDCommandExample({ - 'commands': [cmd_name + ' ' + ' '.join(params)] - }) + for k, v in self.ref_instance.get("parameters", {}).items(): + if k not in args_name_map: + continue + + example_key = args_name_map[k] + example_val = json.dumps(v) + params.extend([example_key, example_val]) + + command = cmd_name + " " + " ".join(params) + + return CMDCommandExample({"commands": [command.strip()]}) class XmsExamplesField(DictType): diff --git a/src/aaz_dev/swagger/model/schema/path_item.py b/src/aaz_dev/swagger/model/schema/path_item.py index 4023b59d..1ddaa12a 100644 --- a/src/aaz_dev/swagger/model/schema/path_item.py +++ b/src/aaz_dev/swagger/model/schema/path_item.py @@ -52,20 +52,20 @@ def link(self, swagger_loader, *traces): if self.patch is not None: self.patch.link(swagger_loader, *self.traces, 'patch') - def link_examples(self, swagger_loader, operation_id, *traces): + def link_examples(self, swagger_loader, operation_ids, *traces): super().link(swagger_loader, *traces) - if self.get is not None and self.get.operation_id == operation_id: + if self.get is not None and self.get.operation_id in operation_ids: self.get.link_examples(swagger_loader, *self.traces, "get") - if self.put is not None and self.put.operation_id == operation_id: + if self.put is not None and self.put.operation_id in operation_ids: self.put.link_examples(swagger_loader, *self.traces, "put") - if self.post is not None and self.post.operation_id == operation_id: + if self.post is not None and self.post.operation_id in operation_ids: self.post.link_examples(swagger_loader, *self.traces, "post") - if self.delete is not None and self.delete.operation_id == operation_id: + if self.delete is not None and self.delete.operation_id in operation_ids: self.delete.link_examples(swagger_loader, *self.traces, "delete") - if self.head is not None and self.head.operation_id == operation_id: + if self.head is not None and self.head.operation_id in operation_ids: self.head.link_examples(swagger_loader, *self.traces, "head") - if self.patch is not None and self.patch.operation_id == operation_id: + if self.patch is not None and self.patch.operation_id in operation_ids: self.patch.link_examples(swagger_loader, *self.traces, "patch") def to_cmd(self, builder, **kwargs): diff --git a/src/aaz_dev/swagger/model/schema/swagger.py b/src/aaz_dev/swagger/model/schema/swagger.py index 5553f186..2e69e6c9 100644 --- a/src/aaz_dev/swagger/model/schema/swagger.py +++ b/src/aaz_dev/swagger/model/schema/swagger.py @@ -86,10 +86,10 @@ def link(self, swagger_loader, *traces): if self.x_ms_parameterized_host is not None: self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') - def link_examples(self, swagger_loader, path, operation_id, *traces): + def link_examples(self, swagger_loader, path, operation_ids, *traces): super().link(swagger_loader, *traces) if self.paths is not None: for key, path_item in self.paths.items(): if key == path: - path_item.link_examples(swagger_loader, operation_id, *self.traces, "paths", key) + path_item.link_examples(swagger_loader, operation_ids, *self.traces, "paths", key) diff --git a/src/aaz_dev/swagger/model/specs/_swagger_loader.py b/src/aaz_dev/swagger/model/specs/_swagger_loader.py index 77a73f45..3f876e45 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_loader.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_loader.py @@ -62,11 +62,11 @@ def link_swaggers(self): swagger.link(self, file_path) self._linked_idx += 1 - def link_examples(self, file_path, path, operation_id): + def link_examples(self, file_path, path, operation_ids): swagger = self.loaded_swaggers[file_path] assert swagger - swagger.link_examples(self, path, operation_id, file_path) + swagger.link_examples(self, path, operation_ids, file_path) def get_loaded(self, *traces): return self._loaded.get(traces, None) From 104c29ad0df1d85c42025201ae1ebab60982ce01 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 8 Nov 2023 13:51:58 +0800 Subject: [PATCH 04/37] test: add test case --- .../command/tests/api_tests/test_editor.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index 33e385f9..0388b87e 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -1902,3 +1902,35 @@ def test_workspace_add_subresource_commands(self, ws_name): self.assertTrue(rv.status_code == 200) data = rv.get_json() self.assertTrue(len(data) == 4) + + @workspace_name("test_workspace_generate_examples") + def test_workspace_generate_examples(self, ws_name): + module = "eventhub" + api_version = "2021-11-01" + + with self.app.test_client() as c: + rv = c.post(f"/AAZ/Editor/Workspaces", json={ + "name": ws_name, + "plane": PlaneEnum.Mgmt, + "modNames": module, + "resourceProvider": "Microsoft.EventHub" + }) + self.assertTrue(rv.status_code == 200) + + ws_url = rv.get_json()["url"] + resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.EventHub/clusters/{clusterName}") + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", json={ + "module": module, + "version": api_version, + "resources": [{ + "id": resource_id, + "options": { + "aaz_version": None + } + }] + }) + self.assertTrue(rv.status_code == 200) + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/cluster/Leaves/create/GenerateExamples") + self.assertTrue(rv.status_code == 200) From d1a044b934a74dfa7326f35d4da28e1fc424872d Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 13 Nov 2023 15:03:45 +0800 Subject: [PATCH 05/37] feat: add example builder --- src/aaz_dev/command/api/editor.py | 36 ++++++++++++++- src/aaz_dev/command/controller/cfg_reader.py | 28 ++++++++++++ .../model/configuration/_example_builder.py | 31 +++++++++++++ .../swagger/controller/example_generator.py | 44 ++++++++----------- .../swagger/model/schema/example_item.py | 23 ++++------ 5 files changed, 121 insertions(+), 41 deletions(-) create mode 100644 src/aaz_dev/command/model/configuration/_example_builder.py diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index 4dd83281..f9c21b4c 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -181,7 +181,7 @@ def editor_workspace_command_tree_node_rename(name, node_names): @bp.route("/Workspaces//CommandTree/Nodes//Leaves//GenerateExamples", methods=["POST"]) -def editor_workspace_generate_example(name, node_names, leaf_name): +def editor_workspace_generate_examples(name, node_names, leaf_name): if node_names[0] != WorkspaceManager.COMMAND_TREE_ROOT_NAME: raise exceptions.ResourceNotFind("Command not exist.") @@ -206,6 +206,40 @@ def editor_workspace_generate_example(name, node_names, leaf_name): return jsonify(result) +@bp.route("/Workspaces//CommandTree/Nodes//Leaves//Examples", + methods=["PATCH"]) +def editor_workspace_examples(name, node_names, leaf_name): + if node_names[0] != WorkspaceManager.COMMAND_TREE_ROOT_NAME: + raise exceptions.ResourceNotFind("Command not exist") + node_names = node_names[1:] + + manager = WorkspaceManager(name) + manager.load() + leaf = manager.find_command_tree_leaf(*node_names, leaf_name) + if not leaf: + raise exceptions.ResourceNotFind("Command not exist") + + data = request.get_json() + if 'examples' in data: + leaf = manager.update_command_tree_leaf_examples(*leaf.names, examples=data['examples']) + cfg_editor = manager.load_cfg_editor_by_command(leaf) + + command = cfg_editor.find_command(*leaf.names) + result = command.to_primitive() + manager.save() + + del result['name'] + result.update({ + 'names': leaf.names, + 'help': leaf.help.to_primitive(), + 'stage': leaf.stage, + }) + if leaf.examples: + result['examples'] = [e.to_primitive() for e in leaf.examples] + + return jsonify(result) + + @bp.route("/Workspaces//CommandTree/Nodes//Leaves/", methods=("GET", "PATCH")) def editor_workspace_command(name, node_names, leaf_name): diff --git a/src/aaz_dev/command/controller/cfg_reader.py b/src/aaz_dev/command/controller/cfg_reader.py index 2161571f..7494235b 100644 --- a/src/aaz_dev/command/controller/cfg_reader.py +++ b/src/aaz_dev/command/controller/cfg_reader.py @@ -298,6 +298,34 @@ def find_arg_with_parent_by_var(self, *cmd_names, arg_var): return None, None, None return self.find_arg_in_command_with_parent_by_var(command, arg_var=arg_var) + @classmethod + def find_arg_in_arg_groups_by_name(cls, arg_groups, arg_name): + assert isinstance(arg_name, str) + + def arg_filter(_parent, _arg, _arg_idx, _arg_var): + if _arg_var.endswith(f".{arg_name}"): + return (_parent, _arg, _arg_idx, _arg_var), True # find match + + elif _arg_var.startswith(f"{arg_name}."): + return (_parent, None, None, None), True # arg_var already been flattened + + return None, False + + for arg_group in arg_groups: + matches = [match for match in cls._iter_args_in_group(arg_group, arg_filter=arg_filter)] + if not matches: + continue + + assert len(matches) == 1 + + parent, arg, arg_idx, arg_var = matches[0] + if arg: + arg_idx = cls.arg_idx_to_str(arg_idx) + + return parent, arg, arg_idx + + return None, None, None + @classmethod def find_arg_in_command_by_var(cls, command, arg_var): _, arg, arg_idx = cls.find_arg_in_command_with_parent_by_var(command, arg_var=arg_var) diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/command/model/configuration/_example_builder.py new file mode 100644 index 00000000..0d1164a2 --- /dev/null +++ b/src/aaz_dev/command/model/configuration/_example_builder.py @@ -0,0 +1,31 @@ +from command.controller.cfg_reader import CfgReader + + +class ExampleBuilder: + def __init__(self, arg_groups=None): + self.arg_groups = arg_groups + + def mapping(self, swagger_params): + raise NotImplementedError() + + +class SwaggerExampleBuilder(ExampleBuilder): + def mapping(self, swagger_params): + import json + + cmd_params = {} + for k, v in swagger_params.items(): + _, _, arg_idx = CfgReader.find_arg_in_arg_groups_by_name(self.arg_groups, k) + if not arg_idx or arg_idx in cmd_params: + continue + + cmd_params[arg_idx] = json.dumps(v) + + return cmd_params + + # def iter_swagger_params(self, swagger_params): + # for k, v in swagger_params.items(): + # yield k, v + # if isinstance(v, dict): + # for k1, v1 in self.iter_swagger_params(v): + # yield k1, v1 diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index 1eade354..d0a799b5 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -1,4 +1,4 @@ -from command.model.configuration import CMDCommandExample +from command.model.configuration._example_builder import SwaggerExampleBuilder from swagger.model.schema.example_item import XmsExamplesField from swagger.model.schema.path_item import PathItem from swagger.model.specs import SwaggerLoader @@ -15,7 +15,7 @@ def load_examples(self, resources, operation_ids): def create_draft_examples(self, resources, operation_ids, cmd_name, arg_groups): cmd_examples = [] - arg_name_map = get_arg_name_map(arg_groups) + example_builder = SwaggerExampleBuilder(arg_groups=arg_groups) for resource in resources: swagger = self.loader.get_loaded(resource.file_path) @@ -28,39 +28,31 @@ def create_draft_examples(self, resources, operation_ids, cmd_name, arg_groups): if not isinstance(path_item, PathItem): continue - linked_examples = XmsExamplesField() + swagger_examples = XmsExamplesField() if path_item.get is not None and path_item.get.operation_id in operation_ids: - linked_examples = path_item.get.x_ms_examples + swagger_examples = path_item.get.x_ms_examples elif path_item.delete is not None and path_item.delete.operation_id in operation_ids: - linked_examples = path_item.delete.x_ms_examples + swagger_examples = path_item.delete.x_ms_examples elif path_item.put is not None and path_item.put.operation_id in operation_ids: - linked_examples = path_item.put.x_ms_examples + swagger_examples = path_item.put.x_ms_examples elif path_item.post is not None and path_item.post.operation_id in operation_ids: - linked_examples = path_item.post.x_ms_examples + swagger_examples = path_item.post.x_ms_examples elif path_item.head is not None and path_item.head.operation_id in operation_ids: - linked_examples = path_item.head.x_ms_examples + swagger_examples = path_item.head.x_ms_examples - for name, example_item in linked_examples.items(): - example = example_item.to_cmd(arg_name_map, cmd_name) - example.name = name - cmd_examples.append(example) + cmd_examples.extend(self.generate_examples(cmd_name, swagger_examples, example_builder)) return cmd_examples - -def get_arg_name_map(arg_groups): - IGNORE_ARGS = ["subscriptionId"] - - arg_name_map = {} - for group in arg_groups: - for arg in group.args: - swagger_name = arg.var.split(".", maxsplit=1)[-1] - if swagger_name in IGNORE_ARGS: + @staticmethod + def generate_examples(cmd_name, examples, example_builder): + cmd_examples = [] + for name, example_item in examples.items(): + cmd_example = example_item.to_cmd(example_builder, cmd_name) + if not cmd_example: continue - arg_name = arg.options[-1] - arg_name = "-" + arg_name if len(arg_name) == 1 else "--" + arg_name + cmd_example.name = name + cmd_examples.append(cmd_example) - arg_name_map[swagger_name] = arg_name - - return arg_name_map + return cmd_examples diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index eaaa11db..c5bf6380 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -1,9 +1,7 @@ -import json - from schematics.models import Model from schematics.types import DictType, ModelType -from command.model.configuration import CMDCommandExample +from command.model.configuration import CMDCommandExample from .reference import Linkable, ReferenceField @@ -22,19 +20,16 @@ def link(self, swagger_loader, *traces): self.ref_instance, instance_traces = swagger_loader.load_ref(self.ref, *self.traces, "ref") - def to_cmd(self, args_name_map, cmd_name, **kwargs): - assert self.ref_instance - - params = [] - for k, v in self.ref_instance.get("parameters", {}).items(): - if k not in args_name_map: - continue + def to_cmd(self, builder, cmd_name, **kwargs): + if not self.ref_instance: + return - example_key = args_name_map[k] - example_val = json.dumps(v) - params.extend([example_key, example_val]) + params = builder.mapping(self.ref_instance.get("parameters", {})) - command = cmd_name + " " + " ".join(params) + command = cmd_name + for k, v in params.items(): + command += f" --{k} {v}" + # command += " --" + k + " " + v return CMDCommandExample({"commands": [command.strip()]}) From 6711ad8b57c94ffdaa79f906d263df62568bc910 Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 20 Nov 2023 23:22:26 +0800 Subject: [PATCH 06/37] feat: build arg_var by cmd schema --- .../command/controller/workspace_manager.py | 25 ++- .../model/configuration/_example_builder.py | 174 ++++++++++++++++-- .../swagger/controller/example_generator.py | 68 +++++-- .../swagger/model/schema/example_item.py | 10 +- src/aaz_dev/swagger/model/schema/path_item.py | 15 +- src/aaz_dev/swagger/model/schema/swagger.py | 8 +- 6 files changed, 237 insertions(+), 63 deletions(-) diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index 0c0deb86..b8f93673 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -4,6 +4,7 @@ import shutil from datetime import datetime +from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand, CMDHttpOperation from command.model.editor import CMDEditorWorkspace, CMDCommandTreeNode, CMDCommandTreeLeaf from swagger.controller.command_generator import CommandGenerator from swagger.controller.example_generator import ExampleGenerator @@ -13,7 +14,6 @@ from utils.config import Config from .specs_manager import AAZSpecsManager from .workspace_cfg_editor import WorkspaceCfgEditor -from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand, CMDHttpOperation logger = logging.getLogger('backend') @@ -203,7 +203,7 @@ def save(self): f.write(data) self._cfg_editors = {} - + def __update_mod_names_and_resource_provider(self): resource_mod_set = set() resource_rp_set = set() @@ -484,32 +484,31 @@ def is_ready_to_skip(operations): return [] return self.generate_operations_examples_by_swagger( - command.resources, - [op.operation_id for op in command.operations], - " ".join(leaf.names), - command.arg_groups + command, + " ".join(leaf.names) ) - def generate_operations_examples_by_swagger(self, resources, operation_ids, cmd_name, arg_groups): + def generate_operations_examples_by_swagger(self, command, cmd_name): root = self.find_command_tree_node() assert root # convert cmd resource to swagger resource swagger_resources = [] - for resource in resources: + for resource in command.resources: swagger_resources.append(self.swagger_specs.get_module_manager( plane=self.ws.plane, mod_names=self.ws.mod_names ).get_resource_in_version(resource["id"], resource["version"], resource.rp_name)) # load swagger resource - self.swagger_example_generator.load_examples(swagger_resources, operation_ids) + cmd_operation_ids = {op.operation_id: op for op in command.operations} + self.swagger_example_generator.load_examples(swagger_resources, cmd_operation_ids) - examples = self.swagger_example_generator.create_draft_examples( + examples = self.swagger_example_generator.create_draft_examples_by_swagger( swagger_resources, - operation_ids, - cmd_name, - arg_groups + command, + cmd_operation_ids, + cmd_name ) return examples diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/command/model/configuration/_example_builder.py index 0d1164a2..a2f4c772 100644 --- a/src/aaz_dev/command/model/configuration/_example_builder.py +++ b/src/aaz_dev/command/model/configuration/_example_builder.py @@ -1,31 +1,169 @@ +import json + +from collections import defaultdict from command.controller.cfg_reader import CfgReader +from command.model.configuration import CMDRequestJson, CMDResponseJson +from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter, \ + FormDataParameter +from ._schema import CMDSchema, CMDObjectSchemaBase, CMDObjectSchemaDiscriminator, CMDArraySchemaBase, \ + CMDObjectSchemaAdditionalProperties +from ._utils import CMDArgBuildPrefix class ExampleBuilder: - def __init__(self, arg_groups=None): - self.arg_groups = arg_groups + def __init__(self, operation=None): + self.operation = operation - def mapping(self, swagger_params): + def param_mapping(self, params): raise NotImplementedError() class SwaggerExampleBuilder(ExampleBuilder): - def mapping(self, swagger_params): - import json + def __init__(self, operation=None, command=None, cmd_operation=None, cmd_builder=None): + super().__init__(operation) + + param_models = defaultdict(dict) + for param in self.operation.parameters: + param_models[param.IN_VALUE][param.name] = param.to_cmd(cmd_builder) + + self.op_param_models = param_models + self.command = command + self.cmd_operation = cmd_operation + + def build_arg_var(self, params): + param_models = self._build_model(params) + self._build_arg_var_by_model(param_models) + + return param_models + + def param_mapping(self, params): + def _mapping(arg): + if hasattr(arg, 'schema'): + arg = arg.schema + + if hasattr(arg, 'arg_var'): + param, arg_idx = CfgReader.find_arg_in_command_by_var(self.command, arg.arg_var) + if arg_idx: + arg.arg_idx = arg_idx + matched_param_models.append(arg) + + if hasattr(arg, 'props') and arg.props: + for prop in arg.props: + _mapping(prop) + + matched_param_models = [] + for params_models in params.values(): + for model in params_models.values(): + _mapping(model) + + return matched_param_models + + def _build_arg_var_by_model(self, param_models): + def _build_arg_var(schema, parent_schema=None, var_prefix=None): + if var_prefix is None: + if parent_schema is None or parent_schema.arg_var is None: + arg_var = "$" + else: + arg_var = parent_schema.arg_var + else: + arg_var = var_prefix + + if parent_schema is None or parent_schema.arg_var is None: + if isinstance(schema, CMDSchema): + if not arg_var.endswith("$") and not schema.name.startswith('[') and not schema.name.startswith( + '{'): + arg_var += '.' + arg_var += f'{schema.name}'.replace('$', '') # some schema name may contain $ + else: + raise NotImplementedError() + else: + if isinstance(parent_schema, CMDArraySchemaBase): + arg_var += '[]' + elif isinstance(parent_schema, CMDObjectSchemaAdditionalProperties): + arg_var += '{}' + elif isinstance(parent_schema, (CMDObjectSchemaBase, CMDObjectSchemaDiscriminator)): + if not isinstance(schema, CMDObjectSchemaAdditionalProperties): + if not arg_var.endswith("$"): + arg_var += '.' + if isinstance(schema, CMDObjectSchemaDiscriminator): + arg_var += schema.get_safe_value() + elif isinstance(schema, CMDSchema): + arg_var += f'{schema.name}'.replace('$', '') # some schema name may contain $ + else: + raise NotImplementedError() + else: + raise NotImplementedError() + cls_name = getattr(parent_schema, 'cls', None) + if cls_name is not None: + arg_var = arg_var.replace(parent_schema.arg_var, f"@{cls_name}") + + return arg_var + + def _build_schema_arg_var(schema, parent_schema=None, var_prefix=None): + arg_var = _build_arg_var(schema, parent_schema, var_prefix) + schema.arg_var = arg_var + + if hasattr(schema, 'props') and schema.props: + for prop in schema.props: + _build_schema_arg_var(prop, schema, var_prefix) + + if PathParameter.IN_VALUE in param_models: + for _, model in param_models[PathParameter.IN_VALUE].items(): + _build_schema_arg_var(schema=model, parent_schema=None, var_prefix=CMDArgBuildPrefix.Path) + + if QueryParameter.IN_VALUE in param_models: + for _, model in param_models[QueryParameter.IN_VALUE].items(): + _build_schema_arg_var(schema=model, parent_schema=None, var_prefix=CMDArgBuildPrefix.Query) + + if HeaderParameter.IN_VALUE in param_models: + for _, model in param_models[HeaderParameter.IN_VALUE].items(): + _build_schema_arg_var(schema=model, parent_schema=None, var_prefix=CMDArgBuildPrefix.Header) + + if BodyParameter.IN_VALUE in param_models: + for _, model in param_models[BodyParameter.IN_VALUE].items(): + _build_schema_arg_var(schema=model.schema, parent_schema=None, var_prefix=None) + + if FormDataParameter.IN_VALUE in param_models: + raise NotImplementedError() + + def _build_model(self, params): + def build_sub_param_model(new_model, old_model, example_params): + new_model.value = json.dumps(example_params) + if not hasattr(old_model, 'props') or not old_model.props: + return + + new_model_props = [] + for prop in old_model.props: + if prop.name in example_params: + new_prop_model = type(prop)() + new_prop_model.name = prop.name + new_model_props.append(new_prop_model) + build_sub_param_model(new_prop_model, prop, example_params[prop.name]) + + new_model.props = new_model_props + + param_models = defaultdict(dict) + for in_value, op_params in self.op_param_models.items(): + for param_name, param_model in op_params.items(): + + if param_name in params: + new_model = type(param_model)() + new_model.name = param_name + param_models[in_value][param_name] = new_model - cmd_params = {} - for k, v in swagger_params.items(): - _, _, arg_idx = CfgReader.find_arg_in_arg_groups_by_name(self.arg_groups, k) - if not arg_idx or arg_idx in cmd_params: - continue + if isinstance(param_model, (CMDRequestJson, CMDResponseJson)): + new_model.schema = type(param_model.schema)() + new_model = new_model.schema + new_model.name = param_model.schema.name + param_model = param_model.schema - cmd_params[arg_idx] = json.dumps(v) + build_sub_param_model(new_model, param_model, params[param_name]) - return cmd_params + return param_models - # def iter_swagger_params(self, swagger_params): - # for k, v in swagger_params.items(): - # yield k, v - # if isinstance(v, dict): - # for k1, v1 in self.iter_swagger_params(v): - # yield k1, v1 + # def _find_param_in_operation(self, example_param): + # for param in self.operation.parameters: + # if param.name == example_param: + # return param + # + # return None diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index d0a799b5..ea9e5149 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -1,5 +1,7 @@ from command.model.configuration._example_builder import SwaggerExampleBuilder +from swagger.model.schema.cmd_builder import CMDBuilder from swagger.model.schema.example_item import XmsExamplesField +from swagger.model.schema.fields import MutabilityEnum from swagger.model.schema.path_item import PathItem from swagger.model.specs import SwaggerLoader @@ -8,14 +10,13 @@ class ExampleGenerator: def __init__(self): self.loader = SwaggerLoader() - def load_examples(self, resources, operation_ids): + def load_examples(self, resources, cmd_operation_ids): for resource in resources: self.loader.load_file(resource.file_path) - self.loader.link_examples(resource.file_path, resource.path, operation_ids) + self.loader.link_examples(resource.file_path, resource.path, cmd_operation_ids) - def create_draft_examples(self, resources, operation_ids, cmd_name, arg_groups): + def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids, cmd_name): cmd_examples = [] - example_builder = SwaggerExampleBuilder(arg_groups=arg_groups) for resource in resources: swagger = self.loader.get_loaded(resource.file_path) @@ -28,19 +29,52 @@ def create_draft_examples(self, resources, operation_ids, cmd_name, arg_groups): if not isinstance(path_item, PathItem): continue - swagger_examples = XmsExamplesField() - if path_item.get is not None and path_item.get.operation_id in operation_ids: - swagger_examples = path_item.get.x_ms_examples - elif path_item.delete is not None and path_item.delete.operation_id in operation_ids: - swagger_examples = path_item.delete.x_ms_examples - elif path_item.put is not None and path_item.put.operation_id in operation_ids: - swagger_examples = path_item.put.x_ms_examples - elif path_item.post is not None and path_item.post.operation_id in operation_ids: - swagger_examples = path_item.post.x_ms_examples - elif path_item.head is not None and path_item.head.operation_id in operation_ids: - swagger_examples = path_item.head.x_ms_examples - - cmd_examples.extend(self.generate_examples(cmd_name, swagger_examples, example_builder)) + example_builder = None + examples = XmsExamplesField() + if path_item.get is not None and path_item.get.operation_id in cmd_operation_ids: + cmd_builder = CMDBuilder(path=resource.path, method='get', mutability=MutabilityEnum.Read) + example_builder = SwaggerExampleBuilder(operation=path_item.get, + command=command, + cmd_operation=cmd_operation_ids[path_item.get.operation_id], + cmd_builder=cmd_builder) + examples = path_item.get.x_ms_examples + + elif path_item.delete is not None and path_item.delete.operation_id in cmd_operation_ids: + cmd_builder = CMDBuilder(path=resource.path, method='delete', mutability=MutabilityEnum.Create) + example_builder = SwaggerExampleBuilder(operation=path_item.delete, + command=command, + cmd_operation=cmd_operation_ids[path_item.delete.operation_id], + cmd_builder=cmd_builder) + examples = path_item.delete.x_ms_examples + + elif path_item.put is not None and path_item.put.operation_id in cmd_operation_ids: + cmd_builder = CMDBuilder(path=resource.path, method='put', mutability=MutabilityEnum.Create) + example_builder = SwaggerExampleBuilder(operation=path_item.put, + command=command, + cmd_operation=cmd_operation_ids[path_item.put.operation_id], + cmd_builder=cmd_builder) + examples = path_item.put.x_ms_examples + + elif path_item.post is not None and path_item.post.operation_id in cmd_operation_ids: + cmd_builder = CMDBuilder(path=resource.path, method='post', mutability=MutabilityEnum.Create) + example_builder = SwaggerExampleBuilder(operation=path_item.post, + command=command, + cmd_operation=cmd_operation_ids[path_item.post.operation_id], + cmd_builder=cmd_builder) + examples = path_item.post.x_ms_examples + + elif path_item.head is not None and path_item.head.operation_id in cmd_operation_ids: + cmd_builder = CMDBuilder(path=resource.path, method='head', mutability=MutabilityEnum.Read) + example_builder = SwaggerExampleBuilder(operation=path_item.head, + command=command, + cmd_operation=cmd_operation_ids[path_item.head.operation_id], + cmd_builder=cmd_builder) + examples = path_item.head.x_ms_examples + + if not example_builder: + continue + + cmd_examples.extend(self.generate_examples(cmd_name, examples, example_builder)) return cmd_examples diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index c5bf6380..98949621 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -20,16 +20,16 @@ def link(self, swagger_loader, *traces): self.ref_instance, instance_traces = swagger_loader.load_ref(self.ref, *self.traces, "ref") - def to_cmd(self, builder, cmd_name, **kwargs): + def to_cmd(self, example_builder, cmd_name, **kwargs): if not self.ref_instance: return - params = builder.mapping(self.ref_instance.get("parameters", {})) + param_models = example_builder.build_arg_var(self.ref_instance.get("parameters", {})) + matched_param_models = example_builder.param_mapping(param_models) command = cmd_name - for k, v in params.items(): - command += f" --{k} {v}" - # command += " --" + k + " " + v + for model in matched_param_models: + command += f" --{model.arg_idx} {model.value}" return CMDCommandExample({"commands": [command.strip()]}) diff --git a/src/aaz_dev/swagger/model/schema/path_item.py b/src/aaz_dev/swagger/model/schema/path_item.py index 1ddaa12a..10c4de97 100644 --- a/src/aaz_dev/swagger/model/schema/path_item.py +++ b/src/aaz_dev/swagger/model/schema/path_item.py @@ -21,7 +21,7 @@ class PathItem(Model, Linkable): # options = ModelType(Operation) # A definition of a OPTIONS operation on this path. # ref = ReferenceType() # Allows for an external definition of this path item. The referenced structure MUST be in the format of a Path Item Object. If there are conflicts between the referenced definition and this Path Item's definition, the behavior is undefined. - def link(self, swagger_loader, *traces): + def link(self, swagger_loader, *traces, **kwargs): if self.is_linked(): return super().link(swagger_loader, *traces) @@ -39,17 +39,18 @@ def link(self, swagger_loader, *traces): assert isinstance(param, ParameterBase) self.parameters[idx] = param - if self.get is not None: + link_operation_ids = kwargs.get('link_operation_ids', None) + if self.get is not None and (not link_operation_ids or self.get.operation_id in link_operation_ids): self.get.link(swagger_loader, *self.traces, 'get') - if self.put is not None: + if self.put is not None and (not link_operation_ids or self.put.operation_id in link_operation_ids): self.put.link(swagger_loader, *self.traces, 'put') - if self.post is not None: + if self.post is not None and (not link_operation_ids or self.post.operation_id in link_operation_ids): self.post.link(swagger_loader, *self.traces, 'post') - if self.delete is not None: + if self.delete is not None and (not link_operation_ids or self.delete.operation_id in link_operation_ids): self.delete.link(swagger_loader, *self.traces, 'delete') - if self.head is not None: + if self.head is not None and (not link_operation_ids or self.head.operation_id in link_operation_ids): self.head.link(swagger_loader, *self.traces, 'head') - if self.patch is not None: + if self.patch is not None and (not link_operation_ids or self.patch.operation_id in link_operation_ids): self.patch.link(swagger_loader, *self.traces, 'patch') def link_examples(self, swagger_loader, operation_ids, *traces): diff --git a/src/aaz_dev/swagger/model/schema/swagger.py b/src/aaz_dev/swagger/model/schema/swagger.py index 2e69e6c9..214c96e8 100644 --- a/src/aaz_dev/swagger/model/schema/swagger.py +++ b/src/aaz_dev/swagger/model/schema/swagger.py @@ -57,14 +57,16 @@ class Swagger(Model, Linkable): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def link(self, swagger_loader, *traces): + def link(self, swagger_loader, *traces, **kwargs): if self.is_linked(): return super().link(swagger_loader, *traces) + link_path = kwargs.get('link_path', None) if self.paths is not None: for key, path in self.paths.items(): - path.link(swagger_loader, *self.traces, 'paths', key) + if not link_path or link_path == key: + path.link(swagger_loader, *self.traces, 'paths', key, **kwargs) if self.definitions is not None: for key, definition in self.definitions.items(): @@ -87,7 +89,7 @@ def link(self, swagger_loader, *traces): self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') def link_examples(self, swagger_loader, path, operation_ids, *traces): - super().link(swagger_loader, *traces) + self.link(swagger_loader, *traces, link_path=path, link_operation_ids=operation_ids) if self.paths is not None: for key, path_item in self.paths.items(): From cc3bb1b0ace9e281aed0725db48da2e994497194 Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 21 Nov 2023 23:00:09 +0800 Subject: [PATCH 07/37] feat: build arg_var by arg schema --- .../model/configuration/_example_builder.py | 184 ++++-------------- .../swagger/controller/example_generator.py | 25 +-- .../swagger/model/schema/example_item.py | 8 +- 3 files changed, 47 insertions(+), 170 deletions(-) diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/command/model/configuration/_example_builder.py index a2f4c772..a4f0f0f2 100644 --- a/src/aaz_dev/command/model/configuration/_example_builder.py +++ b/src/aaz_dev/command/model/configuration/_example_builder.py @@ -1,12 +1,8 @@ import json -from collections import defaultdict from command.controller.cfg_reader import CfgReader -from command.model.configuration import CMDRequestJson, CMDResponseJson -from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter, \ - FormDataParameter -from ._schema import CMDSchema, CMDObjectSchemaBase, CMDObjectSchemaDiscriminator, CMDArraySchemaBase, \ - CMDObjectSchemaAdditionalProperties +from command.model.configuration._arg_group import CMDArgGroup +from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter from ._utils import CMDArgBuildPrefix @@ -19,151 +15,47 @@ def param_mapping(self, params): class SwaggerExampleBuilder(ExampleBuilder): - def __init__(self, operation=None, command=None, cmd_operation=None, cmd_builder=None): + def __init__(self, operation=None, command=None): super().__init__(operation) - param_models = defaultdict(dict) - for param in self.operation.parameters: - param_models[param.IN_VALUE][param.name] = param.to_cmd(cmd_builder) - - self.op_param_models = param_models self.command = command - self.cmd_operation = cmd_operation - def build_arg_var(self, params): - param_models = self._build_model(params) - self._build_arg_var_by_model(param_models) + def param_mapping(self, example_dict): + def build_example_param(arg_var, value): + parent, arg, arg_idx = CfgReader.find_arg_in_command_with_parent_by_var(self.command, arg_var) - return param_models + # ignore parameter flattened or not found + if arg and isinstance(parent, CMDArgGroup): + example_params.append((arg_idx, json.dumps(value))) - def param_mapping(self, params): - def _mapping(arg): - if hasattr(arg, 'schema'): - arg = arg.schema - - if hasattr(arg, 'arg_var'): - param, arg_idx = CfgReader.find_arg_in_command_by_var(self.command, arg.arg_var) - if arg_idx: - arg.arg_idx = arg_idx - matched_param_models.append(arg) - - if hasattr(arg, 'props') and arg.props: - for prop in arg.props: - _mapping(prop) - - matched_param_models = [] - for params_models in params.values(): - for model in params_models.values(): - _mapping(model) - - return matched_param_models - - def _build_arg_var_by_model(self, param_models): - def _build_arg_var(schema, parent_schema=None, var_prefix=None): - if var_prefix is None: - if parent_schema is None or parent_schema.arg_var is None: - arg_var = "$" - else: - arg_var = parent_schema.arg_var - else: - arg_var = var_prefix - - if parent_schema is None or parent_schema.arg_var is None: - if isinstance(schema, CMDSchema): - if not arg_var.endswith("$") and not schema.name.startswith('[') and not schema.name.startswith( - '{'): - arg_var += '.' - arg_var += f'{schema.name}'.replace('$', '') # some schema name may contain $ - else: - raise NotImplementedError() - else: - if isinstance(parent_schema, CMDArraySchemaBase): - arg_var += '[]' - elif isinstance(parent_schema, CMDObjectSchemaAdditionalProperties): - arg_var += '{}' - elif isinstance(parent_schema, (CMDObjectSchemaBase, CMDObjectSchemaDiscriminator)): - if not isinstance(schema, CMDObjectSchemaAdditionalProperties): - if not arg_var.endswith("$"): - arg_var += '.' - if isinstance(schema, CMDObjectSchemaDiscriminator): - arg_var += schema.get_safe_value() - elif isinstance(schema, CMDSchema): - arg_var += f'{schema.name}'.replace('$', '') # some schema name may contain $ - else: - raise NotImplementedError() - else: - raise NotImplementedError() - cls_name = getattr(parent_schema, 'cls', None) - if cls_name is not None: - arg_var = arg_var.replace(parent_schema.arg_var, f"@{cls_name}") - - return arg_var - - def _build_schema_arg_var(schema, parent_schema=None, var_prefix=None): - arg_var = _build_arg_var(schema, parent_schema, var_prefix) - schema.arg_var = arg_var - - if hasattr(schema, 'props') and schema.props: - for prop in schema.props: - _build_schema_arg_var(prop, schema, var_prefix) - - if PathParameter.IN_VALUE in param_models: - for _, model in param_models[PathParameter.IN_VALUE].items(): - _build_schema_arg_var(schema=model, parent_schema=None, var_prefix=CMDArgBuildPrefix.Path) - - if QueryParameter.IN_VALUE in param_models: - for _, model in param_models[QueryParameter.IN_VALUE].items(): - _build_schema_arg_var(schema=model, parent_schema=None, var_prefix=CMDArgBuildPrefix.Query) - - if HeaderParameter.IN_VALUE in param_models: - for _, model in param_models[HeaderParameter.IN_VALUE].items(): - _build_schema_arg_var(schema=model, parent_schema=None, var_prefix=CMDArgBuildPrefix.Header) - - if BodyParameter.IN_VALUE in param_models: - for _, model in param_models[BodyParameter.IN_VALUE].items(): - _build_schema_arg_var(schema=model.schema, parent_schema=None, var_prefix=None) - - if FormDataParameter.IN_VALUE in param_models: - raise NotImplementedError() - - def _build_model(self, params): - def build_sub_param_model(new_model, old_model, example_params): - new_model.value = json.dumps(example_params) - if not hasattr(old_model, 'props') or not old_model.props: + def build_body_example_param(parent_arg_var, example): + if not isinstance(example, dict): return - new_model_props = [] - for prop in old_model.props: - if prop.name in example_params: - new_prop_model = type(prop)() - new_prop_model.name = prop.name - new_model_props.append(new_prop_model) - build_sub_param_model(new_prop_model, prop, example_params[prop.name]) - - new_model.props = new_model_props - - param_models = defaultdict(dict) - for in_value, op_params in self.op_param_models.items(): - for param_name, param_model in op_params.items(): - - if param_name in params: - new_model = type(param_model)() - new_model.name = param_name - param_models[in_value][param_name] = new_model - - if isinstance(param_model, (CMDRequestJson, CMDResponseJson)): - new_model.schema = type(param_model.schema)() - new_model = new_model.schema - new_model.name = param_model.schema.name - param_model = param_model.schema - - build_sub_param_model(new_model, param_model, params[param_name]) - - return param_models - - # def _find_param_in_operation(self, example_param): - # for param in self.operation.parameters: - # if param.name == example_param: - # return param - # - # return None + for example_param, example_value in example.items(): + arg_var = parent_arg_var + '.' + example_param + build_example_param(arg_var, example_value) + build_body_example_param(arg_var, example_value) + + example_params = [] + + for op_param in self.operation.parameters: + if op_param.name in example_dict: + if PathParameter.IN_VALUE == op_param.IN_VALUE: + path_arg_var = CMDArgBuildPrefix.Path + '.' + op_param.name.replace('$', '') + build_example_param(path_arg_var, example_dict[op_param.name]) + + elif QueryParameter.IN_VALUE == op_param.IN_VALUE: + query_arg_var = CMDArgBuildPrefix.Query + '.' + op_param.name.replace('$', '') + build_example_param(query_arg_var, example_dict[op_param.name]) + + elif HeaderParameter.IN_VALUE == op_param.IN_VALUE: + header_arg_var = CMDArgBuildPrefix.Header + '.' + op_param.name.replace('$', '') + build_example_param(header_arg_var, example_dict[op_param.name]) + + elif BodyParameter.IN_VALUE == op_param.IN_VALUE: + body_arg_var = '$' + op_param.name.replace('$', '') + build_example_param(body_arg_var, example_dict[op_param.name]) + build_body_example_param(body_arg_var, example_dict[op_param.name]) + + return example_params diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index ea9e5149..7961f86c 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -32,43 +32,28 @@ def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids example_builder = None examples = XmsExamplesField() if path_item.get is not None and path_item.get.operation_id in cmd_operation_ids: - cmd_builder = CMDBuilder(path=resource.path, method='get', mutability=MutabilityEnum.Read) example_builder = SwaggerExampleBuilder(operation=path_item.get, - command=command, - cmd_operation=cmd_operation_ids[path_item.get.operation_id], - cmd_builder=cmd_builder) + command=command) examples = path_item.get.x_ms_examples elif path_item.delete is not None and path_item.delete.operation_id in cmd_operation_ids: - cmd_builder = CMDBuilder(path=resource.path, method='delete', mutability=MutabilityEnum.Create) example_builder = SwaggerExampleBuilder(operation=path_item.delete, - command=command, - cmd_operation=cmd_operation_ids[path_item.delete.operation_id], - cmd_builder=cmd_builder) + command=command) examples = path_item.delete.x_ms_examples elif path_item.put is not None and path_item.put.operation_id in cmd_operation_ids: - cmd_builder = CMDBuilder(path=resource.path, method='put', mutability=MutabilityEnum.Create) example_builder = SwaggerExampleBuilder(operation=path_item.put, - command=command, - cmd_operation=cmd_operation_ids[path_item.put.operation_id], - cmd_builder=cmd_builder) + command=command) examples = path_item.put.x_ms_examples elif path_item.post is not None and path_item.post.operation_id in cmd_operation_ids: - cmd_builder = CMDBuilder(path=resource.path, method='post', mutability=MutabilityEnum.Create) example_builder = SwaggerExampleBuilder(operation=path_item.post, - command=command, - cmd_operation=cmd_operation_ids[path_item.post.operation_id], - cmd_builder=cmd_builder) + command=command) examples = path_item.post.x_ms_examples elif path_item.head is not None and path_item.head.operation_id in cmd_operation_ids: - cmd_builder = CMDBuilder(path=resource.path, method='head', mutability=MutabilityEnum.Read) example_builder = SwaggerExampleBuilder(operation=path_item.head, - command=command, - cmd_operation=cmd_operation_ids[path_item.head.operation_id], - cmd_builder=cmd_builder) + command=command) examples = path_item.head.x_ms_examples if not example_builder: diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index 98949621..54cedfe9 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -24,12 +24,11 @@ def to_cmd(self, example_builder, cmd_name, **kwargs): if not self.ref_instance: return - param_models = example_builder.build_arg_var(self.ref_instance.get("parameters", {})) - matched_param_models = example_builder.param_mapping(param_models) + example_params = example_builder.param_mapping(self.ref_instance.get("parameters", {})) command = cmd_name - for model in matched_param_models: - command += f" --{model.arg_idx} {model.value}" + for param_option, param_value in example_params: + command += f" --{param_option} {param_value}" return CMDCommandExample({"commands": [command.strip()]}) @@ -41,6 +40,7 @@ class XmsExamplesField(DictType): https://github.com/Azure/azure-rest-api-specs/blob/master/documentation/x-ms-examples.md """ + def __init__(self, **kwargs): super().__init__( field=ModelType(ExampleItem), From 5135350379abeb557c8625bab1dac47431800479 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 22 Nov 2023 14:42:40 +0800 Subject: [PATCH 08/37] refactor: clean recursive build logic --- .../model/configuration/_example_builder.py | 103 ++++++++++-------- .../swagger/controller/example_generator.py | 15 +-- .../swagger/model/schema/example_item.py | 2 +- 3 files changed, 62 insertions(+), 58 deletions(-) diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/command/model/configuration/_example_builder.py index a4f0f0f2..a644ae52 100644 --- a/src/aaz_dev/command/model/configuration/_example_builder.py +++ b/src/aaz_dev/command/model/configuration/_example_builder.py @@ -1,61 +1,70 @@ import json +from abc import abstractmethod from command.controller.cfg_reader import CfgReader -from command.model.configuration._arg_group import CMDArgGroup +from command.model.configuration import CMDArgGroup from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter from ._utils import CMDArgBuildPrefix class ExampleBuilder: - def __init__(self, operation=None): - self.operation = operation - - def param_mapping(self, params): - raise NotImplementedError() - - -class SwaggerExampleBuilder(ExampleBuilder): - def __init__(self, operation=None, command=None): - super().__init__(operation) - + def __init__(self, command=None, operation=None): self.command = command + self.operation = operation + self.example_items = [] - def param_mapping(self, example_dict): - def build_example_param(arg_var, value): - parent, arg, arg_idx = CfgReader.find_arg_in_command_with_parent_by_var(self.command, arg_var) - - # ignore parameter flattened or not found - if arg and isinstance(parent, CMDArgGroup): - example_params.append((arg_idx, json.dumps(value))) - - def build_body_example_param(parent_arg_var, example): - if not isinstance(example, dict): - return - - for example_param, example_value in example.items(): - arg_var = parent_arg_var + '.' + example_param - build_example_param(arg_var, example_value) - build_body_example_param(arg_var, example_value) - - example_params = [] - - for op_param in self.operation.parameters: - if op_param.name in example_dict: - if PathParameter.IN_VALUE == op_param.IN_VALUE: - path_arg_var = CMDArgBuildPrefix.Path + '.' + op_param.name.replace('$', '') - build_example_param(path_arg_var, example_dict[op_param.name]) + def get_option_name(self, arg_var): + if not arg_var: + return - elif QueryParameter.IN_VALUE == op_param.IN_VALUE: - query_arg_var = CMDArgBuildPrefix.Query + '.' + op_param.name.replace('$', '') - build_example_param(query_arg_var, example_dict[op_param.name]) + arg_parent, arg, arg_option = CfgReader.find_arg_in_command_with_parent_by_var(self.command, arg_var) + if isinstance(arg_parent, CMDArgGroup) and arg: + return arg_option # top-level parameter - elif HeaderParameter.IN_VALUE == op_param.IN_VALUE: - header_arg_var = CMDArgBuildPrefix.Header + '.' + op_param.name.replace('$', '') - build_example_param(header_arg_var, example_dict[op_param.name]) + @abstractmethod + def mapping(self, example_dict): + pass - elif BodyParameter.IN_VALUE == op_param.IN_VALUE: - body_arg_var = '$' + op_param.name.replace('$', '') - build_example_param(body_arg_var, example_dict[op_param.name]) - build_body_example_param(body_arg_var, example_dict[op_param.name]) - return example_params +class SwaggerExampleBuilder(ExampleBuilder): + def mapping(self, example_dict): + for param in self.operation.parameters: + if param.name not in example_dict: + continue + + arg_var = None + value = example_dict[param.name] + param_name = param.name.replace("$", "") + + if param.IN_VALUE == BodyParameter.IN_VALUE: + arg_var = f"${param_name}" + self.example_items += self.build(arg_var, value) + else: + if param.IN_VALUE == PathParameter.IN_VALUE: + arg_var = f"{CMDArgBuildPrefix.Path}.{param_name}" + if param.IN_VALUE == QueryParameter.IN_VALUE: + arg_var = f"{CMDArgBuildPrefix.Query}.{param_name}" + if param.IN_VALUE == HeaderParameter.IN_VALUE: + arg_var = f"{CMDArgBuildPrefix.Header}.{param_name}" + + option = self.get_option_name(arg_var) + if option: + self.example_items.append((option, json.dumps(value))) + + return self.example_items + + def build(self, var_prefix, example_dict): + if not isinstance(example_dict, dict): + return [] + + example_items = [] + for name, value in example_dict.items(): + arg_var = f"{var_prefix}.{name}" + + option = self.get_option_name(arg_var) + if option: + example_items.append((option, json.dumps(value))) + + example_items += self.build(arg_var, value) + + return example_items diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index 7961f86c..2ef52048 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -32,28 +32,23 @@ def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids example_builder = None examples = XmsExamplesField() if path_item.get is not None and path_item.get.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(operation=path_item.get, - command=command) + example_builder = SwaggerExampleBuilder(command=command, operation=path_item.get) examples = path_item.get.x_ms_examples elif path_item.delete is not None and path_item.delete.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(operation=path_item.delete, - command=command) + example_builder = SwaggerExampleBuilder(command=command, operation=path_item.delete) examples = path_item.delete.x_ms_examples elif path_item.put is not None and path_item.put.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(operation=path_item.put, - command=command) + example_builder = SwaggerExampleBuilder(command=command, operation=path_item.put) examples = path_item.put.x_ms_examples elif path_item.post is not None and path_item.post.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(operation=path_item.post, - command=command) + example_builder = SwaggerExampleBuilder(command=command, operation=path_item.post) examples = path_item.post.x_ms_examples elif path_item.head is not None and path_item.head.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(operation=path_item.head, - command=command) + example_builder = SwaggerExampleBuilder(command=command, operation=path_item.head) examples = path_item.head.x_ms_examples if not example_builder: diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index 54cedfe9..6f03f816 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -24,7 +24,7 @@ def to_cmd(self, example_builder, cmd_name, **kwargs): if not self.ref_instance: return - example_params = example_builder.param_mapping(self.ref_instance.get("parameters", {})) + example_params = example_builder.mapping(self.ref_instance.get("parameters", {})) command = cmd_name for param_option, param_value in example_params: From f791932a4c5bf965e93d5de2fb4a53f539493715 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 29 Nov 2023 16:02:52 +0800 Subject: [PATCH 09/37] feat: support array/dict type --- .../model/configuration/_example_builder.py | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/command/model/configuration/_example_builder.py index a644ae52..241fbf38 100644 --- a/src/aaz_dev/command/model/configuration/_example_builder.py +++ b/src/aaz_dev/command/model/configuration/_example_builder.py @@ -7,20 +7,32 @@ from ._utils import CMDArgBuildPrefix +class ExampleItem: + def __init__(self, command=None, arg_var=None, key=None, val=None): + self.arg_var = arg_var + self.key = key + self.val = val + + self.arg_parent, self.arg, self.arg_option = CfgReader.find_arg_in_command_with_parent_by_var(command, arg_var) + + if self.arg_option is not None: + self.arg_option = self.arg_option.split(".")[-1] + + @property + def is_flat(self): + return self.arg_parent and not self.arg + + @property + def is_top_level(self): + return isinstance(self.arg_parent, CMDArgGroup) and self.arg + + class ExampleBuilder: def __init__(self, command=None, operation=None): self.command = command self.operation = operation self.example_items = [] - def get_option_name(self, arg_var): - if not arg_var: - return - - arg_parent, arg, arg_option = CfgReader.find_arg_in_command_with_parent_by_var(self.command, arg_var) - if isinstance(arg_parent, CMDArgGroup) and arg: - return arg_option # top-level parameter - @abstractmethod def mapping(self, example_dict): pass @@ -47,24 +59,34 @@ def mapping(self, example_dict): if param.IN_VALUE == HeaderParameter.IN_VALUE: arg_var = f"{CMDArgBuildPrefix.Header}.{param_name}" - option = self.get_option_name(arg_var) - if option: - self.example_items.append((option, json.dumps(value))) + item = ExampleItem(command=self.command, arg_var=arg_var, key=param_name, val=value) + if item.is_top_level: + self.example_items.append((item.arg_option, json.dumps(value))) return self.example_items def build(self, var_prefix, example_dict): - if not isinstance(example_dict, dict): - return [] - example_items = [] - for name, value in example_dict.items(): - arg_var = f"{var_prefix}.{name}" - - option = self.get_option_name(arg_var) - if option: - example_items.append((option, json.dumps(value))) - - example_items += self.build(arg_var, value) + if isinstance(example_dict, list): + arg_var = f"{var_prefix}[]" + for item in example_dict: + example_items += self.build(arg_var, item) + elif isinstance(example_dict, dict): + for name, value in example_dict.copy().items(): + item = ExampleItem(command=self.command, arg_var=f"{var_prefix}{{}}.{name}", key=name, val=value) + if item.arg is None: + item = ExampleItem(command=self.command, arg_var=f"{var_prefix}.{name}", key=name, val=value) + + example_items += self.build(item.arg_var, value) + + if item.is_top_level: + example_items.append((item.arg_option, json.dumps(value))) + elif item.arg_option: + example_dict.pop(item.key) + example_dict[item.arg_option] = item.val + elif item.is_flat: + example_dict.pop(item.key, None) + for k, v in item.val.items(): + example_dict[k] = v return example_items From c4e440926b56af588237346bdcc6570a6fdc3356 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 29 Nov 2023 16:03:36 +0800 Subject: [PATCH 10/37] test: add test case for array/dict type --- .../command/tests/api_tests/test_editor.py | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index 0388b87e..24fcd740 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -1903,8 +1903,48 @@ def test_workspace_add_subresource_commands(self, ws_name): data = rv.get_json() self.assertTrue(len(data) == 4) - @workspace_name("test_workspace_generate_examples") - def test_workspace_generate_examples(self, ws_name): + @workspace_name("test_workspace_generate_examples_array") + def test_workspace_generate_examples_array(self, ws_name): + module = "eventhub" + api_version = "2021-11-01" + + with self.app.test_client() as c: + rv = c.post(f"/AAZ/Editor/Workspaces", json={ + "name": ws_name, + "plane": PlaneEnum.Mgmt, + "modNames": module, + "resourceProvider": "Microsoft.EventHub" + }) + self.assertTrue(rv.status_code == 200) + + ws_url = rv.get_json()["url"] + resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.EventHub/namespaces/{namespaceName}/networkRuleSets/default") + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", json={ + "module": module, + "version": api_version, + "resources": [{ + "id": resource_id, + "options": { + "aaz_version": None + } + }] + }) + self.assertTrue(rv.status_code == 200) + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/" + f"Arguments/$parameters.properties.virtualNetworkRules[].subnet/Flatten", json={ + "subArgsOptions": { + "$parameters.properties.virtualNetworkRules[].subnet.id": ["id"] + } + }) + self.assertTrue(rv.status_code == 200) + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/GenerateExamples") + self.assertTrue(rv.status_code == 200) + + @workspace_name("test_workspace_generate_examples_dict") + def test_workspace_generate_examples_dict(self, ws_name): module = "eventhub" api_version = "2021-11-01" From 2efc7e08a0298716ad0bef6074b5329a683e60e7 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 30 Nov 2023 14:53:09 +0800 Subject: [PATCH 11/37] feat: support generation when apply flatten --- .../command/model/configuration/_example_builder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/command/model/configuration/_example_builder.py index 241fbf38..ebc43702 100644 --- a/src/aaz_dev/command/model/configuration/_example_builder.py +++ b/src/aaz_dev/command/model/configuration/_example_builder.py @@ -46,7 +46,7 @@ def mapping(self, example_dict): arg_var = None value = example_dict[param.name] - param_name = param.name.replace("$", "") + param_name = param.name.replace("$", "") # schema name may contain $ if param.IN_VALUE == BodyParameter.IN_VALUE: arg_var = f"${param_name}" @@ -81,12 +81,12 @@ def build(self, var_prefix, example_dict): if item.is_top_level: example_items.append((item.arg_option, json.dumps(value))) - elif item.arg_option: - example_dict.pop(item.key) - example_dict[item.arg_option] = item.val elif item.is_flat: - example_dict.pop(item.key, None) + example_dict.pop(item.key) for k, v in item.val.items(): example_dict[k] = v + elif item.arg_option: + example_dict.pop(item.key) + example_dict[item.arg_option] = item.val return example_items From 4d1b3ce14a0bd7841bfa6bb3d22bc4a0408b07f2 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 6 Dec 2023 18:16:18 +0800 Subject: [PATCH 12/37] test: add test case for poly --- .../command/tests/api_tests/test_editor.py | 103 +++++++++++++----- 1 file changed, 78 insertions(+), 25 deletions(-) diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index 24fcd740..31e20e98 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -1920,24 +1920,31 @@ def test_workspace_generate_examples_array(self, ws_name): ws_url = rv.get_json()["url"] resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.EventHub/namespaces/{namespaceName}/networkRuleSets/default") - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", json={ - "module": module, - "version": api_version, - "resources": [{ - "id": resource_id, - "options": { - "aaz_version": None - } - }] - }) + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", + json={ + "module": module, + "version": api_version, + "resources": [ + { + "id": resource_id, + "options": { + "aaz_version": None + } + } + ] + } + ) self.assertTrue(rv.status_code == 200) - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/" - f"Arguments/$parameters.properties.virtualNetworkRules[].subnet/Flatten", json={ - "subArgsOptions": { - "$parameters.properties.virtualNetworkRules[].subnet.id": ["id"] + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/Arguments/$parameters.properties.virtualNetworkRules[].subnet/Flatten", + json={ + "subArgsOptions": { + "$parameters.properties.virtualNetworkRules[].subnet.id": ["id"] + } } - }) + ) self.assertTrue(rv.status_code == 200) rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/GenerateExamples") @@ -1960,17 +1967,63 @@ def test_workspace_generate_examples_dict(self, ws_name): ws_url = rv.get_json()["url"] resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.EventHub/clusters/{clusterName}") - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", json={ - "module": module, - "version": api_version, - "resources": [{ - "id": resource_id, - "options": { - "aaz_version": None - } - }] - }) + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", + json={ + "module": module, + "version": api_version, + "resources": [ + { + "id": resource_id, + "options": { + "aaz_version": None + } + } + ] + } + ) self.assertTrue(rv.status_code == 200) rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/cluster/Leaves/create/GenerateExamples") self.assertTrue(rv.status_code == 200) + + @workspace_name("test_workspace_generate_examples_poly") + def test_workspace_generate_examples_poly(self, ws_name): + module = "monitor" + api_version = "2018-03-01" + + with self.app.test_client() as c: + rv = c.post( + f"/AAZ/Editor/Workspaces", + json={ + "name": ws_name, + "plane": PlaneEnum.Mgmt, + "modNames": module, + "resourceProvider": "Microsoft.Insights" + } + ) + self.assertTrue(rv.status_code == 200) + + ws_url = rv.get_json()["url"] + + resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Insights/metricAlerts/{ruleName}") + + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", + json={ + "module": module, + "version": api_version, + "resources": [ + { + "id": resource_id, + "options": { + "aaz_version": None + } + } + ] + } + ) + self.assertTrue(rv.status_code == 200) + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/insights/metric-alert/Leaves/create/GenerateExamples") + self.assertTrue(rv.status_code == 200) From a3df057956f0d52520160af16b8db4769c76dda2 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 13 Dec 2023 12:26:38 +0800 Subject: [PATCH 13/37] style: clean ui modifications --- .../workspace/WSEditorCommandContent.tsx | 31 +------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/src/web/src/views/workspace/WSEditorCommandContent.tsx b/src/web/src/views/workspace/WSEditorCommandContent.tsx index a8cf0566..ba410d79 100644 --- a/src/web/src/views/workspace/WSEditorCommandContent.tsx +++ b/src/web/src/views/workspace/WSEditorCommandContent.tsx @@ -1012,35 +1012,6 @@ class ExampleDialog extends React.Component { - let { workspaceUrl, command } = this.props; - - const leafUrl = `${workspaceUrl}/CommandTree/Nodes/aaz/` + command.names.slice(0, -1).join('/') + '/Leaves/' + command.names[command.names.length - 1] + '/GenerateExamples'; - - this.setState({ - updating: true, - }) - axios.post(leafUrl, { - }).then(res => { - const cmd = DecodeResponseCommand(res.data); - this.setState({ - updating: false, - }) - this.props.onClose(cmd); - }).catch(err => { - console.error(err.response); - if (err.response?.data?.message) { - const data = err.response!.data!; - this.setState({ - invalidText: `ResponseError: ${data.message!}: ${JSON.stringify(data.details)}`, - }) - } - this.setState({ - updating: false, - }) - }); - } - handleClose = () => { this.setState({ invalidText: undefined @@ -1174,7 +1145,7 @@ class ExampleDialog extends React.ComponentDelete } - {isAdd && } + {isAdd && } } From 02cf0e49e79e9d898878ab0602d78a8cb1ea2541 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 13 Dec 2023 14:55:06 +0800 Subject: [PATCH 14/37] style: clean code --- src/aaz_dev/command/controller/cfg_reader.py | 28 -------- .../command/controller/workspace_manager.py | 4 +- .../command/tests/api_tests/test_editor.py | 66 +++++++++---------- 3 files changed, 35 insertions(+), 63 deletions(-) diff --git a/src/aaz_dev/command/controller/cfg_reader.py b/src/aaz_dev/command/controller/cfg_reader.py index 0ffae084..c9fe6559 100644 --- a/src/aaz_dev/command/controller/cfg_reader.py +++ b/src/aaz_dev/command/controller/cfg_reader.py @@ -309,34 +309,6 @@ def find_arg_with_parent_by_var(self, *cmd_names, arg_var): return None, None, None return self.find_arg_in_command_with_parent_by_var(command, arg_var=arg_var) - @classmethod - def find_arg_in_arg_groups_by_name(cls, arg_groups, arg_name): - assert isinstance(arg_name, str) - - def arg_filter(_parent, _arg, _arg_idx, _arg_var): - if _arg_var.endswith(f".{arg_name}"): - return (_parent, _arg, _arg_idx, _arg_var), True # find match - - elif _arg_var.startswith(f"{arg_name}."): - return (_parent, None, None, None), True # arg_var already been flattened - - return None, False - - for arg_group in arg_groups: - matches = [match for match in cls._iter_args_in_group(arg_group, arg_filter=arg_filter)] - if not matches: - continue - - assert len(matches) == 1 - - parent, arg, arg_idx, arg_var = matches[0] - if arg: - arg_idx = cls.arg_idx_to_str(arg_idx) - - return parent, arg, arg_idx - - return None, None, None - @classmethod def find_arg_in_command_by_var(cls, command, arg_var): _, arg, arg_idx = cls.find_arg_in_command_with_parent_by_var(command, arg_var=arg_var) diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index 8b146fcc..95ef81ec 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -4,7 +4,8 @@ import shutil from datetime import datetime -from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand, CMDHttpOperation +from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand, \ + CMDBuildInVariants, CMDHttpOperation from command.model.editor import CMDEditorWorkspace, CMDCommandTreeNode, CMDCommandTreeLeaf from swagger.controller.command_generator import CommandGenerator from swagger.controller.example_generator import ExampleGenerator @@ -16,7 +17,6 @@ from .specs_manager import AAZSpecsManager from .workspace_cfg_editor import WorkspaceCfgEditor, build_endpoint_selector_for_client_config from .workspace_client_cfg_editor import WorkspaceClientCfgEditor -from command.model.configuration import CMDHelp, CMDResource, CMDCommandExample, CMDArg, CMDCommand, CMDBuildInVariants logger = logging.getLogger('backend') diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index 07c86317..eaba0f5f 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -2670,41 +2670,41 @@ def test_workspace_generate_examples_dict(self, ws_name): @workspace_name("test_workspace_generate_examples_poly") def test_workspace_generate_examples_poly(self, ws_name): - module = "monitor" - api_version = "2018-03-01" + module = "monitor" + api_version = "2018-03-01" - with self.app.test_client() as c: - rv = c.post( - f"/AAZ/Editor/Workspaces", - json={ + with self.app.test_client() as c: + rv = c.post( + f"/AAZ/Editor/Workspaces", + json={ "name": ws_name, "plane": PlaneEnum.Mgmt, "modNames": module, "resourceProvider": "Microsoft.Insights" - } - ) - self.assertTrue(rv.status_code == 200) - - ws_url = rv.get_json()["url"] - - resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Insights/metricAlerts/{ruleName}") - - rv = c.post( - f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", - json={ - "module": module, - "version": api_version, - "resources": [ - { - "id": resource_id, - "options": { - "aaz_version": None - } - } - ] - } - ) - self.assertTrue(rv.status_code == 200) - - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/insights/metric-alert/Leaves/create/GenerateExamples") - self.assertTrue(rv.status_code == 200) + } + ) + self.assertTrue(rv.status_code == 200) + + ws_url = rv.get_json()["url"] + + resource_id = swagger_resource_path_to_resource_id("/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Insights/metricAlerts/{ruleName}") + + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/AddSwagger", + json={ + "module": module, + "version": api_version, + "resources": [ + { + "id": resource_id, + "options": { + "aaz_version": None + } + } + ] + } + ) + self.assertTrue(rv.status_code == 200) + + rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/insights/metric-alert/Leaves/create/GenerateExamples") + self.assertTrue(rv.status_code == 200) From baf7b12a3a4fd2b8cf6fc1fa951a9a6bec575277 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 14 Dec 2023 16:12:09 +0800 Subject: [PATCH 15/37] style: remove useless snippets --- src/aaz_dev/command/api/editor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index ac5e7e47..942922c2 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -3,7 +3,6 @@ from flask import Blueprint, jsonify, request, url_for, redirect from command.controller.workspace_manager import WorkspaceManager -from swagger.model.specs import SwaggerLoader from utils import exceptions from utils.config import Config from command.model.configuration._utils import CMDArgBuildPrefix From 58b1096e7c7699458c8d84f28be8a672a86ef093 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 14 Dec 2023 23:07:52 +0800 Subject: [PATCH 16/37] chore: resolve comments --- .../controller}/_example_builder.py | 13 ++++++++----- src/aaz_dev/swagger/controller/example_generator.py | 8 +++----- 2 files changed, 11 insertions(+), 10 deletions(-) rename src/aaz_dev/{command/model/configuration => swagger/controller}/_example_builder.py (94%) diff --git a/src/aaz_dev/command/model/configuration/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py similarity index 94% rename from src/aaz_dev/command/model/configuration/_example_builder.py rename to src/aaz_dev/swagger/controller/_example_builder.py index ebc43702..d12721d5 100644 --- a/src/aaz_dev/command/model/configuration/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -4,7 +4,7 @@ from command.controller.cfg_reader import CfgReader from command.model.configuration import CMDArgGroup from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter -from ._utils import CMDArgBuildPrefix +from command.model.configuration._utils import CMDArgBuildPrefix class ExampleItem: @@ -19,7 +19,7 @@ def __init__(self, command=None, arg_var=None, key=None, val=None): self.arg_option = self.arg_option.split(".")[-1] @property - def is_flat(self): + def is_flatten(self): return self.arg_parent and not self.arg @property @@ -28,9 +28,8 @@ def is_top_level(self): class ExampleBuilder: - def __init__(self, command=None, operation=None): + def __init__(self, command=None): self.command = command - self.operation = operation self.example_items = [] @abstractmethod @@ -39,6 +38,10 @@ def mapping(self, example_dict): class SwaggerExampleBuilder(ExampleBuilder): + def __init__(self, command=None, operation=None): + super().__init__(command=command) + self.operation = operation + def mapping(self, example_dict): for param in self.operation.parameters: if param.name not in example_dict: @@ -81,7 +84,7 @@ def build(self, var_prefix, example_dict): if item.is_top_level: example_items.append((item.arg_option, json.dumps(value))) - elif item.is_flat: + elif item.is_flatten: example_dict.pop(item.key) for k, v in item.val.items(): example_dict[k] = v diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index 2ef52048..02770ec4 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -1,7 +1,5 @@ -from command.model.configuration._example_builder import SwaggerExampleBuilder -from swagger.model.schema.cmd_builder import CMDBuilder +from swagger.controller._example_builder import SwaggerExampleBuilder from swagger.model.schema.example_item import XmsExamplesField -from swagger.model.schema.fields import MutabilityEnum from swagger.model.schema.path_item import PathItem from swagger.model.specs import SwaggerLoader @@ -30,7 +28,7 @@ def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids continue example_builder = None - examples = XmsExamplesField() + examples = None if path_item.get is not None and path_item.get.operation_id in cmd_operation_ids: example_builder = SwaggerExampleBuilder(command=command, operation=path_item.get) examples = path_item.get.x_ms_examples @@ -51,7 +49,7 @@ def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids example_builder = SwaggerExampleBuilder(command=command, operation=path_item.head) examples = path_item.head.x_ms_examples - if not example_builder: + if not example_builder or not examples: continue cmd_examples.extend(self.generate_examples(cmd_name, examples, example_builder)) From 50754de7d3efc10af96fe8da5d327f57e5d64477 Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 18 Dec 2023 19:30:53 +0800 Subject: [PATCH 17/37] chore: resolve comments again --- .../command/controller/workspace_manager.py | 2 +- .../swagger/controller/example_generator.py | 4 +-- .../swagger/model/schema/example_item.py | 5 ++- src/aaz_dev/swagger/model/schema/operation.py | 9 +++--- src/aaz_dev/swagger/model/schema/path_item.py | 31 +++++-------------- src/aaz_dev/swagger/model/schema/swagger.py | 14 ++------- .../swagger/model/specs/_swagger_loader.py | 6 ---- 7 files changed, 21 insertions(+), 50 deletions(-) diff --git a/src/aaz_dev/command/controller/workspace_manager.py b/src/aaz_dev/command/controller/workspace_manager.py index 95ef81ec..06b82d72 100644 --- a/src/aaz_dev/command/controller/workspace_manager.py +++ b/src/aaz_dev/command/controller/workspace_manager.py @@ -517,7 +517,7 @@ def generate_operations_examples_by_swagger(self, command, cmd_name): # load swagger resource cmd_operation_ids = {op.operation_id: op for op in command.operations} - self.swagger_example_generator.load_examples(swagger_resources, cmd_operation_ids) + self.swagger_example_generator.load_examples(swagger_resources) examples = self.swagger_example_generator.create_draft_examples_by_swagger( swagger_resources, diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index 02770ec4..fb648ed2 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -8,10 +8,10 @@ class ExampleGenerator: def __init__(self): self.loader = SwaggerLoader() - def load_examples(self, resources, cmd_operation_ids): + def load_examples(self, resources): for resource in resources: self.loader.load_file(resource.file_path) - self.loader.link_examples(resource.file_path, resource.path, cmd_operation_ids) + self.loader.link_swaggers() def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids, cmd_name): cmd_examples = [] diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index 6f03f816..a30b5c45 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -28,7 +28,10 @@ def to_cmd(self, example_builder, cmd_name, **kwargs): command = cmd_name for param_option, param_value in example_params: - command += f" --{param_option} {param_value}" + if len(param_option) == 1: + command += f" -{param_option} {param_value}" + else: + command += f" --{param_option} {param_value}" return CMDCommandExample({"commands": [command.strip()]}) diff --git a/src/aaz_dev/swagger/model/schema/operation.py b/src/aaz_dev/swagger/model/schema/operation.py index 417f5792..76d2e4fc 100644 --- a/src/aaz_dev/swagger/model/schema/operation.py +++ b/src/aaz_dev/swagger/model/schema/operation.py @@ -1,3 +1,4 @@ +import logging from urllib.parse import urljoin from schematics.models import Model @@ -134,12 +135,12 @@ def link(self, swagger_loader, *traces): *self.traces, "x_ms_long_running_operation_options", "final_state_schema" ) - def link_examples(self, swagger_loader, *traces): - super().link(swagger_loader, *traces) - if self.x_ms_examples is not None: for key, example in self.x_ms_examples.items(): - example.link(swagger_loader, *self.traces, "x_ms_examples", key) + try: + example.link(swagger_loader, *self.traces, "x_ms_examples", key) + except e: + logging.warning(f'Link example failed: {key}') def to_cmd(self, builder, parent_parameters, host_path, **kwargs): cmd_op = CMDHttpOperation() diff --git a/src/aaz_dev/swagger/model/schema/path_item.py b/src/aaz_dev/swagger/model/schema/path_item.py index a91a3cd1..06c77409 100644 --- a/src/aaz_dev/swagger/model/schema/path_item.py +++ b/src/aaz_dev/swagger/model/schema/path_item.py @@ -22,7 +22,7 @@ class PathItem(Model, Linkable): # options = ModelType(Operation) # A definition of a OPTIONS operation on this path. # ref = ReferenceType() # Allows for an external definition of this path item. The referenced structure MUST be in the format of a Path Item Object. If there are conflicts between the referenced definition and this Path Item's definition, the behavior is undefined. - def link(self, swagger_loader, *traces, **kwargs): + def link(self, swagger_loader, *traces): if self.is_linked(): return super().link(swagger_loader, *traces) @@ -40,36 +40,19 @@ def link(self, swagger_loader, *traces, **kwargs): assert isinstance(param, ParameterBase) self.parameters[idx] = param - link_operation_ids = kwargs.get('link_operation_ids', None) - if self.get is not None and (not link_operation_ids or self.get.operation_id in link_operation_ids): + if self.get is not None: self.get.link(swagger_loader, *self.traces, 'get') - if self.put is not None and (not link_operation_ids or self.put.operation_id in link_operation_ids): + if self.put is not None: self.put.link(swagger_loader, *self.traces, 'put') - if self.post is not None and (not link_operation_ids or self.post.operation_id in link_operation_ids): + if self.post is not None: self.post.link(swagger_loader, *self.traces, 'post') - if self.delete is not None and (not link_operation_ids or self.delete.operation_id in link_operation_ids): + if self.delete is not None: self.delete.link(swagger_loader, *self.traces, 'delete') - if self.head is not None and (not link_operation_ids or self.head.operation_id in link_operation_ids): + if self.head is not None: self.head.link(swagger_loader, *self.traces, 'head') - if self.patch is not None and (not link_operation_ids or self.patch.operation_id in link_operation_ids): + if self.patch is not None: self.patch.link(swagger_loader, *self.traces, 'patch') - def link_examples(self, swagger_loader, operation_ids, *traces): - super().link(swagger_loader, *traces) - - if self.get is not None and self.get.operation_id in operation_ids: - self.get.link_examples(swagger_loader, *self.traces, "get") - if self.put is not None and self.put.operation_id in operation_ids: - self.put.link_examples(swagger_loader, *self.traces, "put") - if self.post is not None and self.post.operation_id in operation_ids: - self.post.link_examples(swagger_loader, *self.traces, "post") - if self.delete is not None and self.delete.operation_id in operation_ids: - self.delete.link_examples(swagger_loader, *self.traces, "delete") - if self.head is not None and self.head.operation_id in operation_ids: - self.head.link_examples(swagger_loader, *self.traces, "head") - if self.patch is not None and self.patch.operation_id in operation_ids: - self.patch.link_examples(swagger_loader, *self.traces, "patch") - def to_cmd(self, builder, **kwargs): op = getattr(self, builder.method, None) if op is None: diff --git a/src/aaz_dev/swagger/model/schema/swagger.py b/src/aaz_dev/swagger/model/schema/swagger.py index 214c96e8..c57b5521 100644 --- a/src/aaz_dev/swagger/model/schema/swagger.py +++ b/src/aaz_dev/swagger/model/schema/swagger.py @@ -62,11 +62,9 @@ def link(self, swagger_loader, *traces, **kwargs): return super().link(swagger_loader, *traces) - link_path = kwargs.get('link_path', None) if self.paths is not None: for key, path in self.paths.items(): - if not link_path or link_path == key: - path.link(swagger_loader, *self.traces, 'paths', key, **kwargs) + path.link(swagger_loader, *self.traces, 'paths', key) if self.definitions is not None: for key, definition in self.definitions.items(): @@ -86,12 +84,4 @@ def link(self, swagger_loader, *traces, **kwargs): path.link(swagger_loader, *self.traces, 'x_ms_paths', key) if self.x_ms_parameterized_host is not None: - self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') - - def link_examples(self, swagger_loader, path, operation_ids, *traces): - self.link(swagger_loader, *traces, link_path=path, link_operation_ids=operation_ids) - - if self.paths is not None: - for key, path_item in self.paths.items(): - if key == path: - path_item.link_examples(swagger_loader, operation_ids, *self.traces, "paths", key) + self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') \ No newline at end of file diff --git a/src/aaz_dev/swagger/model/specs/_swagger_loader.py b/src/aaz_dev/swagger/model/specs/_swagger_loader.py index 3f876e45..1213f26c 100644 --- a/src/aaz_dev/swagger/model/specs/_swagger_loader.py +++ b/src/aaz_dev/swagger/model/specs/_swagger_loader.py @@ -62,12 +62,6 @@ def link_swaggers(self): swagger.link(self, file_path) self._linked_idx += 1 - def link_examples(self, file_path, path, operation_ids): - swagger = self.loaded_swaggers[file_path] - assert swagger - - swagger.link_examples(self, path, operation_ids, file_path) - def get_loaded(self, *traces): return self._loaded.get(traces, None) From 30738d796b50f450d751cadc0a0bbd2e218da55b Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 18 Dec 2023 19:46:58 +0800 Subject: [PATCH 18/37] chore: add source to request body --- src/aaz_dev/command/api/editor.py | 14 +++++++------ .../command/tests/api_tests/test_editor.py | 21 ++++++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index 942922c2..b52ef960 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -268,17 +268,19 @@ def editor_workspace_generate_examples(name, node_names, leaf_name): if not leaf: raise exceptions.ResourceNotFind("Command not exist.") + data = request.get_json() + source = data.get("source", None) + cfg_editor = manager.load_cfg_editor_by_command(leaf) command = cfg_editor.find_command(*leaf.names) - - if command: - examples = manager.generate_examples_by_swagger(leaf, command) - else: + if not command: raise exceptions.ResourceNotFind("Command not exist.") - result = [example.to_primitive() for example in examples] + if source == "swagger": + examples = manager.generate_examples_by_swagger(leaf, command) + result = [example.to_primitive() for example in examples] - return jsonify(result) + return jsonify(result) @bp.route("/Workspaces//CommandTree/Nodes//Leaves//Examples", diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index eaba0f5f..56736d16 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -2628,7 +2628,12 @@ def test_workspace_generate_examples_array(self, ws_name): ) self.assertTrue(rv.status_code == 200) - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/GenerateExamples") + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/event-hub/namespace/network-rule-set/default/Leaves/create/GenerateExamples", + json={ + "source": "swagger" + } + ) self.assertTrue(rv.status_code == 200) @workspace_name("test_workspace_generate_examples_dict") @@ -2665,7 +2670,12 @@ def test_workspace_generate_examples_dict(self, ws_name): ) self.assertTrue(rv.status_code == 200) - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/event-hub/cluster/Leaves/create/GenerateExamples") + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/event-hub/cluster/Leaves/create/GenerateExamples", + json={ + "source": "swagger" + } + ) self.assertTrue(rv.status_code == 200) @workspace_name("test_workspace_generate_examples_poly") @@ -2706,5 +2716,10 @@ def test_workspace_generate_examples_poly(self, ws_name): ) self.assertTrue(rv.status_code == 200) - rv = c.post(f"{ws_url}/CommandTree/Nodes/aaz/insights/metric-alert/Leaves/create/GenerateExamples") + rv = c.post( + f"{ws_url}/CommandTree/Nodes/aaz/insights/metric-alert/Leaves/create/GenerateExamples", + json={ + "source": "swagger" + } + ) self.assertTrue(rv.status_code == 200) From 856496a4ecfce0d13541e876a307c655c2b336bf Mon Sep 17 00:00:00 2001 From: necusjz Date: Mon, 18 Dec 2023 19:49:50 +0800 Subject: [PATCH 19/37] style: add blank line --- src/aaz_dev/swagger/model/schema/operation.py | 2 +- src/aaz_dev/swagger/model/schema/swagger.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aaz_dev/swagger/model/schema/operation.py b/src/aaz_dev/swagger/model/schema/operation.py index 76d2e4fc..4a69793a 100644 --- a/src/aaz_dev/swagger/model/schema/operation.py +++ b/src/aaz_dev/swagger/model/schema/operation.py @@ -140,7 +140,7 @@ def link(self, swagger_loader, *traces): try: example.link(swagger_loader, *self.traces, "x_ms_examples", key) except e: - logging.warning(f'Link example failed: {key}') + logging.warning(f"Link example failed at {key}.") def to_cmd(self, builder, parent_parameters, host_path, **kwargs): cmd_op = CMDHttpOperation() diff --git a/src/aaz_dev/swagger/model/schema/swagger.py b/src/aaz_dev/swagger/model/schema/swagger.py index c57b5521..d0e60415 100644 --- a/src/aaz_dev/swagger/model/schema/swagger.py +++ b/src/aaz_dev/swagger/model/schema/swagger.py @@ -57,7 +57,7 @@ class Swagger(Model, Linkable): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def link(self, swagger_loader, *traces, **kwargs): + def link(self, swagger_loader, *traces): if self.is_linked(): return super().link(swagger_loader, *traces) @@ -84,4 +84,4 @@ def link(self, swagger_loader, *traces, **kwargs): path.link(swagger_loader, *self.traces, 'x_ms_paths', key) if self.x_ms_parameterized_host is not None: - self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') \ No newline at end of file + self.x_ms_parameterized_host.link(swagger_loader, *self.traces, 'x_ms_parameterized_host') From 43caef3389868fd962a3a1406f79e667b2d0b4fa Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 19 Dec 2023 11:37:11 +0800 Subject: [PATCH 20/37] chore: handle exception --- src/aaz_dev/command/api/editor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aaz_dev/command/api/editor.py b/src/aaz_dev/command/api/editor.py index b52ef960..96177b45 100644 --- a/src/aaz_dev/command/api/editor.py +++ b/src/aaz_dev/command/api/editor.py @@ -279,8 +279,10 @@ def editor_workspace_generate_examples(name, node_names, leaf_name): if source == "swagger": examples = manager.generate_examples_by_swagger(leaf, command) result = [example.to_primitive() for example in examples] + else: + raise exceptions.InvalidAPIUsage("Invalid request.") - return jsonify(result) + return jsonify(result) @bp.route("/Workspaces//CommandTree/Nodes//Leaves//Examples", From ef06654dc141bfa451ebef71f18622a37a3f9a4b Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 28 Dec 2023 14:26:31 +0800 Subject: [PATCH 21/37] fix: handle exception --- src/aaz_dev/swagger/model/schema/operation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aaz_dev/swagger/model/schema/operation.py b/src/aaz_dev/swagger/model/schema/operation.py index 4a69793a..4ca948b6 100644 --- a/src/aaz_dev/swagger/model/schema/operation.py +++ b/src/aaz_dev/swagger/model/schema/operation.py @@ -21,6 +21,8 @@ from .x_ms_odata import XmsODataField from .x_ms_pageable import XmsPageableField +logger = logging.getLogger('backend') + class Operation(Model, Linkable): """Describes a single API operation on a path.""" @@ -139,8 +141,8 @@ def link(self, swagger_loader, *traces): for key, example in self.x_ms_examples.items(): try: example.link(swagger_loader, *self.traces, "x_ms_examples", key) - except e: - logging.warning(f"Link example failed at {key}.") + except Exception as e: + logger.error(f"Link example failed: {e}: {key}.") def to_cmd(self, builder, parent_parameters, host_path, **kwargs): cmd_op = CMDHttpOperation() From 1e035ddc48fcfcd3fbec213d642ee520579bfb91 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 28 Dec 2023 14:27:26 +0800 Subject: [PATCH 22/37] feat: support discriminators --- .../swagger/controller/_example_builder.py | 87 +++++++++++++++++-- .../swagger/controller/example_generator.py | 30 +++++-- 2 files changed, 104 insertions(+), 13 deletions(-) diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index d12721d5..a6c2850e 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -1,19 +1,21 @@ import json +import re from abc import abstractmethod from command.controller.cfg_reader import CfgReader from command.model.configuration import CMDArgGroup -from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter from command.model.configuration._utils import CMDArgBuildPrefix +from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter class ExampleItem: - def __init__(self, command=None, arg_var=None, key=None, val=None): + def __init__(self, command=None, cmd_operation=None, arg_var=None, key=None, val=None): self.arg_var = arg_var self.key = key self.val = val self.arg_parent, self.arg, self.arg_option = CfgReader.find_arg_in_command_with_parent_by_var(command, arg_var) + self.schemas = CfgReader.iter_schema_in_operation_by_arg_var(cmd_operation, arg_var) if self.arg_option is not None: self.arg_option = self.arg_option.split(".")[-1] @@ -26,6 +28,14 @@ def is_flatten(self): def is_top_level(self): return isinstance(self.arg_parent, CMDArgGroup) and self.arg + @property + def discriminators(self): + for _, schema, _ in self.schemas: + if hasattr(schema, "discriminators") and schema.discriminators: + return schema.discriminators + + return [] + class ExampleBuilder: def __init__(self, command=None): @@ -38,9 +48,10 @@ def mapping(self, example_dict): class SwaggerExampleBuilder(ExampleBuilder): - def __init__(self, command=None, operation=None): + def __init__(self, command=None, operation=None, cmd_operation=None): super().__init__(command=command) self.operation = operation + self.cmd_operation = cmd_operation def mapping(self, example_dict): for param in self.operation.parameters: @@ -62,7 +73,13 @@ def mapping(self, example_dict): if param.IN_VALUE == HeaderParameter.IN_VALUE: arg_var = f"{CMDArgBuildPrefix.Header}.{param_name}" - item = ExampleItem(command=self.command, arg_var=arg_var, key=param_name, val=value) + item = ExampleItem( + command=self.command, + cmd_operation=self.cmd_operation, + arg_var=arg_var, + key=param_name, + val=value + ) if item.is_top_level: self.example_items.append((item.arg_option, json.dumps(value))) @@ -76,11 +93,56 @@ def build(self, var_prefix, example_dict): example_items += self.build(arg_var, item) elif isinstance(example_dict, dict): for name, value in example_dict.copy().items(): - item = ExampleItem(command=self.command, arg_var=f"{var_prefix}{{}}.{name}", key=name, val=value) + item = ExampleItem( + command=self.command, + cmd_operation=self.cmd_operation, + arg_var=f"{var_prefix}{{}}.{name}", + key=name, + val=value + ) if item.arg is None: - item = ExampleItem(command=self.command, arg_var=f"{var_prefix}.{name}", key=name, val=value) - - example_items += self.build(item.arg_var, value) + item = ExampleItem( + command=self.command, + cmd_operation=self.cmd_operation, + arg_var=f"{var_prefix}.{name}", + key=name, + val=value + ) + + for disc in item.discriminators: + if disc.property not in value or value[disc.property] != disc.value or "allOf" not in value: + continue + + formatted = dict() + save_value = self.get_safe_value(disc.value) + disc_item = ExampleItem( + command=self.command, + arg_var=f"{item.arg_var}.{save_value}" + ) + + if disc_name := disc_item.arg_option: + formatted[disc_name] = dict() + disc_item = ExampleItem( + command=self.command, + cmd_operation=self.cmd_operation, + arg_var=f"{item.arg_var}.{save_value}.allOf", + key=name, + val=value["allOf"] + ) + + if disc_item.arg_option: + formatted[disc_name][disc_item.arg_option] = value["allOf"] + + value = formatted + example_dict[item.key] = formatted + item.val = formatted + + example_items += self.build(disc_item.arg_var, disc_item.val) + + break + + else: + example_items += self.build(item.arg_var, value) if item.is_top_level: example_items.append((item.arg_option, json.dumps(value))) @@ -93,3 +155,12 @@ def build(self, var_prefix, example_dict): example_dict[item.arg_option] = item.val return example_items + + @staticmethod + def get_safe_value(value): + """Some value may contain special characters such as Microsoft.db/mysql, it will cause issues. + This function will replace them by `_` + """ + safe_value = re.sub(r'[^A-Za-z0-9_-]', '_', value) + + return safe_value diff --git a/src/aaz_dev/swagger/controller/example_generator.py b/src/aaz_dev/swagger/controller/example_generator.py index fb648ed2..0109fb03 100644 --- a/src/aaz_dev/swagger/controller/example_generator.py +++ b/src/aaz_dev/swagger/controller/example_generator.py @@ -30,23 +30,43 @@ def create_draft_examples_by_swagger(self, resources, command, cmd_operation_ids example_builder = None examples = None if path_item.get is not None and path_item.get.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(command=command, operation=path_item.get) + example_builder = SwaggerExampleBuilder( + command=command, + operation=path_item.get, + cmd_operation=cmd_operation_ids[path_item.get.operation_id] + ) examples = path_item.get.x_ms_examples elif path_item.delete is not None and path_item.delete.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(command=command, operation=path_item.delete) + example_builder = SwaggerExampleBuilder( + command=command, + operation=path_item.delete, + cmd_operation=cmd_operation_ids[path_item.delete.operation_id] + ) examples = path_item.delete.x_ms_examples elif path_item.put is not None and path_item.put.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(command=command, operation=path_item.put) + example_builder = SwaggerExampleBuilder( + command=command, + operation=path_item.put, + cmd_operation=cmd_operation_ids[path_item.put.operation_id] + ) examples = path_item.put.x_ms_examples elif path_item.post is not None and path_item.post.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(command=command, operation=path_item.post) + example_builder = SwaggerExampleBuilder( + command=command, + operation=path_item.post, + cmd_operation=cmd_operation_ids[path_item.post.operation_id] + ) examples = path_item.post.x_ms_examples elif path_item.head is not None and path_item.head.operation_id in cmd_operation_ids: - example_builder = SwaggerExampleBuilder(command=command, operation=path_item.head) + example_builder = SwaggerExampleBuilder( + command=command, + operation=path_item.head, + cmd_operation=cmd_operation_ids[path_item.head.operation_id] + ) examples = path_item.head.x_ms_examples if not example_builder or not examples: From 7a867d04e444353537640cc12336cf9acf3a872b Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 28 Dec 2023 21:23:15 +0800 Subject: [PATCH 23/37] fix: `allOf` is not static --- .../swagger/controller/_example_builder.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index a6c2850e..6edc2b0d 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -110,35 +110,28 @@ def build(self, var_prefix, example_dict): ) for disc in item.discriminators: - if disc.property not in value or value[disc.property] != disc.value or "allOf" not in value: + if disc.property not in value or value[disc.property] != disc.value: continue - formatted = dict() - save_value = self.get_safe_value(disc.value) + value.pop(disc.property) # ignore discriminator prop + + safe_value = self.get_safe_value(disc.value) disc_item = ExampleItem( command=self.command, - arg_var=f"{item.arg_var}.{save_value}" + arg_var=f"{item.arg_var}.{safe_value}" ) + formatted = dict() if disc_name := disc_item.arg_option: - formatted[disc_name] = dict() - disc_item = ExampleItem( - command=self.command, - cmd_operation=self.cmd_operation, - arg_var=f"{item.arg_var}.{save_value}.allOf", - key=name, - val=value["allOf"] - ) - - if disc_item.arg_option: - formatted[disc_name][disc_item.arg_option] = value["allOf"] - - value = formatted + formatted[disc_name] = value + else: + formatted[safe_value] = value + + original, value = value, formatted example_dict[item.key] = formatted item.val = formatted - example_items += self.build(disc_item.arg_var, disc_item.val) - + example_items += self.build(disc_item.arg_var, original) break else: From 0a03e5399e3958fe2a161089582dcb08b700e253 Mon Sep 17 00:00:00 2001 From: necusjz Date: Fri, 5 Jan 2024 11:04:59 +0800 Subject: [PATCH 24/37] refactor: handle partial poly --- .../swagger/controller/_example_builder.py | 139 ++++++++++++------ 1 file changed, 90 insertions(+), 49 deletions(-) diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index 6edc2b0d..5f7b32fe 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -9,17 +9,41 @@ class ExampleItem: - def __init__(self, command=None, cmd_operation=None, arg_var=None, key=None, val=None): + def __init__( + self, + cmd_operation=None, + arg_var=None, + key=None, + val=None, + arg_parent=None, + arg=None, + arg_option=None + ): + self.cmd_operation = cmd_operation self.arg_var = arg_var self.key = key self.val = val - self.arg_parent, self.arg, self.arg_option = CfgReader.find_arg_in_command_with_parent_by_var(command, arg_var) - self.schemas = CfgReader.iter_schema_in_operation_by_arg_var(cmd_operation, arg_var) + self.arg_parent, self.arg, self.arg_option = arg_parent, arg, arg_option if self.arg_option is not None: self.arg_option = self.arg_option.split(".")[-1] + @classmethod + def new_instance(cls, command=None, cmd_operation=None, arg_var=None, key=None, val=None): + arg_parent, arg, arg_option = CfgReader.find_arg_in_command_with_parent_by_var(command, arg_var) + + if arg_parent or arg or arg_option: + return cls( + cmd_operation=cmd_operation, + arg_var=arg_var, + key=key, + val=val, + arg_parent=arg_parent, + arg=arg, + arg_option=arg_option + ) + @property def is_flatten(self): return self.arg_parent and not self.arg @@ -30,7 +54,7 @@ def is_top_level(self): @property def discriminators(self): - for _, schema, _ in self.schemas: + for _, schema, _ in CfgReader.iter_schema_in_operation_by_arg_var(self.cmd_operation, self.arg_var): if hasattr(schema, "discriminators") and schema.discriminators: return schema.discriminators @@ -73,35 +97,70 @@ def mapping(self, example_dict): if param.IN_VALUE == HeaderParameter.IN_VALUE: arg_var = f"{CMDArgBuildPrefix.Header}.{param_name}" - item = ExampleItem( + item = ExampleItem.new_instance( command=self.command, cmd_operation=self.cmd_operation, arg_var=arg_var, key=param_name, val=value ) - if item.is_top_level: + if item and item.is_top_level: self.example_items.append((item.arg_option, json.dumps(value))) return self.example_items - def build(self, var_prefix, example_dict): + def build(self, var_prefix, example_obj, disc=None): example_items = [] - if isinstance(example_dict, list): + if isinstance(example_obj, list): arg_var = f"{var_prefix}[]" - for item in example_dict: - example_items += self.build(arg_var, item) - elif isinstance(example_dict, dict): - for name, value in example_dict.copy().items(): - item = ExampleItem( + item = ExampleItem.new_instance( + command=self.command, + cmd_operation=self.cmd_operation, + arg_var=arg_var + ) + if item: + discs = item.discriminators + for obj in example_obj: + for disc in discs: + if disc.property not in obj or obj[disc.property] != disc.value: + continue + + example_items += self.build(arg_var, obj, disc) + break + else: + example_items += self.build(arg_var, obj) + + elif isinstance(example_obj, dict): + disc_name = None + if disc is not None: # handle discriminator + example_obj.pop(disc.property) # ignore discriminator prop + + safe_value = self.get_safe_value(disc.value) + disc_item = ExampleItem.new_instance( + command=self.command, + cmd_operation=self.cmd_operation, + arg_var=f"{var_prefix}.{safe_value}" + ) + + if disc_item and (disc_name := disc_item.arg_option): + example_obj[disc_name] = example_obj.copy() + example_items += self.build(disc_item.arg_var, example_obj[disc_name]) + + for name, value in example_obj.copy().items(): + if name == disc_name: + continue + + example_obj.pop(name) # will push back if arg_var valid + + item = ExampleItem.new_instance( command=self.command, cmd_operation=self.cmd_operation, arg_var=f"{var_prefix}{{}}.{name}", key=name, val=value ) - if item.arg is None: - item = ExampleItem( + if not item: + item = ExampleItem.new_instance( command=self.command, cmd_operation=self.cmd_operation, arg_var=f"{var_prefix}.{name}", @@ -109,43 +168,25 @@ def build(self, var_prefix, example_dict): val=value ) - for disc in item.discriminators: - if disc.property not in value or value[disc.property] != disc.value: - continue + if item: + for disc in item.discriminators: + if disc.property not in value or value[disc.property] != disc.value: + continue - value.pop(disc.property) # ignore discriminator prop + example_items += self.build(item.arg_var, value, disc) + break + else: + example_items += self.build(item.arg_var, value) - safe_value = self.get_safe_value(disc.value) - disc_item = ExampleItem( - command=self.command, - arg_var=f"{item.arg_var}.{safe_value}" - ) + if item.is_top_level: + example_items.append((item.arg_option, json.dumps(value))) - formatted = dict() - if disc_name := disc_item.arg_option: - formatted[disc_name] = value - else: - formatted[safe_value] = value - - original, value = value, formatted - example_dict[item.key] = formatted - item.val = formatted - - example_items += self.build(disc_item.arg_var, original) - break - - else: - example_items += self.build(item.arg_var, value) - - if item.is_top_level: - example_items.append((item.arg_option, json.dumps(value))) - elif item.is_flatten: - example_dict.pop(item.key) - for k, v in item.val.items(): - example_dict[k] = v - elif item.arg_option: - example_dict.pop(item.key) - example_dict[item.arg_option] = item.val + elif item.is_flatten: + for k, v in item.val.items(): + example_obj[k] = v + + elif item.arg_option: + example_obj[item.arg_option] = item.val return example_items From 4fdba99c4a8e7837c802448cee439c4a7de4f2c7 Mon Sep 17 00:00:00 2001 From: necusjz Date: Fri, 5 Jan 2024 11:05:57 +0800 Subject: [PATCH 25/37] test: add assertion for poly --- src/aaz_dev/command/tests/api_tests/test_editor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index 56736d16..f692e67a 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -2723,3 +2723,8 @@ def test_workspace_generate_examples_poly(self, ws_name): } ) self.assertTrue(rv.status_code == 200) + + self.assertTrue( + "microsoft-azure-monitor-multiple-resource-multiple-metric-criteria" in rv.json[0]["commands"][0] + ) + self.assertTrue("static-threshold-criterion" in rv.json[0]["commands"][0]) From 41a54bdd8bceefe3e58c58163b25e53a63d139d1 Mon Sep 17 00:00:00 2001 From: necusjz Date: Fri, 5 Jan 2024 16:46:44 +0800 Subject: [PATCH 26/37] fix: use deepcopy instead --- src/aaz_dev/swagger/controller/_example_builder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index 5f7b32fe..4d8bdb71 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -1,3 +1,4 @@ +import copy import json import re from abc import abstractmethod @@ -143,7 +144,7 @@ def build(self, var_prefix, example_obj, disc=None): ) if disc_item and (disc_name := disc_item.arg_option): - example_obj[disc_name] = example_obj.copy() + example_obj[disc_name] = copy.deepcopy(example_obj) # further trim (polymorphic or not) example_items += self.build(disc_item.arg_var, example_obj[disc_name]) for name, value in example_obj.copy().items(): From 3e06b79cda1b5e5e89d4d20acd227a447a17d0e6 Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 10 Jan 2024 15:07:12 +0800 Subject: [PATCH 27/37] test: add case for shorthand syntax --- .../test_serialize_shorthand.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py diff --git a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py b/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py new file mode 100644 index 00000000..ce8cce81 --- /dev/null +++ b/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py @@ -0,0 +1,34 @@ +from swagger.controller._shorthand import serialize + + +def test_nested(): + obj = { + "microsoft-azure-monitor-single-resource-multiple-metric-criteria": { + "all-of": [{ + "name": "High_CPU_80", + "metric-name": "Time", + "dimensions": [], + "operator": "GreaterThan", + "threshold": 80.5, + "time-aggregation": "Average" + }] + } + } + + assert serialize(obj) == '"{microsoft-azure-monitor-single-resource-multiple-metric-criteria:{all-of:[{name:High_CPU_80,metric-name:Time,dimensions:[],operator:GreaterThan,threshold:80.5,time-aggregation:Average}]}}"' + + +def test_with_single_quote(): + obj = { + "name": "monitor's" + } + + assert serialize(obj) == '"{name:monitor\'/s}"' + + +def test_with_special_characters(): + obj = { + "name": "monitor metric" + } + + assert serialize(obj) == '"{name:\'monitor metric\'}"' From 2d69e4d00c1ddfdcb797f857fc92f379ef7df60c Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 10 Jan 2024 15:07:42 +0800 Subject: [PATCH 28/37] feat: support shorthand syntax --- .../swagger/controller/_example_builder.py | 6 +++--- src/aaz_dev/swagger/controller/_shorthand.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 src/aaz_dev/swagger/controller/_shorthand.py diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index 4d8bdb71..1b0753a9 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -1,11 +1,11 @@ import copy -import json import re from abc import abstractmethod from command.controller.cfg_reader import CfgReader from command.model.configuration import CMDArgGroup from command.model.configuration._utils import CMDArgBuildPrefix +from swagger.controller._shorthand import serialize from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter @@ -106,7 +106,7 @@ def mapping(self, example_dict): val=value ) if item and item.is_top_level: - self.example_items.append((item.arg_option, json.dumps(value))) + self.example_items.append((item.arg_option, serialize(value))) return self.example_items @@ -180,7 +180,7 @@ def build(self, var_prefix, example_obj, disc=None): example_items += self.build(item.arg_var, value) if item.is_top_level: - example_items.append((item.arg_option, json.dumps(value))) + example_items.append((item.arg_option, serialize(value))) elif item.is_flatten: for k, v in item.val.items(): diff --git a/src/aaz_dev/swagger/controller/_shorthand.py b/src/aaz_dev/swagger/controller/_shorthand.py new file mode 100644 index 00000000..adfbb64a --- /dev/null +++ b/src/aaz_dev/swagger/controller/_shorthand.py @@ -0,0 +1,19 @@ +def serialize(obj): + def dfs(obj): + if isinstance(obj, dict): + return "{" + ",".join(f'{k}:{dfs(v)}' for k, v in obj.items()) + "}" + + elif isinstance(obj, list): + return "[" + ",".join(dfs(i) for i in obj) + "]" + + else: + escaped = str(obj) + escaped = escaped.replace("'", "'/") # use '/ to input ' + + # with space, null/help expressions or other special characters + if any(char in escaped for char in (" ", "null", "??", ":", ",", "{", "}", "[", "]")): + escaped = "'" + escaped + "'" + + return repr(escaped)[1:-1] # ignore quotes + + return '"' + dfs(obj) + '"' if isinstance(obj, dict) or isinstance(obj, list) else obj From 876cb7c5a66e28a108af6eb71ce6323d7144ed9c Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 10 Jan 2024 15:15:09 +0800 Subject: [PATCH 29/37] style: clean code --- .../swagger/tests/controller_tests/test_serialize_shorthand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py b/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py index ce8cce81..1dd9c565 100644 --- a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py +++ b/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py @@ -1,7 +1,7 @@ from swagger.controller._shorthand import serialize -def test_nested(): +def test_with_nesting(): obj = { "microsoft-azure-monitor-single-resource-multiple-metric-criteria": { "all-of": [{ From 31c65288fc42151afceaafdb161dce35474b6cee Mon Sep 17 00:00:00 2001 From: necusjz Date: Wed, 10 Jan 2024 15:24:20 +0800 Subject: [PATCH 30/37] test: add more checks --- .../tests/controller_tests/test_serialize_shorthand.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py b/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py index 1dd9c565..7dd3107b 100644 --- a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py +++ b/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py @@ -28,7 +28,9 @@ def test_with_single_quote(): def test_with_special_characters(): obj = { - "name": "monitor metric" + "name": "monitor metric", + "dimensions": "null", + "data": "{a: [1, 2]}" } - assert serialize(obj) == '"{name:\'monitor metric\'}"' + assert serialize(obj) == '"{name:\'monitor metric\',dimensions:\'null\',data:\'{a: [1, 2]}\'}"' From db32b67489c257fc24066a2a02532ab1723feeac Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 11 Jan 2024 10:48:39 +0800 Subject: [PATCH 31/37] refactor: move from swagger to command --- .../_shorthand.py => command/controller/shorthand.py} | 3 +++ .../tests/editor_tests}/test_serialize_shorthand.py | 6 +++--- src/aaz_dev/swagger/controller/_example_builder.py | 5 ++--- src/aaz_dev/swagger/model/schema/example_item.py | 5 +++-- 4 files changed, 11 insertions(+), 8 deletions(-) rename src/aaz_dev/{swagger/controller/_shorthand.py => command/controller/shorthand.py} (88%) rename src/aaz_dev/{swagger/tests/controller_tests => command/tests/editor_tests}/test_serialize_shorthand.py (88%) diff --git a/src/aaz_dev/swagger/controller/_shorthand.py b/src/aaz_dev/command/controller/shorthand.py similarity index 88% rename from src/aaz_dev/swagger/controller/_shorthand.py rename to src/aaz_dev/command/controller/shorthand.py index adfbb64a..e3c5745b 100644 --- a/src/aaz_dev/swagger/controller/_shorthand.py +++ b/src/aaz_dev/command/controller/shorthand.py @@ -7,6 +7,9 @@ def dfs(obj): return "[" + ",".join(dfs(i) for i in obj) + "]" else: + if obj is None: + return repr("null")[1:-1] # replace None by null + escaped = str(obj) escaped = escaped.replace("'", "'/") # use '/ to input ' diff --git a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py b/src/aaz_dev/command/tests/editor_tests/test_serialize_shorthand.py similarity index 88% rename from src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py rename to src/aaz_dev/command/tests/editor_tests/test_serialize_shorthand.py index 7dd3107b..4be8bc14 100644 --- a/src/aaz_dev/swagger/tests/controller_tests/test_serialize_shorthand.py +++ b/src/aaz_dev/command/tests/editor_tests/test_serialize_shorthand.py @@ -1,4 +1,4 @@ -from swagger.controller._shorthand import serialize +from command.controller.shorthand import serialize def test_with_nesting(): @@ -29,8 +29,8 @@ def test_with_single_quote(): def test_with_special_characters(): obj = { "name": "monitor metric", - "dimensions": "null", + "dimensions": None, "data": "{a: [1, 2]}" } - assert serialize(obj) == '"{name:\'monitor metric\',dimensions:\'null\',data:\'{a: [1, 2]}\'}"' + assert serialize(obj) == '"{name:\'monitor metric\',dimensions:null,data:\'{a: [1, 2]}\'}"' diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index 1b0753a9..1ea37481 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -5,7 +5,6 @@ from command.controller.cfg_reader import CfgReader from command.model.configuration import CMDArgGroup from command.model.configuration._utils import CMDArgBuildPrefix -from swagger.controller._shorthand import serialize from swagger.model.schema.parameter import PathParameter, QueryParameter, HeaderParameter, BodyParameter @@ -106,7 +105,7 @@ def mapping(self, example_dict): val=value ) if item and item.is_top_level: - self.example_items.append((item.arg_option, serialize(value))) + self.example_items.append((item.arg_option, value)) return self.example_items @@ -180,7 +179,7 @@ def build(self, var_prefix, example_obj, disc=None): example_items += self.build(item.arg_var, value) if item.is_top_level: - example_items.append((item.arg_option, serialize(value))) + example_items.append((item.arg_option, value)) elif item.is_flatten: for k, v in item.val.items(): diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index a30b5c45..3f825b93 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -2,6 +2,7 @@ from schematics.types import DictType, ModelType from command.model.configuration import CMDCommandExample +from command.controller.shorthand import serialize from .reference import Linkable, ReferenceField @@ -29,9 +30,9 @@ def to_cmd(self, example_builder, cmd_name, **kwargs): command = cmd_name for param_option, param_value in example_params: if len(param_option) == 1: - command += f" -{param_option} {param_value}" + command += f" -{param_option} {serialize(param_value)}" else: - command += f" --{param_option} {param_value}" + command += f" --{param_option} {serialize(param_value)}" return CMDCommandExample({"commands": [command.strip()]}) From 7a0bef26ce215b007e5e665e394341d36485c568 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 11 Jan 2024 10:51:12 +0800 Subject: [PATCH 32/37] test: fix invalid case --- src/aaz_dev/command/tests/api_tests/test_editor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aaz_dev/command/tests/api_tests/test_editor.py b/src/aaz_dev/command/tests/api_tests/test_editor.py index f692e67a..f7f57921 100644 --- a/src/aaz_dev/command/tests/api_tests/test_editor.py +++ b/src/aaz_dev/command/tests/api_tests/test_editor.py @@ -2725,6 +2725,6 @@ def test_workspace_generate_examples_poly(self, ws_name): self.assertTrue(rv.status_code == 200) self.assertTrue( - "microsoft-azure-monitor-multiple-resource-multiple-metric-criteria" in rv.json[0]["commands"][0] + "microsoft-azure-monitor-multiple-resource-multiple-metric-criteria" in rv.json[1]["commands"][0] ) - self.assertTrue("static-threshold-criterion" in rv.json[0]["commands"][0]) + self.assertTrue("static-threshold-criterion" in rv.json[1]["commands"][0]) From 27795668993b477798ed24fefe8ef79002f1b98a Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 11 Jan 2024 11:01:18 +0800 Subject: [PATCH 33/37] style: clean code --- src/aaz_dev/swagger/model/schema/example_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index 3f825b93..2f3b2f2e 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -1,8 +1,8 @@ from schematics.models import Model from schematics.types import DictType, ModelType -from command.model.configuration import CMDCommandExample from command.controller.shorthand import serialize +from command.model.configuration import CMDCommandExample from .reference import Linkable, ReferenceField From e9c2c07a4e80d180916b2036dc3c10ca1848d217 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 11 Jan 2024 14:05:32 +0800 Subject: [PATCH 34/37] Update src/aaz_dev/command/controller/shorthand.py Co-authored-by: kai ru <69238381+kairu-ms@users.noreply.github.com> --- src/aaz_dev/command/controller/shorthand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aaz_dev/command/controller/shorthand.py b/src/aaz_dev/command/controller/shorthand.py index e3c5745b..dec156db 100644 --- a/src/aaz_dev/command/controller/shorthand.py +++ b/src/aaz_dev/command/controller/shorthand.py @@ -8,7 +8,7 @@ def dfs(obj): else: if obj is None: - return repr("null")[1:-1] # replace None by null + return "null" # replace None by null escaped = str(obj) escaped = escaped.replace("'", "'/") # use '/ to input ' From 15de891b43f4bdb4aae46dc9f13ab9c285389913 Mon Sep 17 00:00:00 2001 From: necusjz Date: Tue, 20 Feb 2024 14:48:05 +0800 Subject: [PATCH 35/37] support ui --- .../workspace/WSEditorCommandContent.tsx | 183 ++++++++++++++---- .../views/workspace/WSEditorExamplePicker.tsx | 61 ++++++ 2 files changed, 203 insertions(+), 41 deletions(-) create mode 100644 src/web/src/views/workspace/WSEditorExamplePicker.tsx diff --git a/src/web/src/views/workspace/WSEditorCommandContent.tsx b/src/web/src/views/workspace/WSEditorCommandContent.tsx index ba410d79..bd36c3e9 100644 --- a/src/web/src/views/workspace/WSEditorCommandContent.tsx +++ b/src/web/src/views/workspace/WSEditorCommandContent.tsx @@ -10,6 +10,9 @@ import KeyboardDoubleArrowRightIcon from '@mui/icons-material/KeyboardDoubleArro import LabelIcon from '@mui/icons-material/Label'; import WSEditorCommandArgumentsContent, { ClsArgDefinitionMap, CMDArg, DecodeArgs } from './WSEditorCommandArgumentsContent'; import EditIcon from '@mui/icons-material/Edit'; +import CloseIcon from '@mui/icons-material/Close'; +import Stack from '@mui/material/Stack'; +import { ExampleItemSelector } from './WSEditorExamplePicker'; interface Plane { @@ -866,6 +869,8 @@ interface ExampleDialogState { isAdd: boolean invalidText?: string updating: boolean + source?: string + exampleOptions: Example[] } const ExampleCommandTypography = styled(Typography)(({ theme }) => ({ @@ -887,6 +892,8 @@ class ExampleDialog extends React.Component { + try { + let { workspaceUrl, command } = this.props; + + const leafUrl = `${workspaceUrl}/CommandTree/Nodes/aaz/` + command.names.slice(0, -1).join('/') + '/Leaves/' + command.names[command.names.length - 1] + '/GenerateExamples'; + + this.setState({ + source: "swagger", + updating: true, + }) + let res = await axios.post(leafUrl, { + source: "swagger" + }) + + const examples: Example[] = res.data.map((v: any) => { + return { + name: v.name, + commands: v.commands + } + }) + this.setState({ + exampleOptions: examples, + updating: false + }) + if (examples.length > 0) { + this.onExampleSelectorUpdate(examples[0].name); + } + } catch (err: any) { + console.error(err.response); + if (err.response?.data?.message) { + const data = err.response!.data!; + this.setState({ + updating: false, + invalidText: `ResponseError: ${data.message!}`, + }) + } + } + } + + onExampleSelectorUpdate = (exampleDisplayName: string | null) => { + let example = this.state.exampleOptions.find((v) => v.name === exampleDisplayName) ?? undefined; + + if (example === undefined) { + this.setState({ + name: exampleDisplayName ?? "", + }) + } else { + this.setState({ + name: example?.name ?? "", + exampleCommands: example?.commands ?? ["",], + }) + } + } + render() { - const { name, exampleCommands, isAdd, invalidText, updating } = this.state; + const { name, exampleCommands, isAdd, invalidText, updating, source, exampleOptions } = this.state; + + const selectedName = name const buildExampleInput = (cmd: string, idx: number) => { return ( @@ -1095,44 +1160,82 @@ class ExampleDialog extends React.Component - {isAdd ? "Add Example" : "Modify Example"} + + {isAdd ? "Add Example" : "Modify Example"} + + + + - {invalidText && {invalidText} } - { - this.setState({ - name: event.target.value, - }) - }} - margin="normal" - required - /> - Commands - {exampleCommands.map(buildExampleInput)} - - - - - One more command - + {isAdd && source === undefined && + + + + + + } + {(!isAdd || source != undefined) && + + {invalidText && {invalidText} } + {!isAdd && + + { + this.setState({ + name: event.target.value, + }) + }} + margin="normal" + required + /> + + } + {source === "swagger" && + + v.name)} + value={selectedName} + onValueUpdate={this.onExampleSelectorUpdate} + /> + + } + Commands + {exampleCommands.map(buildExampleInput)} + + + + + One more command + + + } + {(!isAdd || source != undefined) && {updating && @@ -1140,14 +1243,13 @@ class ExampleDialog extends React.Component } {!updating && - {!isAdd && } {isAdd && } } - + } ) } @@ -1358,5 +1460,4 @@ const DecodeResponseClientConfig = (clientConfig: any): ClientConfig => { export default WSEditorCommandContent; export { DecodeResponseCommand }; -export type { Plane, Command, Resource, ResponseCommand, ResponseCommands }; - +export type { Plane, Command, Resource, ResponseCommand, ResponseCommands, Example }; diff --git a/src/web/src/views/workspace/WSEditorExamplePicker.tsx b/src/web/src/views/workspace/WSEditorExamplePicker.tsx new file mode 100644 index 00000000..63a72f98 --- /dev/null +++ b/src/web/src/views/workspace/WSEditorExamplePicker.tsx @@ -0,0 +1,61 @@ +import React from 'react'; +import { Box, Autocomplete, TextField } from '@mui/material'; + +interface ExampleItemsSelectorProps { + commonPrefix: string, + options: string[], + name: string, + value: string | null, + onValueUpdate: (value: string | null) => void +} + + +class ExampleItemSelector extends React.Component { + + constructor(props: ExampleItemsSelectorProps) { + super(props); + this.state = { + value: this.props.options.length === 1 ? this.props.options[0] : null, + } + } + + render() { + const { name, options, commonPrefix, value } = this.props; + return ( + { + this.props.onValueUpdate(newValue); + }} + getOptionLabel={(option) => { + return option.replace(commonPrefix, ''); + }} + renderOption={(props, option) => { + return ( + + {option.replace(commonPrefix, '')} + + ) + }} + selectOnFocus + // clearOnBlur + freeSolo + renderInput={(params) => ( + + )} + /> + ) + } +} + +export { ExampleItemSelector }; From f8bed1cdb2f76c2c27b0d29571377dbebc9de423 Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 21 Mar 2024 13:40:03 +0800 Subject: [PATCH 36/37] chore: polish --tags and --subscription --- src/aaz_dev/swagger/controller/_example_builder.py | 4 ++-- src/aaz_dev/swagger/model/schema/example_item.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/aaz_dev/swagger/controller/_example_builder.py b/src/aaz_dev/swagger/controller/_example_builder.py index 1ea37481..354d176c 100644 --- a/src/aaz_dev/swagger/controller/_example_builder.py +++ b/src/aaz_dev/swagger/controller/_example_builder.py @@ -150,8 +150,6 @@ def build(self, var_prefix, example_obj, disc=None): if name == disc_name: continue - example_obj.pop(name) # will push back if arg_var valid - item = ExampleItem.new_instance( command=self.command, cmd_operation=self.cmd_operation, @@ -169,6 +167,8 @@ def build(self, var_prefix, example_obj, disc=None): ) if item: + example_obj.pop(name) # will push back if arg_var valid + for disc in item.discriminators: if disc.property not in value or value[disc.property] != disc.value: continue diff --git a/src/aaz_dev/swagger/model/schema/example_item.py b/src/aaz_dev/swagger/model/schema/example_item.py index 2f3b2f2e..191d940e 100644 --- a/src/aaz_dev/swagger/model/schema/example_item.py +++ b/src/aaz_dev/swagger/model/schema/example_item.py @@ -29,6 +29,9 @@ def to_cmd(self, example_builder, cmd_name, **kwargs): command = cmd_name for param_option, param_value in example_params: + if param_option == "subscription": + continue + if len(param_option) == 1: command += f" -{param_option} {serialize(param_value)}" else: From 306102b75dcf30c65790194c3aaed4efef27771b Mon Sep 17 00:00:00 2001 From: necusjz Date: Thu, 18 Apr 2024 16:47:28 +0800 Subject: [PATCH 37/37] chore: hide other sources --- src/web/src/views/workspace/WSEditorCommandContent.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/src/views/workspace/WSEditorCommandContent.tsx b/src/web/src/views/workspace/WSEditorCommandContent.tsx index bd36c3e9..059f8267 100644 --- a/src/web/src/views/workspace/WSEditorCommandContent.tsx +++ b/src/web/src/views/workspace/WSEditorCommandContent.tsx @@ -1172,12 +1172,12 @@ class ExampleDialog extends React.Component {this.loadSwaggerExamples()}}> By OpenAPI Specification - + */} } {(!isAdd || source != undefined) &&