scripts execution in orchestration (Azure#2)

* initial custom scripts execution

* updated recordings, unit tests. included custom script exec in orchestration

* updated bash file name reference

* fixed custom-script dependency retrival from storage account

* removed scripts from binaries

* updated unit test path assert

archetypes/shared-services/archetype.test.json

@@ -47,6 +47,10 @@
             "log-analytics": {
                 "region": "West US 2"
             }
+        },
+        "custom-script": {
+            "bash-result": "",
+            "pwsh-result": ""
         }
     },
     "orchestration": {
@@ -79,6 +83,31 @@
                     },
                     "dependencies": []
                 },
+                {
+                    "module": "custom-script-0",
+                    "type": "bash",
+                    "command": "chmod +x scripts/linux/sample-output.sh"
+                },
+                {
+                    "module": "custom-script-1",
+                    "type": "bash",
+                    "command": "scripts/linux/sample-output.sh",
+                    "output-file": "file(result-bash.json)",
+                    "property-path": "general.custom-script.bash-result",
+                    "dependencies": [
+                        "custom-script-0"
+                    ]
+                },
+                {
+                    "module": "custom-script-2",
+                    "type": "powershell",
+                    "command": "scripts/windows/sample-output.ps1 -TestString 'Hello World'",
+                    "output-file": "file(result-pwsh.json)",
+                    "property-path": "general.custom-script.pwsh-result",
+                    "dependencies": [
+                        "custom-script-1"
+                    ]
+                },
                 {
                     "module": "shared-services-net",
                     "resource-group-name": "${general.organization-name}-${shared-services.deployment-name}-net-rg",

custom_script.py

@@ -0,0 +1,3 @@
+if __name__ == '__main__':
+    from executables.custom_script_exec import main
+    main()

executables/custom_script_exec.py

@@ -0,0 +1,144 @@
+from pathlib import Path
+from sys import argv
+from argparse import ArgumentParser, FileType
+from orchestration.integration.custom_scripts.script_execution import CustomScriptExecution
+from logging.config import dictConfig
+from orchestration.common import helper
+from orchestration.models.script_type import ScriptType
+import logging
+import json
+
+# Logging preperation.
+#-----------------------------------------------------------------------------
+
+# Set the log configuration using a json config file.        
+if Path('logging/config.json').exists():
+    with open('logging/config.json', 'rt') as f:
+        config = json.load(f)
+        dictConfig(config)
+else:
+    logging.basicConfig(level=logging.INFO)
+
+# Create a new logger instance using the provided configuration.
+_logger = logging.getLogger(__name__)
+
+def _write_to_console(result: dict):
+    import sys
+    sys.stdout.write("\n")
+    sys.stdout.write(json.dumps(result))
+    sys.stdout.write("\n")
+    sys.stdout.write("\n")
+
+def _execute_custom_script(
+    args,
+    script_type: ScriptType):
+    from orchestration.integration.custom_scripts.script_execution import CustomScriptExecution
+    script_execution = CustomScriptExecution()
+    return script_execution.execute(
+        script_type=script_type,
+        command=args.command,
+        output_file_path=args.output,
+        property_path=args.property_path,
+        file_path_to_update=args.path)
+
+def run_powershell_script(args):
+
+    result = _execute_custom_script(
+        args=args,
+        script_type=ScriptType.POWERSHELL)
+
+    if args.show:
+        _write_to_console(result)
+
+def run_bash_script(args):
+    result = _execute_custom_script(
+        args=args,
+        script_type=ScriptType.BASH)
+
+    if args.show:
+        _write_to_console(result)
+
+def set_default_arguments(parser):
+
+    parser.add_argument('-c', '--command',
+                dest='command',
+                required=True,
+                help='Custom script command. Powershell or Bash')
+
+    parser.add_argument('-s', '--show',
+                dest='show',
+                required=False,
+                action="store_true",
+                help='Show output on console')
+
+    parser.add_argument('-o', '--output',
+                dest='output',
+                required=False,
+                help='Output file name')
+
+    parser.add_argument('-property', '--property-path',
+                dest='property_path',
+                required=False,
+                help='Property path, corresponds to a property from config.json')
+
+    parser.add_argument('-path',
+                dest='path',
+                required=False,
+                help='Config.json path to modify')
+
+def main():
+
+    #-----------------------------------------------------------------------------
+    # Script argument definitions.
+    #-----------------------------------------------------------------------------
+
+    # Define a top level parser.
+    parser = ArgumentParser(
+        description='Set of commands to run custom scripts (Powershell or Bash)')
+
+    # Create a subparser to distinguish between the different deployment commands.
+    subparsers = parser.add_subparsers(
+        help='Executes custom scripts. Output is a JSON string: { "output": "" }')
+
+    powershell_subparser = subparsers\
+        .add_parser(
+            'powershell', 
+            help='Executes custom powershell scripts')
+
+    set_default_arguments(powershell_subparser)
+
+    powershell_subparser\
+        .set_defaults(
+            func=run_powershell_script)
+
+    bash_subparser = subparsers\
+        .add_parser(
+            'bash', 
+            help='Executes custom bash scripts')
+
+    set_default_arguments(bash_subparser)
+
+    bash_subparser\
+        .set_defaults(
+            func=run_bash_script)
+
+    #-----------------------------------------------------------------------------
+    # Process parameter arguments.
+    #-----------------------------------------------------------------------------
+
+    # Gather the provided argument within an array.
+    args = parser.parse_args()
+
+    # Let's check if there are parameters passed, if not, print function usage
+    if len(vars(args)) == 0:
+        parser.print_usage()
+        exit()
+
+    #-----------------------------------------------------------------------------
+    # Call the function indicated by the invocation command.
+    #-----------------------------------------------------------------------------
+    try:
+        args.func(args)
+    except Exception as ex:
+        _logger.error('There was an error provisioning the resources: {}'.format(str(ex)))
+        _logger.error(ex)

executables/vdc_exec.py

@@ -77,6 +77,7 @@ def create_deployment(parsed_args):
                 parameter_initializer._vdc_storage_account_resource_group,
                 parameter_initializer._validate_deployment,
                 parameter_initializer._deploy_all_modules,
+                parameter_initializer._deployment_configuration_path,
                 parameter_initializer._module_deployment_order,
                 parameter_initializer._resource_group,
                 parameter_initializer._single_module,

orchestration/common/helper.py

@@ -646,4 +646,54 @@ def create_unique_string(
 def findAll_regex(
     regex_pattern: str,
     text: str) -> list:
-    return re.findall(regex_pattern, text)
+    return re.findall(regex_pattern, text)
+
+def save_json_file(
+    output: dict, 
+    path: str):
+
+    with open(path, 'w+') as f:
+        f.write(json.dumps(output))
+
+def modify_json_file(
+    prop_value: object,
+    prop_key: str,
+    path: str):
+
+    with open(path, 'r') as template_file_fd:
+        local_file: dict = json.load(template_file_fd)
+
+    # i.e. shared-services.network.subnets
+    parent_keys = prop_key.split('.')
+
+    # Position on first index
+    parent = local_file[parent_keys[0]]
+    for parent_key in range(1, len(parent_keys) - 1):
+        _key = parent_keys[parent_key]
+        parent = parent[_key]
+
+    last_parent_key = parent_keys[len(parent_keys) - 1]
+    parent[last_parent_key] = prop_value
+
+    with open(path, "w+") as jsonFile:
+        json.dump(local_file, jsonFile, indent=4)
+
+def modify_json_object(
+    prop_value: object,
+    prop_key: str,
+    json_object: dict):
+
+    # i.e. shared-services.network.subnets
+    parent_keys = prop_key.split('.')
+
+    # Position on first index
+    parent = json_object[parent_keys[0]]
+
+    for parent_key in range(1, len(parent_keys) - 1):
+        _key = parent_keys[parent_key]
+        parent = parent[_key]
+
+    last_parent_key = parent_keys[len(parent_keys) - 1]
+    parent[last_parent_key] = prop_value
+
+    return json_object

orchestration/common/parameter_initializer.py

@@ -39,6 +39,7 @@ def initialize(
         self._service_principals = list()
         self._deploy_all_modules = True
         self._deploy_module_dependencies = False
+        self._deployment_configuration_path = deployment_configuration_path
         self._upload_scripts = False
         self._service_principals = None
         self._module_dependencies = None

orchestration/integration/custom_scripts/bash_execution.py

@@ -0,0 +1,67 @@
+import subprocess as sp
+import json
+import logging
+import sys
+
+class BashScriptExecution(object):
+
+    _sp_shell_flag: bool = False
+
+    # Logging preperation.
+    #-----------------------------------------------------------------------------
+
+    # Retrieve the main logger; picks up parent logger instance if invoked as a module.
+    _logger = logging.getLogger(__name__)
+    '''
+    logger: Logger instance used to process all module log statements.
+    '''
+
+    def __init__(self):
+
+        if sys.platform == "linux" or sys.platform == "linux2":
+            self._sp_shell_flag = False
+        elif sys.platform == "win32":
+            self._sp_shell_flag = True
+
+    def execute(
+        self, 
+        command: str) -> dict:
+        try:
+            self._logger.debug('Executing the following command: {}.'
+                    .format(command))
+            bash_command = "sh -c \"{}\"".format(command)
+            # Package that splits based on whitespace and
+            # preserves quoted strings
+            import shlex
+            command_exec = shlex.split(bash_command)
+
+            # Run the check command.
+            result = sp.check_output(command_exec, shell=self._sp_shell_flag, universal_newlines=True)
+
+            self._logger.debug('The command executed successfuly, producing the following output: {}'
+                        .format(result))
+
+            try:
+                result = json.loads(result)
+            except:
+                result = \
+                        result\
+                            .replace('\n', '')\
+                            .replace('\\n', '')
+
+            result_dict = dict({
+                'output': result
+            })
+
+            return result_dict
+
+        except sp.CalledProcessError as e:
+
+            self._logger.error('The following error occurred: ' + str(e))
+            sys.exit(1) # Exit the module as it is not possible to progress the deployment.
+            raise(e)
+        except ValueError as e:
+
+            self._logger.error(e)
+            sys.exit(1) # Exit the module as it is not possible to progress the deployment.
+            raise(e)