#427 start refactor skaffold generation

.dockerignore

@@ -1,3 +1,10 @@
 **/node_modules
 .tox
-docs
+docs
+applications
+infrastructure
+blueprint
+test
+.github
+.git
+.vscode

tools/cloudharness_utilities/skaffold.py

@@ -2,36 +2,47 @@
 import logging
 import json
 import time
+import shutil
+
+from os.path import join, dirname, relpath, basename
 
 from cloudharness_utilities.constants import HELM_CHART_PATH, DEPLOYMENT_CONFIGURATION_PATH, DEPLOYMENT_PATH, \
     BASE_IMAGES_PATH, STATIC_IMAGES_PATH
 from cloudharness_utilities.helm import KEY_APPS, KEY_HARNESS, KEY_DEPLOYMENT, KEY_TASK_IMAGES
 from cloudharness_utilities.utils import get_template, dict_merge, find_dockerfiles_paths, app_name_from_path, \
-    find_file_paths, guess_build_dependencies_from_dockerfile, merge_to_yaml_file, get_json_template, get_image_name
+    find_file_paths, guess_build_dependencies_from_dockerfile, merge_configuration_directories, merge_to_yaml_file, get_json_template, get_image_name
 
 
-def create_skaffold_configuration(root_paths, helm_values, output_path='.', manage_task_images=True):
+def create_skaffold_configuration(root_paths, helm_values, output_path='.', manage_task_images=True, merge_build_path="./build"):
     skaffold_conf = get_template('skaffold-template.yaml', True)
     apps = helm_values[KEY_APPS]
-    base_image_name = helm_values['registry'].get('name', '') + helm_values['name']
+    base_image_name = helm_values['registry'].get(
+        'name', '') + helm_values['name']
     artifacts = {}
     overrides = {}
     release_config = skaffold_conf['deploy']['helm']['releases'][0]
     release_config['name'] = helm_values['namespace']
     release_config['namespace'] = helm_values['namespace']
 
+    if not os.path.exists(merge_build_path):
+        os.makedirs(merge_build_path)
+    else:
+        shutil.rmtree(merge_build_path)
+
     def remove_tag(image_name):
         return image_name[0:-len(str(helm_values['tag']))-1]
 
     def get_image_tag(name):
         return f"{get_image_name(name, base_image_name)}"
 
-    def build_artifact(image_name, root_path, requirements=None, dockerfile_path=''):
+    builds = {}
+
+    def build_artifact(image_name, context_path, requirements=None, dockerfile_path=''):
         artifact_spec = {
             'image': image_name,
-            'context': root_path,
+            'context': context_path,
             'docker': {
-                'dockerfile': os.path.join(dockerfile_path, 'Dockerfile'),
+                'dockerfile': join(dockerfile_path, 'Dockerfile'),
                 'buildArgs': {
                     'REGISTRY': helm_values["registry"]["name"],
                     'TAG': helm_values["tag"],
@@ -47,49 +58,80 @@ def build_artifact(image_name, root_path, requirements=None, dockerfile_path='')
 
     release_config['artifactOverrides'][KEY_APPS] = {}
     base_images = set()
+
+    def process_build_dockerfile(dockerfile_path, root_path, global_context=False):
+        app_name = app_name_from_path(basename(dockerfile_path))
+        if app_name in helm_values[KEY_TASK_IMAGES]:
+            context_path = relpath(root_path, output_path)
+            if app_name in builds:
+                if global_context:
+                    source_path = join(root_path, 'libraries')
+                    dest_path = join(merge_build_path, 'libraries')
+                else:
+                    source_path = context_path
+                    dest_path = join(merge_build_path, dirname(
+                        relpath(root_path, dockerfile_path)))
+                if os.path.exists(source_path):
+                    merge_configuration_directories(
+                        builds[app_name], dest_path)
+                    merge_configuration_directories(source_path, dest_path)
+                    builds[app_name] = context_path
+                    context_path = relpath(
+                        merge_build_path, output_path)
+
+            builds[app_name] = context_path
+            base_images.add(get_image_name(app_name))
+            artifacts[app_name] = build_artifact(
+                get_image_tag(app_name),
+                context_path,
+                dockerfile_path=os.relpath(dockerfile_path, output_path)
+            )
+
     for root_path in root_paths:
         skaffold_conf = dict_merge(skaffold_conf, get_template(
-            os.path.join(root_path, DEPLOYMENT_CONFIGURATION_PATH, 'skaffold-template.yaml')))
-
+            join(root_path, DEPLOYMENT_CONFIGURATION_PATH, 'skaffold-template.yaml')))
 
-        base_dockerfiles = find_dockerfiles_paths(os.path.join(root_path, BASE_IMAGES_PATH))
+        base_dockerfiles = find_dockerfiles_paths(
+            join(root_path, BASE_IMAGES_PATH))
 
-
         for dockerfile_path in base_dockerfiles:
-            context_path = os.path.relpath(root_path, output_path)
-            app_name = app_name_from_path(os.path.basename(dockerfile_path))
-            if app_name in helm_values[KEY_TASK_IMAGES]:
-                base_images.add(get_image_name(app_name))
-                artifacts[app_name] = build_artifact(get_image_tag(app_name), context_path,
-                                                    dockerfile_path=os.path.relpath(dockerfile_path, context_path))
+            process_build_dockerfile(root_path, dockerfile_path, app_name)
+
     static_images = set()
     for root_path in root_paths:
-        static_dockerfiles = find_dockerfiles_paths(os.path.join(root_path, STATIC_IMAGES_PATH))
-
+        static_dockerfiles = find_dockerfiles_paths(
+            join(root_path, STATIC_IMAGES_PATH))
+
         for dockerfile_path in static_dockerfiles:
             context_path = os.path.relpath(dockerfile_path, output_path)
-            app_name = app_name_from_path(os.path.basename(context_path))
+            app_name = app_name_from_path(basename(context_path))
             if app_name in helm_values[KEY_TASK_IMAGES]:
                 static_images.add(get_image_name(app_name))
-                artifacts[app_name] = build_artifact(get_image_tag(app_name), context_path, guess_build_dependencies_from_dockerfile(dockerfile_path))
+                artifacts[app_name] = build_artifact(
+                    get_image_tag(app_name),
+                    context_path,
+                    guess_build_dependencies_from_dockerfile(dockerfile_path)
+                )
 
     for root_path in root_paths:
-        apps_path = os.path.join(root_path, 'applications')
+        apps_path = join(root_path, 'applications')
         app_dockerfiles = find_dockerfiles_paths(apps_path)
-        
+
         release_config['artifactOverrides'][KEY_TASK_IMAGES] = {
             task_image: remove_tag(helm_values[KEY_TASK_IMAGES][task_image])
             for task_image in helm_values[KEY_TASK_IMAGES]
         }
         for dockerfile_path in app_dockerfiles:
-            app_relative_to_skaffold = os.path.relpath(dockerfile_path, output_path)
+            app_relative_to_skaffold = os.path.relpath(
+                dockerfile_path, output_path)
             context_path = os.path.relpath(dockerfile_path, '.')
             app_relative_to_base = os.path.relpath(dockerfile_path, apps_path)
             app_name = app_name_from_path(app_relative_to_base)
             app_key = app_name.replace('-', '_')
             if app_key not in apps:
                 if 'tasks' in app_relative_to_base and manage_task_images:
-                    parent_app_name = app_name_from_path(app_relative_to_base.split('/tasks')[0])
+                    parent_app_name = app_name_from_path(
+                        app_relative_to_base.split('/tasks')[0])
                     parent_app_key = parent_app_name.replace('-', '_')
 
                     if parent_app_key in apps:
@@ -98,10 +140,12 @@ def build_artifact(image_name, root_path, requirements=None, dockerfile_path='')
 
                 continue
 
-
-            build_requirements = apps[app_key][KEY_HARNESS]['dependencies'].get('build', [])
-            app_image_tag = remove_tag(apps[app_key][KEY_HARNESS][KEY_DEPLOYMENT]['image'])
-            artifacts[app_key] = build_artifact(app_image_tag, app_relative_to_skaffold, build_requirements)
+            build_requirements = apps[app_key][KEY_HARNESS]['dependencies'].get('build', [
+            ])
+            app_image_tag = remove_tag(
+                apps[app_key][KEY_HARNESS][KEY_DEPLOYMENT]['image'])
+            artifacts[app_key] = build_artifact(
+                app_image_tag, app_relative_to_skaffold, build_requirements)
 
             app = apps[app_key]
             if app[KEY_HARNESS][KEY_DEPLOYMENT]['image']:
@@ -112,7 +156,7 @@ def build_artifact(image_name, root_path, requirements=None, dockerfile_path='')
                                 'image': remove_tag(app[KEY_HARNESS][KEY_DEPLOYMENT]['image'])
                             }
                         }
-                    }
+                }
 
             flask_main = find_file_paths(context_path, '__main__.py')
 
@@ -125,15 +169,17 @@ def build_artifact(image_name, root_path, requirements=None, dockerfile_path='')
                                 'args': [f'/usr/src/app/{os.path.basename(flask_main[0])}/__main__.py']
                             }
                         }
-                    }
+                }
 
         skaffold_conf['build']['artifacts'] = [v for v in artifacts.values()]
-        merge_to_yaml_file(skaffold_conf, os.path.join(output_path, 'skaffold.yaml'))
+        merge_to_yaml_file(skaffold_conf, os.path.join(
+            output_path, 'skaffold.yaml'))
     return skaffold_conf
 
 
 def create_vscode_debug_configuration(root_paths, helm_values):
-    logging.info("Creating VS code cloud build configuration.\nCloud build extension is needed to debug.")
+    logging.info(
+        "Creating VS code cloud build configuration.\nCloud build extension is needed to debug.")
 
     vscode_launch_path = '.vscode/launch.json'
 

tools/cloudharness_utilities/utils.py

@@ -197,6 +197,8 @@ def movedircontent(root_src_dir, root_dst_dir):
 
 
 def merge_configuration_directories(source, dest):
+    if source == dest:
+        return
     if not os.path.exists(source):
         return
     if not os.path.exists(dest):

tools/harness-deployment

@@ -46,7 +46,7 @@ if __name__ == "__main__":
     parser.add_argument('-ex', '--exclude', dest='exclude', action="append", default=[],
                         help='Specify application to exclude from the deployment')
     parser.add_argument('-e', '--env', dest='env', action="store", default=None,
-                        help='Specify the name of the environment (will load values-[ENV].yaml files')
+                        help='Specify the name of the environment (will load values-[ENV].yaml files)')
     parser.add_argument('-m', '--merge', dest='merge', action="store", default=None,
                         help='Deprecated -- Merge is now automatic')