feat: add a basic manifest repository implementation (eduNEXT#1)

The `BaseManfests` builder will render a standard Tutor environment
based on the templates used in version 13.3.1 of Tutor and use it
as a base of Kustomization application with additional resources as
overlays.

The `TutorExtendedConfig` will return the Tutor configuration values
of the current `TUTOR_ROOT` and will use the default values of
the template set (defined in a file `defaults.yml`) as a fallback.

README.rst

@@ -1,20 +1,51 @@
 drydock: a flexible manifest builder for Open edX
 =================================================
 
+**⚠️ Warning**: drydock is currently in an alpha stage and may suffer substantial changes
+while it transitions in to a stable stage.
+
+
 Installation
 ------------
 
 ::
 
     tvm plugins install -e git+https://github.com/edunext/drydock#egg=drydock
+    tutor plugins enable drydock
 
-Usage
------
+Getting started
+---------------
 
-::
+Drydock aims to offer flexibility on the kind of environment you want to generate, for that reason
+you can chose different kind of implementations (of renderers?) depending on your needs. At the
+moment Drydock ships with a basic manifest builder that wraps the Kubernetes generated files by tutor
+in a Kustomize application with some useful extra resources.
 
-    tutor plugins enable drydock
-    tutor drydock save -r drydock/references/tutor_v13.yml
+To use drydock just use the reference that defines the implementation you want drydock to use and run:
+
+..  code:: bash
+
+    tutor drydock save -r reference.yml
+
+An example reference would look something like this:
+
+..  code:: yaml
+
+    drydock:
+      builder_class:  drydock.manifest_builder.application.manifest_builder.ManifestBuilder
+      config_class: drydock.manifest_builder.infrastructure.tutor_config.TutorConfig
+      manifest_class: drydock.manifest_builder.infrastructure.tutor_based_manifests.BaseManifests
+      manifest_options:
+        output: "drydock-environment"
+
+This will render the default drydock environment on the `drydock-environment` directory, allowing
+you to check the files onto version control and use tools for continuous deployment such as
+Flux or ArgoCD. The default environment is generated using Tutor and its templates, as a result
+it should be compatible with all the plugins, variables and patches.
+
+**Note:** If you are using module or yaml plugins in tutor and set ``manifest_options.output=env``
+you will have to define your ``TUTOR_PLUGINS_ROOT`` outside of your ``TUTOR_ROOT`` because 
+drydock will override your tutor env and erase your plugins.
 
 
 Rationale

drydock/manifest_builder/domain/config.py

@@ -4,5 +4,9 @@
 class DrydockConfig(ABC):
 
     @abstractmethod
-    def get_data() -> dict:
+    def get_data(self) -> dict:
+        pass
+
+    @abstractmethod
+    def get_root(self) -> str:
         pass

drydock/manifest_builder/domain/manifest_repository.py

@@ -5,5 +5,5 @@
 class ManifestRepository(ABC):
 
     @abstractmethod
-    def save(config: DrydockConfig) -> None:
+    def save(self, config: DrydockConfig) -> None:
         pass

drydock/manifest_builder/infrastructure/tutor_based_manifests.py

@@ -0,0 +1,97 @@
+"""
+Collection of tutor based renderers for Kubernetes manifests.
+"""
+import os
+import shutil
+import tempfile
+from os.path import join as path_join
+
+import pkg_resources
+from tutor import env as tutor_env
+from tutor import fmt, hooks
+
+from drydock.manifest_builder.domain.config import DrydockConfig
+from drydock.manifest_builder.domain.manifest_repository import ManifestRepository
+
+
+class BaseManifests(ManifestRepository):
+    """Baseline of Kubernetes configuration repository.
+
+    Generates an environment based on Tutor with the relevant Kubernetes configuration
+    to be tracked by version control.
+    """
+    TEMPLATE_ROOT = "kustomized/tutor13"
+    TEMPLATE_TARGETS = [
+        f"{TEMPLATE_ROOT}/base",
+        f"{TEMPLATE_ROOT}/extensions",
+        f"{TEMPLATE_ROOT}/kustomization.yml",
+    ]
+
+    def __init__(self, options: dict) -> None:
+        """Initialize the class based on the `manifest_options` from the reference.
+
+        Parameters
+        ----------
+        options: dict
+            Defines additional configuration options for the generations of the
+            configuration repository.
+
+            - ["output"]: Name of the directory to store the manifests.
+        """
+        self.output_dir = options.get("output", "drydock-env")
+
+    def render(self, root: str, config: DrydockConfig) -> None:
+        """Register drydock custom templates and render a tutor env."""
+        with hooks.Contexts.APP("drydock-base").enter():
+            hooks.Filters.ENV_TEMPLATE_ROOTS.add_item(pkg_resources.resource_filename("drydock", "templates"))
+            hooks.Filters.ENV_TEMPLATE_TARGETS.add_items(
+                [
+                    (target, "drydock") for target in self.TEMPLATE_TARGETS
+                ],
+            )
+        tutor_env.save(root, config.get_data())
+        hooks.Filters.ENV_TEMPLATE_ROOTS.clear(context=hooks.Contexts.APP("drydock-base").name)
+        hooks.Filters.ENV_TEMPLATE_TARGETS.clear(context=hooks.Contexts.APP("drydock-base").name)
+
+    def relocate_env(self, src: str, dst: str) -> None:
+        """Moves the drydock rendered templates and tutor plugins to src.
+
+        At the moment we render a full tutor env with our templates under
+        'env/drydock/default'. The final drydock env is the one in 'env/drydock'
+        **plus** the plugin directory that has to be manually relocated to
+        'env/drydock/base/plugins'.
+
+        Parameters
+        ----------
+        src: str
+            The inital path where the full tutor env was rendered.
+        dst: str
+            The path to save the final drydock env.
+        """
+        base_dir = path_join(src, "env/drydock", f"{self.TEMPLATE_ROOT}")
+        plugins_dir = path_join(src, "env/plugins")
+        shutil.move(base_dir, dst)
+        if os.path.exists(plugins_dir):
+            shutil.move(plugins_dir, path_join(dst, "base"))
+
+    def save(self, config: DrydockConfig) -> None:
+        """Creates an alternative tutor env with the base openedx installation.
+
+        The generated env consists of a Kustomize application that uses the original
+        tutor env as a base in addition to an `extensions` overlay to include additional
+        resources.
+
+        Parameters
+        ----------
+        config: DrydockConfig
+            A Tutor configuration extension.
+        """
+        tutor_root = config.get_root()
+        dst = path_join(tutor_root, self.output_dir)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            src = path_join(tmpdir)
+            self.render(src, config)
+            shutil.rmtree(path=dst, ignore_errors=True)
+            self.relocate_env(src, dst)
+
+        fmt.echo_info(f"Drydock environment generated in {dst}")

drydock/manifest_builder/infrastructure/tutor_config.py

@@ -1,14 +1,65 @@
-from abc import abstractmethod
-from tutor import config as tutor_config
+import pkg_resources
+import tutor
+import yaml
+
 from drydock.manifest_builder.domain.config import DrydockConfig
 
 
 class TutorConfig(DrydockConfig):
-
     def __init__(self, context, options: dict):
         self.context = context
         self.options = options
 
     def get_data(self) -> dict:
-        config = tutor_config.load_full(self.context.obj.root)
+        config = tutor.config.load_full(self.context.obj.root)
         return config
+
+    def get_root(self) -> dict:
+        return self.context.obj.root
+
+
+class TutorExtendedConfig(DrydockConfig):
+    """Drydock configuration based on tutor."""
+    DEFAULT_TEMPLATE_SET = "kustomized/tutor13"
+
+    def __init__(self, context, options: dict):
+        """Initialize the class based on the `config_options` from the manifest.
+
+        Parameters
+        ----------
+        context: clic.core.Context
+            context of the current Tutor command.
+        options: dict
+
+            - ["template_set"]: template set to render default values from.
+        """
+        self.context = context
+        self.options = options
+
+    def get_data(self) -> dict:
+        """Return tutor config values using a template set defaults as fallback.
+
+        Retrieve the Tutor configuration values for the current TUTOR_ROOT and 
+        use the the values in `defaults.yml` of the chosen template set
+
+        Returns
+        -------
+        base: dict
+            Tutor configuration values taken from the config.yml at the TUTOR_ROOT
+            using the defaults from the template set as a fallback.
+        """
+        template_set = self.options.get("template_set", self.DEFAULT_TEMPLATE_SET)
+
+        try:
+            defaults_path = pkg_resources.resource_filename("drydock", f"templates/{template_set}/defaults.yml")
+            with open(defaults_path, encoding="utf-8") as file:
+                base = yaml.safe_load(file)
+        except FileNotFoundError:
+            base = {}
+
+        config = tutor.config.load_full(self.get_root())
+        base.update(config)
+        return base
+
+    def get_root(self) -> str:
+        return self.context.obj.root

drydock/references/kustomized_v13.yml

@@ -0,0 +1,10 @@
+# The kustomized_v13 reference is a Kustomize based instalation that uses the environment
+# generated by tutor v13.3.1 as a Base with additional resources such as an ingress and
+# a flower deployment used as overlays.
+#
+drydock:
+  builder_class:  drydock.manifest_builder.application.manifest_builder.ManifestBuilder
+  config_class: drydock.manifest_builder.infrastructure.tutor_config.TutorConfig
+  manifest_class: drydock.manifest_builder.infrastructure.tutor_based_manifests.BaseManifests
+  manifest_options:
+    output: "drydock-environment"

drydock/templates/README.md

@@ -0,0 +1,32 @@
+# Drydock Templates
+
+The inventory of template sets used by drydock.
+
+## Template list
+
+1. `kustomized/tutor13`: Kubernetes focused template set. Uses the templates from Tutor v13 as a Kustomize
+    base (rendered in the `base` directory) and adds additional resources as an overlay in the
+    `extensions` directory.
+    The additional resources include:
+
+    ```yaml
+    - flowers.yml # A flowers deployment to track celery tasks.
+    - hpa.yml     # An Horizontal Pod Autoscaler for lms, cms and workers.
+    - ingress.yml # An ingress with certmanager to use instead of caddy as web proxy.
+    ```
+
+    The list of variables with their default values can be found in [defaults.yml](kustomized/tutor13/defaults.yml)
+
+    <details>
+    <summary>List of patches</summary>
+
+    ```
+    {{ patch("drydock-kustomization-resources")}}
+    {{ patch("drydock-kustomization-patches")}}
+    {{ patch("drydock-overrides") }}
+    ```
+    </details>
+
+2. `tutor/v13`: Small subset of templates from Tutor v13. Mostly as a proof con concept.
+
+3. `tutor/v13`: Small subset of templates from Tutor v12. Mostly as a proof con concept.

drydock/templates/kustomized/tutor13/base/apps/caddy/Caddyfile

@@ -0,0 +1,62 @@
+# Global configuration
+{
+    {{ patch("caddyfile-global")|indent(4) }}
+}
+
+# proxy directive snippet (with logging) to be used as follows:
+#
+#     import proxy "containername:port"
+(proxy) {
+    log {
+        output stdout
+        format filter {
+            wrap json
+            fields {
+                common_log delete
+                request>headers delete
+                resp_headers delete
+                tls delete
+            }
+        }
+    }
+
+    reverse_proxy {args.0} {
+        header_up X-Forwarded-Port {{ 443 if ENABLE_HTTPS else 80 }}
+    }
+}
+
+{{ LMS_HOST }}{$default_site_port}, {{ PREVIEW_LMS_HOST }}{$default_site_port} {
+    @favicon_matcher {
+        path_regexp ^/favicon.ico$
+    }
+    rewrite @favicon_matcher /theming/asset/images/favicon.ico
+
+    # Limit profile image upload size
+    request_body /api/profile_images/*/*/upload {
+        max_size 1MB
+    }
+    request_body {
+        max_size 4MB
+    }
+
+    import proxy "lms:8000"
+
+    {{ patch("caddyfile-lms")|indent(4) }}
+}
+
+{{ CMS_HOST }}{$default_site_port} {
+    @favicon_matcher {
+        path_regexp ^/favicon.ico$
+    }
+    rewrite @favicon_matcher /theming/asset/images/favicon.ico
+
+    request_body {
+        max_size 250MB
+    }
+
+    import proxy "cms:8000"
+
+    {{ patch("caddyfile-cms")|indent(4) }}
+}
+
+{{ patch("caddyfile") }}