feat: add templates with tutor15 support. (eduNEXT#35)

- update forum job according to the k8s-jobs patch from tutor-forum.
- use simplified hooks API introduced in tutor V15.3.0.
- fix getting incompatible yaml files.
- use DRYDOCK_CMS_SSO_USER variable instead of hard coded value.

drydock/manifest_builder/infrastructure/flex_tutor_manifest.py

@@ -36,9 +36,9 @@ def my_base_dir(root):
         for path in ["dev", "build", "local"]:
             shutil.rmtree(os.path.join(fixed_root, path))
 
-        hooks.filters.clear("env:templates:targets")
-        hooks.filters.add_items(
-            "env:templates:targets",
+        hooks.Filters.ENV_TEMPLATE_TARGETS.clear()
+
+        hooks.Filters.ENV_TEMPLATE_TARGETS.add_items(
             [
                 (
                     self.options.get("tutor_templates_version"),

drydock/references/tutor_v15.yml

@@ -0,0 +1,10 @@
+#
+# The tutor15 reference.
+---
+drydock:
+  builder_class:  drydock.manifest_builder.application.manifest_builder.ManifestBuilder
+  config_class: drydock.manifest_builder.infrastructure.tutor_config.TutorExtendedConfig
+  manifest_class: drydock.manifest_builder.infrastructure.flex_tutor_manifest.FlexibleTutorManifest
+  manifest_options:
+    output: "manifest-output"
+    tutor_templates_version: "kustomized/tutor15"

drydock/templates/kustomized/tutor15/base/apps/caddy/Caddyfile

@@ -0,0 +1,83 @@
+# Global configuration
+{
+
+    {% if not ENABLE_WEB_PROXY %}
+    # Enable proxying from all servers by default. Otherwise, X-Forwarded-* headers will
+    # be overwritten.
+    # https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#defaults
+    servers {
+        trusted_proxies static 0.0.0.0/0 ::/0
+    }
+    {% endif %}
+    {{ patch("caddyfile-global")|indent(4) }}
+}
+
+# proxy directive snippet (with logging) to be used as follows:
+#
+#     import proxy "containername:port"
+(proxy) {
+    log {
+        output stdout
+        format filter {
+            wrap json
+            fields {
+                common_log delete
+                request>headers delete
+                resp_headers delete
+                tls delete
+            }
+        }
+    }
+
+    # This will compress requests that matches the default criteria set by Caddy.
+    # see https://caddyserver.com/docs/caddyfile/directives/encode
+    # for information about the defaults; i.e. how/when this will be applied.
+    encode gzip
+
+    reverse_proxy {args.0} {
+        header_up X-Forwarded-Port {{ 443 if ENABLE_HTTPS else 80 }}
+    }
+}
+
+{{ LMS_HOST }}{$default_site_port}, {{ PREVIEW_LMS_HOST }}{$default_site_port} {
+    @favicon_matcher {
+        path_regexp ^/favicon.ico$
+    }
+    rewrite @favicon_matcher /theming/asset/images/favicon.ico
+
+    # Limit profile image upload size
+    handle_path /api/profile_images/*/*/upload {
+        request_body {
+            max_size 1MB
+        }
+    }
+
+    import proxy "lms:8000"
+
+    {{ patch("caddyfile-lms")|indent(4) }}
+
+    handle_path /* {
+        request_body {
+            max_size 4MB
+        }
+    }
+}
+
+{{ CMS_HOST }}{$default_site_port} {
+    @favicon_matcher {
+        path_regexp ^/favicon.ico$
+    }
+    rewrite @favicon_matcher /theming/asset/images/favicon.ico
+
+    import proxy "cms:8000"
+
+    {{ patch("caddyfile-cms")|indent(4) }}
+
+    handle_path /* {
+        request_body {
+            max_size 250MB
+        }
+    }
+}
+
+{{ patch("caddyfile") }}

drydock/templates/kustomized/tutor15/base/apps/openedx/config/cms.env.yml

@@ -0,0 +1,40 @@
+SITE_NAME: "{{ CMS_HOST }}"
+BOOK_URL: ""
+LOG_DIR: "/openedx/data/logs"
+LOGGING_ENV: "sandbox"
+OAUTH_OIDC_ISSUER: "{{ JWT_COMMON_ISSUER }}"
+PLATFORM_NAME: "{{ PLATFORM_NAME }}"
+FEATURES:
+  {{ patch("common-env-features")|indent(2) }}
+  {{ patch("cms-env-features")|indent(2) }}
+  CERTIFICATES_HTML_VIEW: true
+  PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}"
+  ENABLE_COURSEWARE_INDEX: true
+  ENABLE_CSMH_EXTENDED: false
+  ENABLE_LEARNER_RECORDS: false
+  ENABLE_LIBRARY_INDEX: true
+  MILESTONES_APP: true
+  ENABLE_PREREQUISITE_COURSES: true
+LMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}"
+CMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ CMS_HOST }}"
+CMS_BASE: "{{ CMS_HOST }}"
+LMS_BASE: "{{ LMS_HOST }}"
+CONTACT_EMAIL: "{{ CONTACT_EMAIL }}"
+CELERY_BROKER_TRANSPORT: "redis"
+CELERY_BROKER_HOSTNAME: "{{ REDIS_HOST }}:{{ REDIS_PORT }}"
+CELERY_BROKER_VHOST: "{{ OPENEDX_CELERY_REDIS_DB }}"
+CELERY_BROKER_USER: "{{ REDIS_USERNAME }}"
+CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}"
+ALTERNATE_WORKER_QUEUES: "lms"
+ENABLE_COMPREHENSIVE_THEMING: true
+COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"]
+STATIC_ROOT_BASE: "/openedx/staticfiles"
+EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
+EMAIL_HOST: "{{ SMTP_HOST }}"
+EMAIL_PORT: {{ SMTP_PORT }}
+EMAIL_USE_TLS: {{ "true" if SMTP_USE_TLS else "false" }}
+HTTPS: "{{ "on" if ENABLE_HTTPS else "off" }}"
+LANGUAGE_CODE: "{{ LANGUAGE_CODE }}"
+SESSION_COOKIE_DOMAIN: "{{ CMS_HOST }}"
+{{ patch("cms-env") }}
+{% include "apps/openedx/config/partials/auth.yml" %}

drydock/templates/kustomized/tutor15/base/apps/openedx/config/lms.env.yml

@@ -0,0 +1,47 @@
+SITE_NAME: "{{ LMS_HOST }}"
+BOOK_URL: ""
+LOG_DIR: "/openedx/data/logs"
+LOGGING_ENV: "sandbox"
+OAUTH_OIDC_ISSUER: "{{ JWT_COMMON_ISSUER }}"
+PLATFORM_NAME: "{{ PLATFORM_NAME }}"
+FEATURES:
+  {{ patch("common-env-features")|indent(2) }}
+  {{ patch("lms-env-features")|indent(2) }}
+  CERTIFICATES_HTML_VIEW: true
+  PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}"
+  ENABLE_COURSE_DISCOVERY: true
+  ENABLE_COURSEWARE_SEARCH: true
+  ENABLE_CSMH_EXTENDED: false
+  ENABLE_DASHBOARD_SEARCH:  true
+  ENABLE_COMBINED_LOGIN_REGISTRATION: true
+  ENABLE_GRADE_DOWNLOADS: true
+  ENABLE_LEARNER_RECORDS: false
+  ENABLE_MOBILE_REST_API: true
+  ENABLE_OAUTH2_PROVIDER: true
+  ENABLE_PREREQUISITE_COURSES: true
+  ENABLE_THIRD_PARTY_AUTH: true
+  MILESTONES_APP: true
+LMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}"
+CMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ CMS_HOST }}"
+CMS_BASE: "{{ CMS_HOST }}"
+LMS_BASE: "{{ LMS_HOST }}"
+CONTACT_EMAIL: "{{ CONTACT_EMAIL }}"
+CELERY_BROKER_TRANSPORT: "redis"
+CELERY_BROKER_HOSTNAME: "{{ REDIS_HOST }}:{{ REDIS_PORT }}"
+CELERY_BROKER_VHOST: "{{ OPENEDX_CELERY_REDIS_DB }}"
+CELERY_BROKER_USER: "{{ REDIS_USERNAME }}"
+CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}"
+ALTERNATE_WORKER_QUEUES: "cms"
+ENABLE_COMPREHENSIVE_THEMING: true
+COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"]
+STATIC_ROOT_BASE: "/openedx/staticfiles"
+EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
+EMAIL_HOST: "{{ SMTP_HOST }}"
+EMAIL_PORT: {{ SMTP_PORT }}
+EMAIL_USE_TLS: {{ "true" if SMTP_USE_TLS else "false" }}
+ACE_ROUTING_KEY: "edx.lms.core.default"
+HTTPS: "{{ "on" if ENABLE_HTTPS else "off" }}"
+LANGUAGE_CODE: "{{ LANGUAGE_CODE }}"
+SESSION_COOKIE_DOMAIN: "{{ LMS_HOST }}"
+{{ patch("lms-env") }}
+{% include "apps/openedx/config/partials/auth.yml" %}

drydock/templates/kustomized/tutor15/base/apps/openedx/config/partials/auth.yml

@@ -0,0 +1,21 @@
+SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}"
+AWS_ACCESS_KEY_ID: "{{ OPENEDX_AWS_ACCESS_KEY }}"
+AWS_SECRET_ACCESS_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
+DOC_STORE_CONFIG: null
+{{ patch("openedx-auth") }}
+XQUEUE_INTERFACE:
+  django_auth: null
+  url: null
+DATABASES:
+  default:
+    ENGINE: "django.db.backends.mysql"
+    HOST: "{{ MYSQL_HOST }}"
+    PORT: {{ MYSQL_PORT }}
+    NAME: "{{ OPENEDX_MYSQL_DATABASE }}"
+    USER: "{{ OPENEDX_MYSQL_USERNAME }}"
+    PASSWORD: "{{ OPENEDX_MYSQL_PASSWORD }}"
+    ATOMIC_REQUESTS: true
+    OPTIONS:
+      init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
+EMAIL_HOST_USER: "{{ SMTP_USERNAME }}"
+EMAIL_HOST_PASSWORD: "{{ SMTP_PASSWORD }}"

drydock/templates/kustomized/tutor15/base/apps/openedx/settings/cms/development.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+import os
+from cms.envs.devstack import *
+
+LMS_BASE = "{{ LMS_HOST }}:8000"
+LMS_ROOT_URL = "http://" + LMS_BASE
+
+# Authentication
+SOCIAL_AUTH_EDX_OAUTH2_KEY = "{{ CMS_OAUTH2_KEY_SSO_DEV }}"
+SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT = LMS_ROOT_URL
+
+FEATURES["PREVIEW_LMS_BASE"] = "{{ PREVIEW_LMS_HOST }}:8000"
+
+{% include "apps/openedx/settings/partials/common_cms.py" %}
+
+# Setup correct webpack configuration file for development
+WEBPACK_CONFIG_PATH = "webpack.dev.config.js"
+
+{{ patch("openedx-development-settings") }}
+{{ patch("openedx-cms-development-settings") }}

drydock/templates/kustomized/tutor15/base/apps/openedx/settings/cms/production.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+import os
+from cms.envs.production import *
+
+{% include "apps/openedx/settings/partials/common_cms.py" %}
+
+ALLOWED_HOSTS = [
+    ENV_TOKENS.get("CMS_BASE"),
+    "cms",
+]
+CORS_ORIGIN_WHITELIST.append("{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ CMS_HOST }}")
+
+# Authentication
+SOCIAL_AUTH_EDX_OAUTH2_KEY = "{{ CMS_OAUTH2_KEY_SSO }}"
+SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT = "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}"
+
+{{ patch("openedx-cms-production-settings") }}

drydock/templates/kustomized/tutor15/base/apps/openedx/settings/cms/test.py

@@ -0,0 +1,3 @@
+from cms.envs.test import *
+
+{% include "apps/openedx/settings/partials/common_test.py" %}

drydock/templates/kustomized/tutor15/base/apps/openedx/settings/lms/development.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+import os
+from lms.envs.devstack import *
+
+{% include "apps/openedx/settings/partials/common_lms.py" %}
+
+# Setup correct webpack configuration file for development
+WEBPACK_CONFIG_PATH = "webpack.dev.config.js"
+
+LMS_BASE = "{{ LMS_HOST}}:8000"
+LMS_ROOT_URL = "http://{}".format(LMS_BASE)
+LMS_INTERNAL_ROOT_URL = LMS_ROOT_URL
+SITE_NAME = LMS_BASE
+CMS_BASE = "{{ CMS_HOST}}:8001"
+CMS_ROOT_URL = "http://{}".format(CMS_BASE)
+LOGIN_REDIRECT_WHITELIST.append(CMS_BASE)
+
+# Session cookie
+SESSION_COOKIE_DOMAIN = "{{ LMS_HOST }}"
+SESSION_COOKIE_SECURE = False
+CSRF_COOKIE_SECURE = False
+SESSION_COOKIE_SAMESITE = "Lax"
+
+# CMS authentication
+IDA_LOGOUT_URI_LIST.append("http://{{ CMS_HOST }}:8001/logout/")
+
+FEATURES["ENABLE_COURSEWARE_MICROFRONTEND"] = False
+
+# Disable enterprise integration
+FEATURES["ENABLE_ENTERPRISE_INTEGRATION"] = False
+SYSTEM_WIDE_ROLE_CLASSES.remove("enterprise.SystemWideEnterpriseUserRoleAssignment")
+
+LOGGING["loggers"]["oauth2_provider"] = {
+    "handlers": ["console"],
+    "level": "DEBUG"
+}
+
+{{ patch("openedx-development-settings") }}
+{{ patch("openedx-lms-development-settings") }}

drydock/templates/kustomized/tutor15/base/apps/openedx/settings/lms/production.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+import os
+from lms.envs.production import *
+
+{% include "apps/openedx/settings/partials/common_lms.py" %}
+
+ALLOWED_HOSTS = [
+    ENV_TOKENS.get("LMS_BASE"),
+    FEATURES["PREVIEW_LMS_BASE"],
+    "lms",
+]
+CORS_ORIGIN_WHITELIST.append("{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}")
+
+{% if ENABLE_HTTPS %}
+# Properly set the "secure" attribute on session/csrf cookies. This is required in
+# Chrome to support samesite=none cookies.
+SESSION_COOKIE_SECURE = True
+CSRF_COOKIE_SECURE = True
+SESSION_COOKIE_SAMESITE = "None"
+{% else %}
+# When we cannot provide secure session/csrf cookies, we must disable samesite=none
+SESSION_COOKIE_SECURE = False
+CSRF_COOKIE_SECURE = False
+SESSION_COOKIE_SAMESITE = "Lax"
+{% endif %}
+
+# CMS authentication
+IDA_LOGOUT_URI_LIST.append("{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ CMS_HOST }}/logout/")
+
+# Required to display all courses on start page
+SEARCH_SKIP_ENROLLMENT_START_DATE_FILTERING = True
+
+{{ patch("openedx-lms-production-settings") }}

drydock/templates/kustomized/tutor15/base/apps/openedx/settings/lms/test.py

@@ -0,0 +1,3 @@
+from lms.envs.test import *
+
+{% include "apps/openedx/settings/partials/common_test.py" %}