feat: add extra plugin (eduNEXT#48)

README.rst

@@ -12,6 +12,14 @@ Drydock is an opinionated tool offering a set of Tutor plugins aiming to provide
 - Add a custom nginx and cert-manager configuration
 - Add a set of debug resources to help diagnose issues
 
+Extra plugins added:
+
+- A patch that allows for the tuning of celery workers via `DRYDOCK_ENABLE_CELERY_TUNING`
+- Allows caddy to catch requests for multiple domains through `DRYDOCK_ENABLE_MULTITENANCY`
+- Add scorm matcher to caddy through `DRYDOCK_ENABLE_SCORM`
+- A patch that allows for the use of sentry via `DRYDOCK_ENABLE_SENTRY` and `DRYDOCK_SENTRY_DSN`
+- Patch for cms and lms worker pods to allow pod probes and lifecycle to work properly. Enabled via `DRYDOCK_POD_LIFECYCLE`
+
 Installation
 ------------
 
@@ -43,6 +51,12 @@ The following configuration options are available:
 - `DRYDOCK_CUSTOM_CERTS`: A dictionary of custom certificates to use with cert-manager. Defaults to `{}`.
 - `DRYDOCK_NEWRELIC_LICENSE_KEY`: The New Relic license key. Defaults to `""`.
 - `DRYDOCK_DEBUG`: Whether to deploy debug resources. Defaults to `false`.
+- `DRYDOCK_ENABLE_CELERY_TUNING` : Whether to enable celery tuning. Defaults to `true`.
+- `DRYDOCK_ENABLE_MULTITENANCY` : Whether to enable multitennacy. Defaults to `true`.
+- `DRYDOCK_ENABLE_SCORM` : Whether to enable scorm. Defaults to `true`.
+- `DRYDOCK_ENABLE_SENTRY` : Whether to enable sentry. Defaults to `true`.
+- `DRYDOCK_SENTRY_DSN` : The sentry DSN. Defaults to `""`.
+- `DRYDOCK_POD_LIFECYCLE` : Whether to enable pod lifecycle. Defaults to `true`.
 
 Rationale
 ---------

drydock/patches/caddyfile

@@ -0,0 +1,33 @@
+{% if DRYDOCK_ENABLE_MULTITENANCY -%}
+{$default_site_port} {
+@favicon_matcher {
+    path_regexp ^/favicon.ico$
+}
+rewrite @favicon_matcher /theming/asset/images/favicon.ico
+
+# Limit profile image upload size
+request_body /api/profile_images/*/*/upload {
+    max_size 1MB
+}
+request_body {
+    max_size 4MB
+}
+import proxy "lms:8000"
+{% if DRYDOCK_ENABLE_SCORM -%}
+@scorm_matcher {
+    path /scorm-proxy/*
+}
+route @scorm_matcher {
+    uri /scorm-proxy/* strip_prefix /scorm-proxy
+{% if MINIO_HOST is defined %}
+    reverse_proxy minio:9000 {
+        header_up Host {{ MINIO_HOST }}
+{% else %}
+    reverse_proxy https://{{ S3_STORAGE_BUCKET }}.s3.amazonaws.com {
+        header_up Host {{ S3_STORAGE_BUCKET }}.s3.amazonaws.com
+{% endif %}
+    }
+}
+{% endif %}
+}
+{% endif %}

drydock/patches/caddyfile-cms

@@ -0,0 +1,17 @@
+{% if DRYDOCK_ENABLE_SCORM -%}
+@scorm_matcher {
+    path /scorm-proxy/*
+}
+route @scorm_matcher {
+{% if MINIO_HOST is defined %}
+    uri replace /scorm-proxy/ /{{ MINIO_BUCKET_NAME }}/
+    reverse_proxy minio:9000 {
+    header_up Host {{ MINIO_HOST }}
+{% else %}
+    uri /scorm-proxy/* strip_prefix /scorm-proxy
+    reverse_proxy https://{{ S3_STORAGE_BUCKET }}.s3.amazonaws.com {
+    header_up Host {{ S3_STORAGE_BUCKET }}.s3.amazonaws.com
+{% endif %}
+    }
+}
+{% endif %}

drydock/patches/caddyfile-lms

@@ -0,0 +1,17 @@
+{% if DRYDOCK_ENABLE_SCORM -%}
+@scorm_matcher {
+    path /scorm-proxy/*
+}
+route @scorm_matcher {
+{% if MINIO_HOST is defined %}
+    uri replace /scorm-proxy/ /{{ MINIO_BUCKET_NAME }}/
+    reverse_proxy minio:9000 {
+    header_up Host {{ MINIO_HOST }}
+{% else %}
+    uri /scorm-proxy/* strip_prefix /scorm-proxy
+    reverse_proxy https://{{ S3_STORAGE_BUCKET }}.s3.amazonaws.com {
+    header_up Host {{ S3_STORAGE_BUCKET }}.s3.amazonaws.com
+{% endif %}
+    }
+}
+{% endif %}

drydock/patches/cms-env

@@ -0,0 +1,25 @@
+{% if DRYDOCK_ENABLE_CELERY_TUNING -%}
+CELERY_ACKS_LATE: true
+{% endif %}
+{% if DRYDOCK_ENABLE_SENTRY -%}
+# eox plugin settings must be defined in the ENV patches. If defined in python settings patches, they won't
+# take effect since plugins settings are loaded before Tutor loads its settings.
+EOX_CORE_SENTRY_INTEGRATION_DSN: "{{ DRYDOCK_SENTRY_DSN }}"
+EOX_CORE_SENTRY_IGNORED_ERRORS: [
+    {
+      "exc_class": "xmodule.exceptions.NotFoundError"
+    },
+    {
+    "exc_class": "openedx.core.djangoapps.user_authn.exceptions.AuthFailedError",
+    "exc_text": [".*Email or password is incorrect"]
+    },
+    {
+    "exc_class": "opaque_keys.InvalidKeyError",
+    "exc_text": [".*No enrollment found for user.*"]
+    },
+    {
+    "exc_class": "opaque_keys.NotFoundError",
+    "exc_text": [".*No course found by course id.*"]
+    },
+]
+{% endif %}

drydock/patches/kustomization

@@ -5,6 +5,11 @@ patches:
     name: lms
   path: plugins/drydock/k8s/newrelic.yml
 {% endif -%}
+- path: plugins/drydock/k8s/forum-overrides.yml
+{% if DRYDOCK_POD_LIFECYCLE -%}
+- path: plugins/drydock/k8s/lifecycle/lms.yml
+- path: plugins/drydock/k8s/lifecycle/cms.yml
+{% endif -%}
 - target:
     kind: Job
     labelSelector: app.kubernetes.io/component=job
@@ -46,3 +51,7 @@ patches:
       value:
         argocd.argoproj.io/sync-wave: "5"
 {%- endif %}
+{% if DRYDOCK_ENABLE_CELERY_TUNING %}
+- path: plugins/drydock/k8s/celery/cms-worker.yml
+- path: plugins/drydock/k8s/celery/lms-worker.yml
+{% endif -%}

drydock/patches/lms-env

@@ -0,0 +1,22 @@
+{% if DRYDOCK_ENABLE_CELERY_TUNING -%}
+CELERY_ACKS_LATE: true
+{% endif %}
+{% if DRYDOCK_ENABLE_SENTRY -%}
+# eox plugin settings must be defined in the ENV patches. If defined in python settings patches, they won't
+# take effect since plugins settings are loaded before Tutor loads its settings.
+EOX_CORE_SENTRY_INTEGRATION_DSN: "{{ DRYDOCK_SENTRY_DSN }}"
+EOX_CORE_SENTRY_IGNORED_ERRORS: [
+  {
+    "exc_class": "openedx.core.djangoapps.user_authn.exceptions.AuthFailedError",
+    "exc_text": ["AuthFailedError.*Email or password is incorrect"]
+  },
+  {
+    "exc_class": "opaque_keys.InvalidKeyError",
+    "exc_text": [".*No enrollment found for user.*"]
+  },
+  {
+    "exc_class": "opaque_keys.NotFoundError",
+    "exc_text": [".*No course found by course id.*"]
+  },
+]
+{% endif %}

drydock/patches/openedx-common-settings

@@ -0,0 +1,20 @@
+{% if DRYDOCK_ENABLE_SCORM -%}
+# SCORM SETTINGS
+def scorm_xblock_storage(xblock):
+    from django.conf import settings
+    from storages.backends.s3boto3 import S3Boto3Storage
+    if SERVICE_VARIANT == "lms":
+        domain = settings.LMS_BASE
+    else:
+        domain = settings.CMS_BASE
+    return S3Boto3Storage(
+        bucket=AWS_STORAGE_BUCKET_NAME,
+        access_key=AWS_ACCESS_KEY_ID,
+        secret_key=AWS_SECRET_ACCESS_KEY,
+        querystring_expire=86400,
+        custom_domain=f"{domain}/scorm-proxy"
+    )
+XBLOCK_SETTINGS["ScormXBlock"] = {
+    "STORAGE_FUNC": scorm_xblock_storage,
+}
+{% endif %}

drydock/plugin.py

@@ -28,6 +28,12 @@
         "NEWRELIC_LICENSE_KEY": "",
         "CUSTOM_CERTS": {},
         "DEBUG": False,
+        "ENABLE_CELERY_TUNING": True,
+        "ENABLE_MULTITENANCY": True,
+        "ENABLE_SCORM": True,
+        "ENABLE_SENTRY": True,
+        "SENTRY_DSN": "",
+        "POD_LIFECYCLE": True,
     },
     # Add here settings that don't have a reasonable default for all users. For
     # instance: passwords, secret keys, etc.

drydock/templates/drydock/k8s/celery/cms-worker.yml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cms-worker
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 300
+      containers:
+        - name: cms-worker
+          args:
+            - celery
+            - --app=cms.celery
+            - worker
+            - --loglevel=info
+            - --concurrency=1
+            - --hostname=edx.cms.core.default.%%h
+            - --max-tasks-per-child=1
+            - --prefetch-multiplier=1
+            - --exclude-queues=edx.lms.core.default

drydock/templates/drydock/k8s/celery/lms-worker.yml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lms-worker
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 300
+      containers:
+        - name: lms-worker
+          args:
+            - celery
+            - --app=lms.celery
+            - worker
+            - --loglevel=info
+            - --concurrency=1
+            - --hostname=edx.lms.core.default.%%h
+            - --max-tasks-per-child=1
+            - --prefetch-multiplier=1
+            - --exclude-queues=edx.cms.core.default

drydock/templates/drydock/k8s/drydock-jobs/forum.yml

@@ -40,7 +40,7 @@ spec:
           - name: MONGOID_USE_SSL
             value: "{{ 'true' if MONGODB_USE_SSL else 'false' }}"
           - name: MONGOID_AUTH_SOURCE
-            value: "{{MONGODB_AUTH_SOURCE}}"
+            value: "{{FORUM_MONGODB_DATABASE}}"
           - name: MONGOID_AUTH_MECH
             value: "{{ MONGODB_AUTH_MECHANISM|auth_mech_as_ruby }}"
 {%- endif %}

drydock/templates/drydock/k8s/forum-overrides.yml

@@ -0,0 +1,14 @@
+{% if FORUM_DOCKER_IMAGE is defined -%}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: forum
+spec:
+  template:
+    spec:
+      containers:
+        - name: forum
+          env:
+            - name: MONGOID_AUTH_SOURCE
+              value: "{{ FORUM_MONGODB_DATABASE }}"
+{% endif -%}

drydock/templates/drydock/k8s/lifecycle/cms.yml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cms
+spec:
+  template:
+    spec:
+      containers:
+        - name: cms
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - "sleep"
+                - "30"
+          startupProbe:
+            httpGet:
+              httpHeaders:
+                - name: Host
+                  value: {{ CMS_HOST }}
+              path: /heartbeat
+              port: 8000
+            initialDelaySeconds: 5
+            timeoutSeconds: 3
+            periodSeconds: 5
+            failureThreshold: 5
+      terminationGracePeriodSeconds: 60

drydock/templates/drydock/k8s/lifecycle/lms.yml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lms
+spec:
+  template:
+    spec:
+      containers:
+        - name: lms
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - "sleep"
+                - "30"
+          startupProbe:
+            httpGet:
+              httpHeaders:
+                - name: Host
+                  value: {{ LMS_HOST }}
+              path: /heartbeat
+              port: 8000
+            initialDelaySeconds: 5
+            timeoutSeconds: 3
+            periodSeconds: 5
+            failureThreshold: 5
+      terminationGracePeriodSeconds: 60