Add scheduled service

airbyte-cron/Dockerfile

@@ -0,0 +1,16 @@
+ARG JDK_VERSION=19-slim-bullseye
+ARG JDK_IMAGE=openjdk:${JDK_VERSION}
+FROM ${JDK_IMAGE} AS cron
+
+ARG VERSION=0.40.0-alpha
+
+ENV APPLICATION airbyte-cron
+ENV VERSION ${VERSION}
+
+WORKDIR /app
+
+# This is automatically unzipped by Docker
+ADD bin/${APPLICATION}-${VERSION}.tar /app
+
+# wait for upstream dependencies to become available before starting server
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-cron/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Airbyte, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

airbyte-cron/build.gradle

@@ -0,0 +1,36 @@
+plugins {
+    id 'application'
+}
+
+dependencies {
+    implementation project(':airbyte-workers')
+
+    runtimeOnly 'io.micronaut:micronaut-http-server-netty:3.6.0'
+
+    annotationProcessor platform(libs.micronaut.bom)
+    annotationProcessor libs.bundles.micronaut.annotation.processor
+
+    implementation platform(libs.micronaut.bom)
+    implementation libs.bundles.micronaut
+}
+
+mainClassName = 'io.airbyte.cron.MicronautCronRunner'
+
+application {
+    mainClass = mainClassName
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+}
+
+task copyGeneratedTar(type: Copy) {
+    dependsOn copyDocker
+    dependsOn distTar
+
+    from('build/distributions') {
+        include 'airbyte-cron-*.tar'
+    }
+    into 'build/docker/bin'
+}
+
+Task dockerBuildTask = getDockerBuildTask("cron", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
+dockerBuildTask.dependsOn(copyGeneratedTar)
+assemble.dependsOn(dockerBuildTask)

airbyte-cron/src/main/java/io/airbyte/cron/MicronautCronRunner.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.cron;
+
+import io.micronaut.runtime.Micronaut;
+
+/**
+ * Micronaut server responsible of running scheduled method. The methods need to be separated in
+ * Bean based on what they are cleaning and contain a method annotated with `@Scheduled`
+ *
+ * Injected object looks unused but they are not
+ */
+public class MicronautCronRunner {
+
+  public static void main(final String[] args) {
+    Micronaut.build(args)
+        .eagerInitSingletons(true)
+        .mainClass(MicronautCronRunner.class)
+        .start();
+  }
+
+}

airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/Temporal.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.cron.selfhealing;
+
+import io.micronaut.scheduling.annotation.Scheduled;
+import javax.inject.Singleton;
+import lombok.extern.slf4j.Slf4j;
+
+@Singleton
+@Slf4j
+public class Temporal {
+
+  public Temporal() {
+    log.info("Creating temporal self-healing");
+  }
+
+  @Scheduled(fixedRate = "10s")
+  void cleanTemporal() {}
+
+}

docker-compose.yaml

@@ -174,6 +174,18 @@ services:
       - POSTGRES_USER=${DATABASE_USER}
     volumes:
       - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
+  airbyte-cron:
+    image: airbyte/cron:${VERSION}
+    logging: *default-logging
+    container_name: airbyte-cron
+    restart: unless-stopped
+    environment:
+      - DB=postgresql
+      - DB_PORT=${DATABASE_PORT}
+      - LOG_LEVEL=${LOG_LEVEL}
+      - POSTGRES_PWD=${DATABASE_PASSWORD}
+      - POSTGRES_SEEDS=${DATABASE_HOST}
+      - POSTGRES_USER=${DATABASE_USER}
 volumes:
   workspace:
     name: ${WORKSPACE_DOCKER_MOUNT}

kube/overlays/dev-integration-test/kustomization.yaml

@@ -19,6 +19,8 @@ images:
     newTag: dev
   - name: temporalio/auto-setup
     newTag: 1.7.0
+  - name: airbyte/cron
+    newTag: dev
 
 configMapGenerator:
   - name: airbyte-env

kube/overlays/dev/kustomization.yaml

@@ -19,6 +19,8 @@ images:
     newTag: dev
   - name: temporalio/auto-setup
     newTag: 1.7.0
+  - name: airbyte/cron
+    newTag: dev
 
 configMapGenerator:
   - name: airbyte-env

kube/overlays/stable-with-resource-limits/kustomization.yaml

@@ -19,6 +19,8 @@ images:
     newTag: 0.40.0-alpha
   - name: temporalio/auto-setup
     newTag: 1.7.0
+  - name: airbyte/cron
+    newTag: 0.40.0-alpha
 
 configMapGenerator:
   - name: airbyte-env

kube/overlays/stable-with-resource-limits/set-resource-limits.yaml

@@ -107,3 +107,17 @@ spec:
             limits:
               cpu: 0.5
               memory: 128Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-cron
+spec:
+  template:
+    spec:
+      containers:
+        - name: airbyte-cron
+          resources:
+            limits:
+              cpu: 0.5
+              memory: 128Mi

kube/overlays/stable/kustomization.yaml

@@ -19,6 +19,8 @@ images:
     newTag: 0.40.0-alpha
   - name: temporalio/auto-setup
     newTag: 1.7.0
+  - name: airbyte/cron
+    newTag: 0.40.0-alpha
 
 configMapGenerator:
   - name: airbyte-env

kube/resources/cron.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: airbyte-cron
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      airbyte: cron
+  template:
+    metadata:
+      labels:
+        airbyte: cron
+    spec:
+      containers:
+        - name: airbyte-cron-container
+          image: airbyte/cron
+          env:
+            - name: POSTGRES_USER
+              valueFrom:
+                secretKeyRef:
+                  name: airbyte-secrets
+                  key: DATABASE_USER
+            - name: POSTGRES_PWD
+              valueFrom:
+                secretKeyRef:
+                  name: airbyte-secrets
+                  key: DATABASE_PASSWORD
+            - name: DB
+              value: "postgresql"
+            - name: DB_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: DATABASE_PORT
+            - name: POSTGRES_SEEDS
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: DATABASE_HOST
+          volumeMounts:
+            - name: airbyte-volume-configs
+              mountPath: /configs
+            - name: gcs-log-creds-volume
+              mountPath: /secrets/gcs-log-creds
+              readOnly: true
+      volumes:
+        - name: airbyte-volume-configs
+          persistentVolumeClaim:
+            claimName: airbyte-volume-configs
+        - name: gcs-log-creds-volume
+          secret:
+            secretName: gcs-log-creds

kube/resources/kustomization.yaml

@@ -4,6 +4,7 @@ kind: Kustomization
 resources:
   - airbyte-minio.yaml
   - bootloader.yaml
+  - cron.yaml
   - db.yaml
   - pod-sweeper.yaml
   - secret-gcs-log-creds.yaml

settings.gradle

@@ -77,6 +77,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
     include ':airbyte-tests'
     include ':airbyte-webapp'
     include ':airbyte-webapp-e2e-tests'
+    include ':airbyte-cron'
 }
 
 // connectors base