diff --git a/airbyte-cron/Dockerfile b/airbyte-cron/Dockerfile new file mode 100644 index 000000000000..01398125e71b --- /dev/null +++ b/airbyte-cron/Dockerfile @@ -0,0 +1,16 @@ +ARG JDK_VERSION=19-slim-bullseye +ARG JDK_IMAGE=openjdk:${JDK_VERSION} +FROM ${JDK_IMAGE} AS cron + +ARG VERSION=0.40.0-alpha + +ENV APPLICATION airbyte-cron +ENV VERSION ${VERSION} + +WORKDIR /app + +# This is automatically unzipped by Docker +ADD bin/${APPLICATION}-${VERSION}.tar /app + +# wait for upstream dependencies to become available before starting server +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"] diff --git a/airbyte-cron/LICENSE b/airbyte-cron/LICENSE new file mode 100644 index 000000000000..387ec9606473 --- /dev/null +++ b/airbyte-cron/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Airbyte, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/airbyte-cron/build.gradle b/airbyte-cron/build.gradle new file mode 100644 index 000000000000..df9d61e10154 --- /dev/null +++ b/airbyte-cron/build.gradle @@ -0,0 +1,36 @@ +plugins { + id 'application' +} + +dependencies { + implementation project(':airbyte-workers') + + runtimeOnly 'io.micronaut:micronaut-http-server-netty:3.6.0' + + annotationProcessor platform(libs.micronaut.bom) + annotationProcessor libs.bundles.micronaut.annotation.processor + + implementation platform(libs.micronaut.bom) + implementation libs.bundles.micronaut +} + +mainClassName = 'io.airbyte.cron.MicronautCronRunner' + +application { + mainClass = mainClassName + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +} + +task copyGeneratedTar(type: Copy) { + dependsOn copyDocker + dependsOn distTar + + from('build/distributions') { + include 'airbyte-cron-*.tar' + } + into 'build/docker/bin' +} + +Task dockerBuildTask = getDockerBuildTask("cron", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") +dockerBuildTask.dependsOn(copyGeneratedTar) +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/MicronautCronRunner.java b/airbyte-cron/src/main/java/io/airbyte/cron/MicronautCronRunner.java new file mode 100644 index 000000000000..408d310f3783 --- /dev/null +++ b/airbyte-cron/src/main/java/io/airbyte/cron/MicronautCronRunner.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cron; + +import io.micronaut.runtime.Micronaut; + +/** + * Micronaut server responsible of running scheduled method. The methods need to be separated in + * Bean based on what they are cleaning and contain a method annotated with `@Scheduled` + * + * Injected object looks unused but they are not + */ +public class MicronautCronRunner { + + public static void main(final String[] args) { + Micronaut.build(args) + .eagerInitSingletons(true) + .mainClass(MicronautCronRunner.class) + .start(); + } + +} diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/Temporal.java b/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/Temporal.java new file mode 100644 index 000000000000..38787b128a16 --- /dev/null +++ b/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/Temporal.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cron.selfhealing; + +import io.micronaut.scheduling.annotation.Scheduled; +import javax.inject.Singleton; +import lombok.extern.slf4j.Slf4j; + +@Singleton +@Slf4j +public class Temporal { + + public Temporal() { + log.info("Creating temporal self-healing"); + } + + @Scheduled(fixedRate = "10s") + void cleanTemporal() {} + +} diff --git a/docker-compose.yaml b/docker-compose.yaml index 61502aa9a1c6..d5d8a2b4d31c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -174,6 +174,18 @@ services: - POSTGRES_USER=${DATABASE_USER} volumes: - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig + airbyte-cron: + image: airbyte/cron:${VERSION} + logging: *default-logging + container_name: airbyte-cron + restart: unless-stopped + environment: + - DB=postgresql + - DB_PORT=${DATABASE_PORT} + - LOG_LEVEL=${LOG_LEVEL} + - POSTGRES_PWD=${DATABASE_PASSWORD} + - POSTGRES_SEEDS=${DATABASE_HOST} + - POSTGRES_USER=${DATABASE_USER} volumes: workspace: name: ${WORKSPACE_DOCKER_MOUNT} diff --git a/kube/overlays/dev-integration-test/kustomization.yaml b/kube/overlays/dev-integration-test/kustomization.yaml index 76c36a1d02e7..e2add7588d9f 100644 --- a/kube/overlays/dev-integration-test/kustomization.yaml +++ b/kube/overlays/dev-integration-test/kustomization.yaml @@ -19,6 +19,8 @@ images: newTag: dev - name: temporalio/auto-setup newTag: 1.7.0 + - name: airbyte/cron + newTag: dev configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/dev/kustomization.yaml b/kube/overlays/dev/kustomization.yaml index d6525f134e5d..fb29ae238581 100644 --- a/kube/overlays/dev/kustomization.yaml +++ b/kube/overlays/dev/kustomization.yaml @@ -19,6 +19,8 @@ images: newTag: dev - name: temporalio/auto-setup newTag: 1.7.0 + - name: airbyte/cron + newTag: dev configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index 39fa4120e5e2..af7692e7f07b 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -19,6 +19,8 @@ images: newTag: 0.40.0-alpha - name: temporalio/auto-setup newTag: 1.7.0 + - name: airbyte/cron + newTag: 0.40.0-alpha configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml b/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml index 09d50ab9baef..b26bbe788387 100644 --- a/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml +++ b/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml @@ -107,3 +107,17 @@ spec: limits: cpu: 0.5 memory: 128Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airbyte-cron +spec: + template: + spec: + containers: + - name: airbyte-cron + resources: + limits: + cpu: 0.5 + memory: 128Mi diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index cb81f22e32ee..468e91b8c895 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -19,6 +19,8 @@ images: newTag: 0.40.0-alpha - name: temporalio/auto-setup newTag: 1.7.0 + - name: airbyte/cron + newTag: 0.40.0-alpha configMapGenerator: - name: airbyte-env diff --git a/kube/resources/cron.yaml b/kube/resources/cron.yaml new file mode 100644 index 000000000000..affc91ecba49 --- /dev/null +++ b/kube/resources/cron.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airbyte-cron +spec: + replicas: 1 + selector: + matchLabels: + airbyte: cron + template: + metadata: + labels: + airbyte: cron + spec: + containers: + - name: airbyte-cron-container + image: airbyte/cron + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: DATABASE_USER + - name: POSTGRES_PWD + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: DATABASE_PASSWORD + - name: DB + value: "postgresql" + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_PORT + - name: POSTGRES_SEEDS + valueFrom: + configMapKeyRef: + name: airbyte-env + key: DATABASE_HOST + volumeMounts: + - name: airbyte-volume-configs + mountPath: /configs + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + volumes: + - name: airbyte-volume-configs + persistentVolumeClaim: + claimName: airbyte-volume-configs + - name: gcs-log-creds-volume + secret: + secretName: gcs-log-creds diff --git a/kube/resources/kustomization.yaml b/kube/resources/kustomization.yaml index 8752d16cdc8c..150074bcefa2 100644 --- a/kube/resources/kustomization.yaml +++ b/kube/resources/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - airbyte-minio.yaml - bootloader.yaml + - cron.yaml - db.yaml - pod-sweeper.yaml - secret-gcs-log-creds.yaml diff --git a/settings.gradle b/settings.gradle index 639b31d81880..20fc7bc37c0c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -77,6 +77,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD" include ':airbyte-tests' include ':airbyte-webapp' include ':airbyte-webapp-e2e-tests' + include ':airbyte-cron' } // connectors base