From fbc93faa502a12f4b39e9b2c42ff5663470bc15c Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 09:47:54 -0800 Subject: [PATCH 1/8] Run a cronJob every 30m to cleanup docker buildkit cache https://github.com/docker/buildx/issues/1065#issuecomment-1098357043 is the best documentation on this command, and it should fully be able to replace what our imageCleaner currently does. And it's necessary, as I think imageCleaner doesn't actually clean up buildkit cache anymore. This should be upstreamed into the helm chart, and imageCleaner deprecated / removed. However, I want to merge https://github.com/2i2c-org/binderhub-service/ into binderhub and turn that into the chart first, so we don't have to keep repeating work there. So in the meantime, this cronjob is here so we don't have mybinder.org outages (as happened in https://jupyter.zulipchat.com/#narrow/channel/469744-jupyterhub/topic/mybinder.2Eorg.20outage) --- mybinder/templates/buildkit-pruner.yaml | 31 +++++++++++++++++++++++++ mybinder/values.yaml | 9 ++++++- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 mybinder/templates/buildkit-pruner.yaml diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml new file mode 100644 index 000000000..23037aba1 --- /dev/null +++ b/mybinder/templates/buildkit-pruner.yaml @@ -0,0 +1,31 @@ +{{ if .Values.buildkitPruner.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hello +spec: + schedule: {{ .Values.buildkitPruner.schedule | quote }} + # If the previous run is still going, replace it with our new one + concurrencyPolicy: Replace + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + volumes: + - name: dind-socket + hostPath: + path: /var/run/dind/docker.sock + containers: + - name: pruner + image: {{ .Values.buildkitPruner.image }} + cmd: + - docker + - builder + - prune + - --all + - --keep-storage={{ .Values.buildkitPruner.buildkitCacheSize }} + volumeMounts: + - name: dind-socket + mountPath: /var/run/dind/docker.sock +{{- end }} \ No newline at end of file diff --git a/mybinder/values.yaml b/mybinder/values.yaml index 06128fb5b..cb9a95c27 100644 --- a/mybinder/values.yaml +++ b/mybinder/values.yaml @@ -1,4 +1,3 @@ -# values ref: https://github.com/yuvipanda/cryptnono/blob/main/cryptnono/values.yaml cryptnono: enabled: true detectors: @@ -10,6 +9,14 @@ cryptnono: containerdHostPath: /run/containerd/containerd.sock dockerHostPath: /run/dind/docker.sock +buildkitPruner: + enabled: true + # Use the same image as we use for dind + image: docker:27.5.1-dind + buildkitCacheSize: 300G + # Run this every 30min + schedule: "*/30 * * * *" + registry: enabled: false replicas: 2 From 4160f8790c7a648c43d8ab5e39d3056b61659834 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 09:57:43 -0800 Subject: [PATCH 2/8] Fix typo --- mybinder/templates/buildkit-pruner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml index 23037aba1..5d0042af8 100644 --- a/mybinder/templates/buildkit-pruner.yaml +++ b/mybinder/templates/buildkit-pruner.yaml @@ -19,7 +19,7 @@ spec: containers: - name: pruner image: {{ .Values.buildkitPruner.image }} - cmd: + command: - docker - builder - prune From 2c4e9bfcea17f495c9c4c1d38d3c3533447b9179 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 16 Feb 2025 17:58:45 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mybinder/templates/buildkit-pruner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml index 5d0042af8..0ce87342c 100644 --- a/mybinder/templates/buildkit-pruner.yaml +++ b/mybinder/templates/buildkit-pruner.yaml @@ -28,4 +28,4 @@ spec: volumeMounts: - name: dind-socket mountPath: /var/run/dind/docker.sock -{{- end }} \ No newline at end of file +{{- end }} From e09a70f1aadb9895a5f305e6b6f8037c2bb8d624 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 10:15:30 -0800 Subject: [PATCH 4/8] Update pod specification of image pruner --- mybinder/templates/buildkit-pruner.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml index 0ce87342c..a4631ccaf 100644 --- a/mybinder/templates/buildkit-pruner.yaml +++ b/mybinder/templates/buildkit-pruner.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: hello + name: build-cache-prune spec: schedule: {{ .Values.buildkitPruner.schedule | quote }} # If the previous run is still going, replace it with our new one @@ -23,8 +23,9 @@ spec: - docker - builder - prune + - --force - --all - - --keep-storage={{ .Values.buildkitPruner.buildkitCacheSize }} + - --max-storage={{ .Values.buildkitPruner.buildkitCacheSize }} volumeMounts: - name: dind-socket mountPath: /var/run/dind/docker.sock From 10712c11557486fd50717f5797098e3b70f0946b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 11:07:30 -0800 Subject: [PATCH 5/8] Specify DOCKER_HOST for pruner --- mybinder/templates/buildkit-pruner.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml index a4631ccaf..fc3f6b66c 100644 --- a/mybinder/templates/buildkit-pruner.yaml +++ b/mybinder/templates/buildkit-pruner.yaml @@ -19,6 +19,9 @@ spec: containers: - name: pruner image: {{ .Values.buildkitPruner.image }} + env: + - name: DOCKER_HOST + value: "unix:///var/run/dind/docker.sock" command: - docker - builder From e6638b36f7f4825964ed70b441f13d30bbf6e7e5 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 11:07:40 -0800 Subject: [PATCH 6/8] Use --keep-storage flag Apparently the error message is wrong and --max-storage is not actually implemented lol --- mybinder/templates/buildkit-pruner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml index fc3f6b66c..f9a2f839e 100644 --- a/mybinder/templates/buildkit-pruner.yaml +++ b/mybinder/templates/buildkit-pruner.yaml @@ -28,7 +28,7 @@ spec: - prune - --force - --all - - --max-storage={{ .Values.buildkitPruner.buildkitCacheSize }} + - --keep-storage={{ .Values.buildkitPruner.buildkitCacheSize }} volumeMounts: - name: dind-socket mountPath: /var/run/dind/docker.sock From f1df40fbdf3f3b8077b813d7efb90f7bafdb25bc Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 19:11:35 -0800 Subject: [PATCH 7/8] Print output of docker system df after running cleaner --- mybinder/templates/buildkit-pruner.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mybinder/templates/buildkit-pruner.yaml b/mybinder/templates/buildkit-pruner.yaml index f9a2f839e..de016a6f8 100644 --- a/mybinder/templates/buildkit-pruner.yaml +++ b/mybinder/templates/buildkit-pruner.yaml @@ -23,12 +23,11 @@ spec: - name: DOCKER_HOST value: "unix:///var/run/dind/docker.sock" command: - - docker - - builder - - prune - - --force - - --all - - --keep-storage={{ .Values.buildkitPruner.buildkitCacheSize }} + - /bin/sh + - -c + - | + docker builder prune --force --all --keep-storage={{ .Values.buildkitPruner.buildkitCacheSize }} && \ + docker system df volumeMounts: - name: dind-socket mountPath: /var/run/dind/docker.sock From 014e118d3ef866b193f9c7300cf6dc2ee4973320 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 16 Feb 2025 19:12:22 -0800 Subject: [PATCH 8/8] Use full unit for buildkitCacheSize --- mybinder/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mybinder/values.yaml b/mybinder/values.yaml index cb9a95c27..387db72ac 100644 --- a/mybinder/values.yaml +++ b/mybinder/values.yaml @@ -13,7 +13,7 @@ buildkitPruner: enabled: true # Use the same image as we use for dind image: docker:27.5.1-dind - buildkitCacheSize: 300G + buildkitCacheSize: 300GB # Run this every 30min schedule: "*/30 * * * *"