From d07622f704a487d4bf8040d3ed1ff0b924fa912a Mon Sep 17 00:00:00 2001 From: sshipway Date: Sun, 13 Mar 2016 23:24:17 +0000 Subject: [PATCH 1/4] Add wildcard checks --- Dockerfile | 11 +++---- ImageIdList | 7 +++++ README.md | 19 ++++++++++++ run.sh | 85 +++++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 96 insertions(+), 26 deletions(-) create mode 100644 ImageIdList diff --git a/Dockerfile b/Dockerfile index 5e483c2..874d59a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,11 @@ FROM alpine:latest # run.sh script uses some bash specific syntax RUN apk add --update bash docker grep +ENV CLEAN_PERIOD **None** DELAY_TIME **None** KEEP_IMAGES **None** KEEP_CONTAINERS **None** LOOP true DEBUG 0 + +# Reorder for efficiency # Install cleanup script -ADD run.sh /run.sh ADD docker-cleanup-volumes.sh /docker-cleanup-volumes.sh - -ENV CLEAN_PERIOD **None** -ENV DELAY_TIME **None** -ENV KEEP_IMAGES **None** -ENV KEEP_CONTAINERS **None** -ENV LOOP true +ADD run.sh /run.sh ENTRYPOINT ["/run.sh"] diff --git a/ImageIdList b/ImageIdList new file mode 100644 index 0000000..aebb004 --- /dev/null +++ b/ImageIdList @@ -0,0 +1,7 @@ +162bb99d02d49ce2a8e665cd4f8fafdf15d5d05896bf605d54e27c4b0f5781d5 +407195ab8b072ce8b237f664b8d032704e8047b8e64139cc2e017163f7161647 +879a79efc21c687c00a1e323cdd792ce1a6bd5485f415125f3357851dd62afdb +a0defa9f2430e42806bc52e861acf69ad2bb58059750cde37302d565b41fd522 +a0defa9f2430e42806bc52e861acf69ad2bb58059750cde37302d565b41fd522 +a28cb78b3343b4bcb3ac9279678ed69f11d882a1b0e327b3798d13a8cea19ea1 +d187436706570939342a3cae6b9a30e17921f09dca2cbb5091e3fd16c007870e diff --git a/README.md b/README.md index 2a205b7..6954d68 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,25 @@ The default parameters can be overridden by setting environment variables on the * **KEEP_IMAGES** - List of images to avoid cleaning, e.g. "ubuntu:trusty, ubuntu:latest". Defaults to clean all unused images. * **KEEP_CONTAINERS** - List of images for exited or dead containers to avoid cleaning, e.g. "ubuntu:trusty, ubuntu:latest". * **LOOP** - Add the ability to do non-looped cleanups, run it once and exit. Options are true, false. Defaults to true to run it forever in loops. + * **DEBUG** - Set to 1 to enable more debugging output on pattern matches + +Note that **KEEP_IMAGES** and **KEEP_CONTAINERS** are left-anchored bash shell pattern matching lists (NOT regexps). Therefore, the image **foo/bar:tag** will be matched by ANY of the following: + + * foo/bar:tag + * foo/bar + * foo/b + * [[:alpha:]]/bar + * */*:tag + * *:tag + * foo/*:tag + +However it will not match + + * foo/baz + * bar:tag + * /bar + * :tag + * [[:alpha:]]:tag ## Deployment The image uses the Docker client to to list and remove containers and images. For this reason the Docker client and socket is mapped into the container. diff --git a/run.sh b/run.sh index 33c96d2..b792d69 100755 --- a/run.sh +++ b/run.sh @@ -28,37 +28,64 @@ if [ "${KEEP_IMAGES}" == "**None**" ]; then unset KEEP_IMAGES fi -arr_keep_containers="" -if [ "${KEEP_CONTAINERS}" != "**None**" ]; then - arr_keep_containers=$(echo ${KEEP_CONTAINERS} | tr "," "\n") +if [ "${KEEP_CONTAINERS}" == "**None**" ]; then + unset KEEP_CONTAINERS +fi +if [ "${KEEP_CONTAINERS}" == "**All**" ]; then + KEEP_CONTAINERS="." fi -unset KEEP_CONTAINERS - if [ "${LOOP}" != "false" ]; then LOOP=true fi +if [ "${DEBUG}" == "0" ]; then + unset DEBUG +fi + +if [ $DEBUG ]; then echo DEBUG ENABLED; fi + echo "=> Run the clean script every ${CLEAN_PERIOD} seconds and delay ${DELAY_TIME} seconds to clean." trap '{ echo "User Interupt."; exit 1; }' SIGINT trap '{ echo "SIGTERM received, exiting."; exit 0; }' SIGTERM while [ 1 ] do + if [ $DEBUG ]; then echo DEBUG: Starting loop; fi + # Cleanup unused volumes + echo "=> Removing unused volumes" /docker-cleanup-volumes.sh + IFS=' + ' + # Cleanup exited/dead containers + echo "=> Removing exited/dead containers" EXITED_CONTAINERS_IDS="`docker ps -a -q -f status=exited -f status=dead | xargs echo`" for CONTAINER_ID in $EXITED_CONTAINERS_IDS; do CONTAINER_IMAGE=$(docker inspect --format='{{(index .Config.Image)}}' $CONTAINER_ID) - if [[ ! "${arr_keep_containers[@]}" =~ "${CONTAINER_IMAGE}" ]]; then - echo "Removing container $CONTAINER_ID" + if [ $DEBUG ]; then echo "DEBUG: Check container $CONTAINER_IMAGE"; fi + keepit=0 + if [ -n "${KEEP_CONTAINERS}" ]; then + for PATTERN in $(echo ${KEEP_CONTAINERS} | tr "," "\n"); do + if [[ "${CONTAINER_IMAGE}" = $PATTERN* ]]; then + if [ $DEBUG ]; then echo "DEBUG: Matches $PATTERN - keeping"; fi + keepit=1 + else + if [ $DEBUG ]; then echo "DEBUG: No match for $PATTERN"; fi + fi + done + fi + if [[ $keepit -eq 0 ]]; then + echo "Removing stopped container $CONTAINER_ID" docker rm -v $CONTAINER_ID fi done unset CONTAINER_ID + echo "=> Removing unused images" + # Get all containers in "created" state rm -f CreatedContainerIdList docker ps -a -q -f status=created | sort > CreatedContainerIdList @@ -67,8 +94,6 @@ do ALL_LAYER_NUM=$(docker images -a | tail -n +2 | wc -l) docker images -q --no-trunc | sort -o ImageIdList CONTAINER_ID_LIST=$(docker ps -aq --no-trunc) - IFS=' - ' # Get Image ID that is used by a containter rm -f ContainerImageIdList touch ContainerImageIdList @@ -79,21 +104,42 @@ do done sort ContainerImageIdList -o ContainerImageIdList - # Remove the images being used by cotnainers from the delete list + # Remove the images being used by containers from the delete list comm -23 ImageIdList ContainerImageIdList > ToBeCleanedImageIdList # Remove those reserved images from the delete list if [ -n "${KEEP_IMAGES}" ]; then - rm -f KeepImageIdList - touch KeepImageIdList - arr=$(echo ${KEEP_IMAGES} | tr "," "\n") - for x in $arr - do - docker inspect $x | grep "\"Id\": \"\(sha256:\)\?[0-9a-fA-F]\{64\}\"" | head -1 | awk -F '"' '{print $4}' >> KeepImageIdList + rm -f KeepImageIdList + touch KeepImageIdList + # This looks to see if anything matches the regexp + docker images --no-trunc | ( + while read repo tag image junk; do + keepit=0 + if [ $DEBUG ]; then echo "DEBUG: Check image $repo:$tag"; fi + for PATTERN in $(echo ${KEEP_IMAGES} | tr "," "\n"); do + if [[ -n "$PATTERN" && "${repo}:${tag}" = $PATTERN* ]]; then + if [ $DEBUG ]; then echo "DEBUG: Matches $PATTERN"; fi + keepit=1 + else + if [ $DEBUG ]; then echo "DEBUG: No match for $PATTERN"; fi + fi + done + if [[ $keepit -eq 1 ]]; then + if [ $DEBUG ]; then echo "DEBUG: Marking image $repo:$tag to keep"; fi + echo $image >> KeepImageIdList + fi done - sort KeepImageIdList -o KeepImageIdList - comm -23 ToBeCleanedImageIdList KeepImageIdList > ToBeCleanedImageIdList2 - mv ToBeCleanedImageIdList2 ToBeCleanedImageIdList + ) + # This explicitly looks for the images specified + arr=$(echo ${KEEP_IMAGES} | tr "," "\n") + for x in $arr + do + if [ $DEBUG ]; then echo "DEBUG: Identifying image $x"; fi + docker inspect $x 2>/dev/null| grep "\"Id\": \"\(sha256:\)\?[0-9a-fA-F]\{64\}\"" | head -1 | awk -F '"' '{print $4}' >> KeepImageIdList + done + sort KeepImageIdList -o KeepImageIdList + comm -23 ToBeCleanedImageIdList KeepImageIdList > ToBeCleanedImageIdList2 + mv ToBeCleanedImageIdList2 ToBeCleanedImageIdList fi # Wait before cleaning containers and images @@ -105,6 +151,7 @@ do comm -12 CreatedContainerIdList <(docker ps -a -q -f status=created | sort) > CreatedContainerToClean if [ -s CreatedContainerToClean ]; then echo "=> Start to clean $(cat CreatedContainerToClean | wc -l) created/stuck containers" + if [ $DEBUG ]; then echo "DEBUG: Removing unstarted containers"; fi docker rm -v $(cat CreatedContainerToClean) fi From 7aba3421237df801baf54e4af2c03c6f42ff9de5 Mon Sep 17 00:00:00 2001 From: sshipway Date: Sun, 13 Mar 2016 23:27:30 +0000 Subject: [PATCH 2/4] Fix documentation --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6954d68..995f2aa 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ Note that **KEEP_IMAGES** and **KEEP_CONTAINERS** are left-anchored bash shell p * foo/bar * foo/b * [[:alpha:]]/bar - * */*:tag - * *:tag - * foo/*:tag + * \*/\*:tag + * \*:tag + * foo/\*:tag However it will not match From b850c0c3fb7ca7e68c15bf44af200fbec0d1e30e Mon Sep 17 00:00:00 2001 From: sshipway Date: Sun, 13 Mar 2016 23:28:41 +0000 Subject: [PATCH 3/4] Remove temporary file --- ImageIdList | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 ImageIdList diff --git a/ImageIdList b/ImageIdList deleted file mode 100644 index aebb004..0000000 --- a/ImageIdList +++ /dev/null @@ -1,7 +0,0 @@ -162bb99d02d49ce2a8e665cd4f8fafdf15d5d05896bf605d54e27c4b0f5781d5 -407195ab8b072ce8b237f664b8d032704e8047b8e64139cc2e017163f7161647 -879a79efc21c687c00a1e323cdd792ce1a6bd5485f415125f3357851dd62afdb -a0defa9f2430e42806bc52e861acf69ad2bb58059750cde37302d565b41fd522 -a0defa9f2430e42806bc52e861acf69ad2bb58059750cde37302d565b41fd522 -a28cb78b3343b4bcb3ac9279678ed69f11d882a1b0e327b3798d13a8cea19ea1 -d187436706570939342a3cae6b9a30e17921f09dca2cbb5091e3fd16c007870e From 873fd1c4e206e8b6738f645ba9188279232dcb99 Mon Sep 17 00:00:00 2001 From: sshipway Date: Sun, 13 Mar 2016 23:38:49 +0000 Subject: [PATCH 4/4] Update documentation on wildcards --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 995f2aa..4b2a7f0 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,10 @@ However it will not match * :tag * [[:alpha:]]:tag +By default, both are set to **\*\*None\*\*** which is the same as the blank string. If you want to keep ALL images or containers, effectively disabling this +part of the cleanup, then you should use **\*:\*** to match all images. Do not +use a bare **\*** as this will be taken as a filename match. + ## Deployment The image uses the Docker client to to list and remove containers and images. For this reason the Docker client and socket is mapped into the container.