Add orphan image support

README.md

@@ -10,7 +10,12 @@
 
 ## General
 
-The image cleanup client is used to clean up Docker images in a Docker Registry when they are tagged using git SHA. The cleaning is done either using git commit hashes or tags. Defaults to hashes otherwise ```-t``` flag should be used.
+The image cleanup client is used to clean up Docker images in a Docker Registry when they are tagged using git SHA. The cleaning is done either using git commit hashes or tags. Defaults to hashes otherwise ```-t``` flag should be used. The tool also allows to clean orphan image stream tags using ```-o``` flag, the orphan image stream tags are images that do not have any Git commit/tag. There are secondary flags which help to norrow the cleaning process such as:
+* ```-i``` regex flag that is used to norrow the orphans to be deleted, works exclusively with ```-o``` flag
+* ```l``` limit the number of Git commits/tags to be fetched from Git repository, prohibited with ```-o``` flag
+* ```-k``` keeps most current n images
+* ```--older-than``` considers images that are older than the last update date.
+* ```--sort``` sorts the Git Tags, works exclusively with the ```-t``` flag
 
 This helps to save space because obsolete images are being removed from the registry.
 

cmd/imagestream.go

@@ -2,12 +2,15 @@ package cmd
 
 import (
 	log "github.com/sirupsen/logrus"
+	"time"
+	"regexp"
 
 	"github.com/appuio/image-cleanup/pkg/cleanup"
 	"github.com/appuio/image-cleanup/pkg/git"
 	"github.com/appuio/image-cleanup/pkg/kubernetes"
 	"github.com/appuio/image-cleanup/pkg/openshift"
 	"github.com/spf13/cobra"
+	"github.com/karrick/tparse"
 )
 
 // ImageStreamCleanupOptions is a struct to support the cleanup command
@@ -20,6 +23,9 @@ type ImageStreamCleanupOptions struct {
 	Namespace   string
 	Tag         bool
 	Sorted      string
+	Orphan      bool
+	Duration    string
+	Regex       string
 }
 
 // NewImageStreamCleanupCommand creates a cobra command to clean up an imagestream based on commits
@@ -33,12 +39,15 @@ func NewImageStreamCleanupCommand() *cobra.Command {
 		Run:     o.cleanupImageStreamTags,
 	}
 	cmd.Flags().BoolVarP(&o.Force, "force", "f", false, "delete image stream tags")
-	cmd.Flags().IntVarP(&o.CommitLimit, "git-commit-limit", "l", 100, "only look at the first <n> commits to compare with tags or use -1 for all commits")
+	cmd.Flags().IntVarP(&o.CommitLimit, "git-commit-limit", "l", 0, "only look at the first <n> commits to compare with tags or use 0 for all commits")
 	cmd.Flags().StringVarP(&o.RepoPath, "git-repo-path", "p", ".", "absolute path to Git repository (for current dir use: $PWD)")
 	cmd.Flags().StringVarP(&o.Namespace, "namespace", "n", "", "Kubernetes namespace")
 	cmd.Flags().IntVarP(&o.Keep, "keep", "k", 10, "keep most current <n> images")
 	cmd.Flags().BoolVarP(&o.Tag, "tag", "t", false, "use tags instead of commit hashes")
 	cmd.Flags().StringVar(&o.Sorted, "sort", string(git.SortOptionVersion), "sort tags by criteria. Allowed values: [version, alphabetical]")
+	cmd.Flags().BoolVarP(&o.Orphan, "orphan", "o", false, "delete images that does not match any git commit")
+	cmd.Flags().StringVar(&o.Duration, "older-than", "", "delete images that are older than the duration. Ex.: [1y2mo3w4d5h6m7s]")
+	cmd.Flags().StringVarP(&o.Regex, "include", "i", "^[a-z0-9]{40}$", "delete images that match the regex, works only with the -o flag, defaults to matching Git SHA commits")
 	return cmd
 }
 
@@ -47,8 +56,36 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 		o.ImageStream = args[0]
 	}
 
+	if o.Orphan == false && o.Regex != "^[a-z0-9]{40}$" {
+		log.WithFields(log.Fields{"Orphan": o.Orphan, "Regex": o.Regex}).
+			Fatal("Missing Orphan flag")
+	}
+
+	regexp, err := regexp.Compile(o.Regex)
+	if err != nil {
+		log.WithField("regex", o.Regex).Fatal("Invalid regex.")
+	}
+
 	if o.Tag && !git.IsValidSortValue(o.Sorted) {
-		log.WithField("sort_criteria", o.Sorted).Fatal("Invalid sort criteria")
+		log.WithField("sort_criteria", o.Sorted).Fatal("Invalid sort criteria.")
+	}
+
+	if o.CommitLimit !=0 && o.Orphan == true {
+		log.WithFields(log.Fields{"CommitLimit": o.CommitLimit, "Orphan": o.Orphan}).
+			Fatal("Mutually exclusive flags")
+	}
+
+	var duration time.Time
+	if len(o.Duration) > 0 {
+		var err error
+		duration, err = tparse.ParseNow(time.RFC3339, "now-" + o.Duration)
+		if err != nil {
+			log.WithError(err).
+				WithField("duration", o.Duration).
+				Fatal("Could not parse duration.")
+		}
+	} else {
+		duration = time.Now()
 	}
 
 	if len(o.Namespace) == 0 {
@@ -96,7 +133,7 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 		}
 	}
 
-	imageStreamTags, err := openshift.GetImageStreamTags(o.Namespace, o.ImageStream)
+	imageStreamObjectTags, err := openshift.GetImageStreamTags(o.Namespace, o.ImageStream)
 	if err != nil {
 		log.WithError(err).
 			WithFields(log.Fields{
@@ -105,12 +142,20 @@ func (o *ImageStreamCleanupOptions) cleanupImageStreamTags(cmd *cobra.Command, a
 			Fatal("Could not retrieve image stream.")
 	}
 
+	imageStreamTags := cleanup.TagsOlderThan(&imageStreamObjectTags, duration)
+
 	var matchOption cleanup.MatchOption
 	if o.Tag {
 		matchOption = cleanup.MatchOptionExact
 	}
 
-	matchingTags := cleanup.GetMatchingTags(&matchValues, &imageStreamTags, matchOption)
+	var matchingTags []string
+	if o.Orphan {
+		matchingTags = cleanup.GetOrphanTags(&matchValues, &imageStreamTags, matchOption)
+		matchingTags = cleanup.FilterByRegex(&imageStreamTags, regexp)
+	} else {
+		matchingTags = cleanup.GetMatchingTags(&matchValues, &imageStreamTags, matchOption)
+	}
 
 	activeImageStreamTags, err := openshift.GetActiveImageStreamTags(o.Namespace, o.ImageStream, imageStreamTags)
 	if err != nil {

go.mod

@@ -9,7 +9,8 @@ require (
 	github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/json-iterator/go v1.1.8 // indirect
-	github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible // indirect
+	github.com/karrick/tparse v2.4.2+incompatible
+	github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible
 	github.com/openshift/client-go v0.0.0-20180830153425-431ec9a26e50
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -101,6 +101,8 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/karrick/tparse v2.4.2+incompatible h1:+cW306qKAzrASC5XieHkgN7/vPaGKIuK62Q7nI7DIRc=
+github.com/karrick/tparse v2.4.2+incompatible/go.mod h1:ASPA+vrIcN1uEW6BZg8vfWbzm69ODPSYZPU6qJyfdK0=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
@@ -146,6 +148,7 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
 github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible h1:p0ypM7AY7k2VY6ILDPbg3LajGA97hFUt2DGVEQz2Yd4=
 github.com/openshift/api v3.9.1-0.20190322043348-8741ff068a47+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
 github.com/openshift/client-go v0.0.0-20180830153425-431ec9a26e50 h1:y59/+XbTbwzEdS2wveRQTZvjkar7sbVjTNnqFBufr74=

pkg/cleanup/cleanup.go

@@ -2,8 +2,11 @@ package cleanup
 
 import (
 	"strings"
+	"regexp"
+	"time"
 
 	log "github.com/sirupsen/logrus"
+	imagev1 "github.com/openshift/api/image/v1"
 )
 
 // MatchOption type defines how the tags should be matched
@@ -62,6 +65,40 @@ func GetInactiveTags(activeTags, tags *[]string) []string {
 	return inactiveTags
 }
 
+// GetOrphanTags returns the tags that does not have any git commit match
+func GetOrphanTags(values, tags *[]string, matchOption MatchOption) []string {
+	var orphans []string
+
+	log.WithField("values", values).Debug("values")
+	log.WithField("tags", tags).Debug("tags")
+
+	for _, tag := range *tags {
+		orphans = append(orphans, tag)
+		for _, value := range *values {
+			if match(tag, value, matchOption) {
+				orphans = orphans[:len(orphans)-1]
+				break
+			}
+		}
+	}
+
+	return orphans
+}
+
+// FilterByRegex returns the tags that match the regexp
+func FilterByRegex(tags *[]string, regexp *regexp.Regexp) []string {
+	var matchedTags []string
+
+	log.WithField("tags", tags).Debug("tags")
+
+	for _, tag := range *tags {
+		if regexp.MatchString(tag) {
+			matchedTags = append(matchedTags, tag)
+		}
+	}
+	return matchedTags
+}
+
 // LimitTags returns the tags which should not be kept by removing the first n tags
 func LimitTags(tags *[]string, keep int) []string {
 	if len(*tags) > keep {
@@ -73,6 +110,26 @@ func LimitTags(tags *[]string, keep int) []string {
 	return []string{}
 }
 
+// TagsOlderThan returns the tags which are older than the specified time
+func TagsOlderThan(imageStreamObjectTags *[]imagev1.NamedTagEventList, olderThan time.Time) []string {
+	var imageStreamTags []string
+
+	for _, imageStreamTag := range *imageStreamObjectTags {
+		lastUpdatedDate := imageStreamTag.Items[0].Created.Time
+		for _, tagEvent := range imageStreamTag.Items {
+			if lastUpdatedDate.Before(tagEvent.Created.Time) {
+				lastUpdatedDate = tagEvent.Created.Time
+			}
+		}
+
+		if lastUpdatedDate.Before(olderThan) {
+			imageStreamTags = append(imageStreamTags, imageStreamTag.Tag)
+		}
+	}
+
+	return imageStreamTags
+}
+
 func match(tag, value string, matchOption MatchOption) bool {
 	switch matchOption {
 	case MatchOptionDefault, MatchOptionPrefix: