Skip to content

Commit

Permalink
Add minimum age check for pod candidates
Browse files Browse the repository at this point in the history
  • Loading branch information
Brian Akins committed Jul 24, 2018
1 parent d5a34a9 commit f242fad
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 3 deletions.
27 changes: 26 additions & 1 deletion chaoskube/chaoskube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ type Chaoskube struct {
ExcludedDaysOfYear []time.Time
// the timezone to apply when detecting the current weekday
Timezone *time.Location
// minimum age of pods to consider
MinimumAge time.Duration
// an instance of logrus.StdLogger to write log messages to
Logger log.FieldLogger
// dry run will not allow any pod terminations
Expand Down Expand Up @@ -63,7 +65,7 @@ var (
// * a time zone to apply to the aforementioned time-based filters
// * a logger implementing logrus.FieldLogger to send log output to
// * whether to enable/disable dry-run mode
func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, logger log.FieldLogger, dryRun bool) *Chaoskube {
func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool) *Chaoskube {
return &Chaoskube{
Client: client,
Labels: labels,
Expand All @@ -73,6 +75,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel
ExcludedTimesOfDay: excludedTimesOfDay,
ExcludedDaysOfYear: excludedDaysOfYear,
Timezone: timezone,
MinimumAge: minimumAge,
Logger: logger,
DryRun: dryRun,
Now: time.Now,
Expand Down Expand Up @@ -154,6 +157,8 @@ func (c *Chaoskube) Candidates() ([]v1.Pod, error) {
pods = filterByAnnotations(pods, c.Annotations)
pods = filterByPhase(pods, v1.PodRunning)

pods = filterByMinimumAge(pods, c.MinimumAge, c.Now())

return pods, nil
}

Expand Down Expand Up @@ -262,3 +267,23 @@ func filterByPhase(pods []v1.Pod, phase v1.PodPhase) []v1.Pod {

return filteredList
}

// filterByMinimumAge filters pods by creation time. Only pods
// older than minimumAge are returned
func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time) []v1.Pod {
if minimumAge <= time.Duration(0) {
return pods
}

creationTime := now.Add(-minimumAge)

filteredList := []v1.Pod{}

for _, pod := range pods {
if pod.ObjectMeta.CreationTimestamp.Time.Before(creationTime) {
filteredList = append(filteredList, pod)
}
}

return filteredList
}
116 changes: 114 additions & 2 deletions chaoskube/chaoskube_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/sirupsen/logrus/hooks/test"

"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes/fake"

Expand Down Expand Up @@ -40,6 +41,7 @@ func (suite *Suite) TestNew() {
excludedWeekdays = []time.Weekday{time.Friday}
excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}}
excludedDaysOfYear = []time.Time{time.Now()}
minimumAge = time.Duration(42)
)

chaoskube := New(
Expand All @@ -51,6 +53,7 @@ func (suite *Suite) TestNew() {
excludedTimesOfDay,
excludedDaysOfYear,
time.UTC,
minimumAge,
logger,
false,
)
Expand All @@ -66,6 +69,7 @@ func (suite *Suite) TestNew() {
suite.Equal(time.UTC, chaoskube.Timezone)
suite.Equal(logger, chaoskube.Logger)
suite.Equal(false, chaoskube.DryRun)
suite.Equal(minimumAge, chaoskube.MinimumAge)
}

func (suite *Suite) TestCandidates() {
Expand Down Expand Up @@ -108,6 +112,7 @@ func (suite *Suite) TestCandidates() {
[]time.Time{},
time.UTC,
false,
time.Duration(42),
)

suite.assertCandidates(chaoskube, tt.pods)
Expand Down Expand Up @@ -141,6 +146,7 @@ func (suite *Suite) TestVictim() {
[]time.Time{},
time.UTC,
false,
time.Duration(42),
)

suite.assertVictim(chaoskube, tt.victim)
Expand All @@ -157,6 +163,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
[]util.TimePeriod{},
[]time.Time{},
time.UTC,
time.Duration(42),
false,
)

Expand Down Expand Up @@ -185,6 +192,7 @@ func (suite *Suite) TestDeletePod() {
[]time.Time{},
time.UTC,
tt.dryRun,
time.Duration(42),
)

victim := util.NewPod("default", "foo", v1.PodRunning)
Expand Down Expand Up @@ -414,6 +422,7 @@ func (suite *Suite) TestTerminateVictim() {
tt.excludedDaysOfYear,
tt.timezone,
false,
time.Duration(42),
)
chaoskube.Now = tt.now

Expand All @@ -437,6 +446,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
[]util.TimePeriod{},
[]time.Time{},
time.UTC,
time.Duration(42),
false,
)

Expand Down Expand Up @@ -486,7 +496,7 @@ func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) {
}
}

func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool) *Chaoskube {
func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool, minimumAge time.Duration) *Chaoskube {
chaoskube := suite.setup(
labelSelector,
annotations,
Expand All @@ -495,6 +505,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
excludedTimesOfDay,
excludedDaysOfYear,
timezone,
minimumAge,
dryRun,
)

Expand All @@ -512,7 +523,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
return chaoskube
}

func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, dryRun bool) *Chaoskube {
func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube {
logOutput.Reset()

return New(
Expand All @@ -524,6 +535,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
excludedTimesOfDay,
excludedDaysOfYear,
timezone,
minimumAge,
logger,
dryRun,
)
Expand All @@ -541,3 +553,103 @@ func (t ThankGodItsFriday) Now() time.Time {
blackFriday, _ := time.Parse(time.RFC1123, "Fri, 24 Sep 1869 15:04:05 UTC")
return blackFriday
}

func (suite *Suite) TestMinimumAge() {
type pod struct {
name string
namespace string
creationTime time.Time
}

for _, tt := range []struct {
minimumAge time.Duration
now func() time.Time
pods []pod
candidates int
}{
// no minimum age set
{
time.Duration(0),
func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
[]pod{
{
name: "test1",
namespace: "test",
creationTime: time.Date(0, 10, 24, 9, 00, 00, 00, time.UTC),
},
},
1,
},
// minimum age set, but pod is too young
{
time.Hour * 1,
func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
[]pod{
{
name: "test1",
namespace: "test",
creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC),
},
},
0,
},
// one pod is too young, one matches
{
time.Hour * 1,
func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
[]pod{
// too young
{
name: "test1",
namespace: "test",
creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC),
},
// matches
{
name: "test2",
namespace: "test",
creationTime: time.Date(0, 10, 23, 8, 00, 00, 00, time.UTC),
},
},
1,
},
// exact time - should not match
{
time.Hour * 1,
func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) },
[]pod{
{
name: "test1",
namespace: "test",
creationTime: time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC),
},
},
0,
},
} {
chaoskube := suite.setup(
labels.Everything(),
labels.Everything(),
labels.Everything(),
[]time.Weekday{},
[]util.TimePeriod{},
[]time.Time{},
time.UTC,
tt.minimumAge,
false,
)
chaoskube.Now = tt.now

for _, p := range tt.pods {
pod := util.NewPod(p.namespace, p.name)
pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: p.creationTime}
_, err := chaoskube.Client.Core().Pods(pod.Namespace).Create(&pod)
suite.Require().NoError(err)
}

pods, err := chaoskube.Candidates()
suite.Require().NoError(err)

suite.Len(pods, tt.candidates)
}
}
4 changes: 4 additions & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ var (
excludedTimesOfDay string
excludedDaysOfYear string
timezone string
minimumAge time.Duration
master string
kubeconfig string
interval time.Duration
Expand All @@ -46,6 +47,7 @@ func init() {
kingpin.Flag("excluded-times-of-day", "A list of time periods of a day when termination is suspended, e.g. 22:00-08:00").StringVar(&excludedTimesOfDay)
kingpin.Flag("excluded-days-of-year", "A list of days of a year when termination is suspended, e.g. Apr1,Dec24").StringVar(&excludedDaysOfYear)
kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Default("UTC").StringVar(&timezone)
kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Default("0s").DurationVar(&minimumAge)
kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master)
kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval)
Expand All @@ -69,6 +71,7 @@ func main() {
"excludedTimesOfDay": excludedTimesOfDay,
"excludedDaysOfYear": excludedDaysOfYear,
"timezone": timezone,
"minimumAge": minimumAge,
"master": master,
"kubeconfig": kubeconfig,
"interval": interval,
Expand Down Expand Up @@ -145,6 +148,7 @@ func main() {
parsedTimesOfDay,
parsedDaysOfYear,
parsedTimezone,
minimumAge,
log.StandardLogger(),
dryRun,
)
Expand Down

0 comments on commit f242fad

Please sign in to comment.