Skip to content

Commit

Permalink
Vatz healthchecker (#190)
Browse files Browse the repository at this point in the history 
Vatz healthchecker
  • Loading branch information
@kim201212
kim201212 authored Jul 22, 2022
1 parent 72ada77 commit 3d2ef4d
Show file tree
Hide file tree
Showing 9 changed files with 68 additions and 7 deletions.
2 changes: 2 additions & 0 deletions default.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ vatz_protocol_info:
notification_info:
discord_secret: "xxxxxxx"
pager_duty_secret: "YYYYY"
health_checker_schedule:
- "0 1 * * *"
plugins_infos:
default_verify_interval: 15
default_execute_interval: 30
Expand Down
7 changes: 7 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,18 @@ require (
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.27.0
github.com/stretchr/objx v0.1.0 // indirect
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 // indirect
Expand Down
14 changes: 14 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -23,6 +24,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -55,12 +57,22 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand Down Expand Up @@ -92,6 +104,8 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI=
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
6 changes: 5 additions & 1 deletion main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ func initiateServer(ch <-chan os.Signal) error {
cfg := config.GetConfig()
vatzConfig := cfg.Vatz
addr := fmt.Sprintf(":%d", vatzConfig.Port)
err := healthManager.VatzHealthCheck(vatzConfig.HealthCheckerSchedule)
if err != nil {
log.Println(err)
}

listener, err := net.Listen("tcp", addr)
if err != nil {
Expand Down Expand Up @@ -140,7 +144,7 @@ func multiPluginExecutor(plugin config.Plugin,
for {
select {
case <-verifyTicker.C:
live, _ := healthManager.HealthCheck(singleClient, plugin)
live, _ := healthManager.PluginHealthCheck(singleClient, plugin)
if live == "UP" {
isOkayToSend = true
} else {
Expand Down
1 change: 1 addition & 0 deletions manager/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type Config struct {
DiscordSecret string `yaml:"discord_secret"`
PagerDutySecret string `yaml:"pager_duty_secret"`
} `yaml:"notification_info"`
HealthCheckerSchedule []string `yaml:"health_checker_schedule"`
} `yaml:"vatz_protocol_info"`

PluginInfos PluginInfo `yaml:"plugins_infos"`
Expand Down
6 changes: 6 additions & 0 deletions manager/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type testConfigExpects struct {
ExpectVatzPort int
ExpectDiscordSecret string
ExpectPagerDutySecret string
ExpectHealthCheckerSchedule []string
ExpectDefaultVerifyInterval int
ExpectDefaultExecuteInterval int
ExpectDefaultPluginName string
Expand All @@ -51,6 +52,7 @@ func TestDefaultConfig(t *testing.T) {
ExpectVatzPort: 9090,
ExpectDiscordSecret: "XXXXX",
ExpectPagerDutySecret: "YYYYY",
ExpectHealthCheckerSchedule: []string{"* 1 * * *"},
ExpectDefaultVerifyInterval: 15,
ExpectDefaultExecuteInterval: 30,
ExpectDefaultPluginName: "vatz-plugin",
Expand Down Expand Up @@ -85,6 +87,7 @@ func TestDefaultConfig(t *testing.T) {
assert.Equal(t, test.ExpectVatzPort, cfg.Vatz.Port)
assert.Equal(t, test.ExpectDiscordSecret, cfg.Vatz.NotificationInfo.DiscordSecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)
assert.Equal(t, test.ExpectHealthCheckerSchedule, cfg.Vatz.HealthCheckerSchedule)

assert.Equal(t, test.ExpectDefaultVerifyInterval, cfg.PluginInfos.DefaultVerifyInterval)
assert.Equal(t, test.ExpectDefaultExecuteInterval, cfg.PluginInfos.DefaultExecuteInterval)
Expand Down Expand Up @@ -138,6 +141,7 @@ func TestOverrideDefaultValues(t *testing.T) {
assert.Equal(t, test.ExpectVatzPort, cfg.Vatz.Port)
assert.Equal(t, test.ExpectDiscordSecret, cfg.Vatz.NotificationInfo.DiscordSecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)

assert.Equal(t, test.ExpectDefaultVerifyInterval, cfg.PluginInfos.DefaultVerifyInterval)
assert.Equal(t, test.ExpectDefaultExecuteInterval, cfg.PluginInfos.DefaultExecuteInterval)
Expand Down Expand Up @@ -258,6 +262,7 @@ func TestGetConfig(t *testing.T) {
ExpectVatzPort: 9090,
ExpectDiscordSecret: "XXXXX",
ExpectPagerDutySecret: "YYYYY",
ExpectHealthCheckerSchedule: []string{"* 1 * * *"},
ExpectDefaultVerifyInterval: 15,
ExpectDefaultExecuteInterval: 30,
ExpectDefaultPluginName: "vatz-plugin",
Expand Down Expand Up @@ -300,6 +305,7 @@ func TestGetConfig(t *testing.T) {
assert.Equal(t, test.ExpectVatzPort, cfg.Vatz.Port)
assert.Equal(t, test.ExpectDiscordSecret, cfg.Vatz.NotificationInfo.DiscordSecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)

assert.Equal(t, test.ExpectDefaultVerifyInterval, cfg.PluginInfos.DefaultVerifyInterval)
assert.Equal(t, test.ExpectDefaultExecuteInterval, cfg.PluginInfos.DefaultExecuteInterval)
Expand Down
3 changes: 2 additions & 1 deletion manager/config/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ vatz_protocol_info:
notification_info:
discord_secret: "XXXXX"
pager_duty_secret: "YYYYY"
health_checker_schedule:
- "* 1 * * *"
plugins_infos:
default_verify_interval: 15
default_execute_interval: 30
Expand Down
28 changes: 25 additions & 3 deletions manager/healthcheck/healthcheck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ package healthcheck

import (
"context"
"time"

"github.com/robfig/cron/v3"
"github.com/rs/zerolog/log"

pluginpb "github.com/dsrvlabs/vatz-proto/plugin/v1"
"github.com/dsrvlabs/vatz/manager/config"
Expand All @@ -17,7 +21,7 @@ var (
type healthCheck struct {
}

func (h healthCheck) HealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
func (h *healthCheck) PluginHealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
// TODO: Magic value always wrong.
isAlive := "UP"
verify, err := gClient.Verify(context.Background(), new(emptypb.Empty))
Expand All @@ -28,7 +32,7 @@ func (h healthCheck) HealthCheck(gClient pluginpb.PluginClient, plugin config.Pl
FuncName: "is_plugin_up",
State: pluginpb.STATE_FAILURE,
Msg: "is Down !!",
Severity: pluginpb.SEVERITY_CRITICAL,
Severity: pluginpb.SEVERITY_INFO,
ResourceType: plugin.Name,
}

Expand All @@ -38,8 +42,26 @@ func (h healthCheck) HealthCheck(gClient pluginpb.PluginClient, plugin config.Pl
return isAlive, nil
}

func (v *healthCheck) VatzHealthCheck(HealthCheckerSchedule []string) error {
c := cron.New(cron.WithLocation(time.UTC))
jsonMessage := msg.ReqMsg{
FuncName: "vatz_healthcheck",
State: pluginpb.STATE_SUCCESS,
Msg: "Vatz is alive!",
Severity: pluginpb.SEVERITY_CRITICAL,
ResourceType: "Vatz",
}
for i := 0; i < len(HealthCheckerSchedule); i++ {
log.Info().Str("module", "VatzHealthCheck").Msgf("%d, %s", i, HealthCheckerSchedule[i])
c.AddFunc(HealthCheckerSchedule[i], func() { dispatchManager.SendNotification(jsonMessage) })
}
c.Start()
return nil
}

type HealthCheck interface {
HealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error)
PluginHealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error)
VatzHealthCheck(schedule []string) error
}

func NewHealthChecker() HealthCheck {
Expand Down
8 changes: 6 additions & 2 deletions manager/healthcheck/healthcheck_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ func init() {
type healthManager struct {
}

func (s *healthManager) HealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
return healthCheckInstance.HealthCheck(gClient, plugin)
func (s *healthManager) PluginHealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
return healthCheckInstance.PluginHealthCheck(gClient, plugin)
}

func (s *healthManager) VatzHealthCheck(schedule []string) error {
return healthCheckInstance.VatzHealthCheck(schedule)
}

0 comments on commit 3d2ef4d

Please sign in to comment.