Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Vatz healthchecker #190

Merged
merged 11 commits into from
Jul 22, 2022
Merged

Vatz healthchecker #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
cf1f628
merge commit
kim201212 Jul 20, 2022
3a2dc80
Edit Vatz HealthChecker
kim201212 Jul 20, 2022
bb07e50
Edit default.ymal
kim201212 Jul 20, 2022
ed3e7c9
Edit default.ymal
kim201212 Jul 20, 2022
9f66f60
Edit Vatz Health Checker default time
kim201212 Jul 20, 2022
86d6f48
Delete duplicate packages
kim201212 Jul 20, 2022
bf96408
Edit Notification state, severity
kim201212 Jul 20, 2022
b24572f
Delete Duplicate Packages, Rename the package
kim201212 Jul 20, 2022
7e5a7a0
Change vatzhealthchecker severity
kim201212 Jul 21, 2022
cd7fb3a
Define member func VatzHealthChecker
kim201212 Jul 22, 2022
892a7a7
Change func name clearly
kim201212 Jul 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions default.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ vatz_protocol_info:
notification_info:
discord_secret: "xxxxxxx"
pager_duty_secret: "YYYYY"
health_checker_schedule:
- "0 1 * * *"
plugins_infos:
default_verify_interval: 15
default_execute_interval: 30
Expand Down
7 changes: 7 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,18 @@ require (
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.27.0
github.com/stretchr/objx v0.1.0 // indirect
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 // indirect
Expand Down
14 changes: 14 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -23,6 +24,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -55,12 +57,22 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand Down Expand Up @@ -92,6 +104,8 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI=
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
6 changes: 5 additions & 1 deletion main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ func initiateServer(ch <-chan os.Signal) error {
cfg := config.GetConfig()
vatzConfig := cfg.Vatz
addr := fmt.Sprintf(":%d", vatzConfig.Port)
err := healthManager.VatzHealthCheck(vatzConfig.HealthCheckerSchedule)
if err != nil {
log.Println(err)
}

listener, err := net.Listen("tcp", addr)
if err != nil {
Expand Down Expand Up @@ -137,7 +141,7 @@ func multiPluginExecutor(plugin config.Plugin,
for {
select {
case <-verifyTicker.C:
live, _ := healthManager.HealthCheck(singleClient, plugin)
live, _ := healthManager.PluginHealthCheck(singleClient, plugin)
if live == "UP" {
isOkayToSend = true
} else {
Expand Down
1 change: 1 addition & 0 deletions manager/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type Config struct {
DiscordSecret string `yaml:"discord_secret"`
PagerDutySecret string `yaml:"pager_duty_secret"`
} `yaml:"notification_info"`
HealthCheckerSchedule []string `yaml:"health_checker_schedule"`
} `yaml:"vatz_protocol_info"`

PluginInfos PluginInfo `yaml:"plugins_infos"`
Expand Down
6 changes: 6 additions & 0 deletions manager/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type testConfigExpects struct {
ExpectVatzPort int
ExpectDiscordSecret string
ExpectPagerDutySecret string
ExpectHealthCheckerSchedule []string
ExpectDefaultVerifyInterval int
ExpectDefaultExecuteInterval int
ExpectDefaultPluginName string
Expand All @@ -51,6 +52,7 @@ func TestDefaultConfig(t *testing.T) {
ExpectVatzPort: 9090,
ExpectDiscordSecret: "XXXXX",
ExpectPagerDutySecret: "YYYYY",
ExpectHealthCheckerSchedule: []string{"* 1 * * *"},
ExpectDefaultVerifyInterval: 15,
ExpectDefaultExecuteInterval: 30,
ExpectDefaultPluginName: "vatz-plugin",
Expand Down Expand Up @@ -85,6 +87,7 @@ func TestDefaultConfig(t *testing.T) {
assert.Equal(t, test.ExpectVatzPort, cfg.Vatz.Port)
assert.Equal(t, test.ExpectDiscordSecret, cfg.Vatz.NotificationInfo.DiscordSecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)
assert.Equal(t, test.ExpectHealthCheckerSchedule, cfg.Vatz.HealthCheckerSchedule)

assert.Equal(t, test.ExpectDefaultVerifyInterval, cfg.PluginInfos.DefaultVerifyInterval)
assert.Equal(t, test.ExpectDefaultExecuteInterval, cfg.PluginInfos.DefaultExecuteInterval)
Expand Down Expand Up @@ -138,6 +141,7 @@ func TestOverrideDefaultValues(t *testing.T) {
assert.Equal(t, test.ExpectVatzPort, cfg.Vatz.Port)
assert.Equal(t, test.ExpectDiscordSecret, cfg.Vatz.NotificationInfo.DiscordSecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)

assert.Equal(t, test.ExpectDefaultVerifyInterval, cfg.PluginInfos.DefaultVerifyInterval)
assert.Equal(t, test.ExpectDefaultExecuteInterval, cfg.PluginInfos.DefaultExecuteInterval)
Expand Down Expand Up @@ -258,6 +262,7 @@ func TestGetConfig(t *testing.T) {
ExpectVatzPort: 9090,
ExpectDiscordSecret: "XXXXX",
ExpectPagerDutySecret: "YYYYY",
ExpectHealthCheckerSchedule: []string{"* 1 * * *"},
ExpectDefaultVerifyInterval: 15,
ExpectDefaultExecuteInterval: 30,
ExpectDefaultPluginName: "vatz-plugin",
Expand Down Expand Up @@ -300,6 +305,7 @@ func TestGetConfig(t *testing.T) {
assert.Equal(t, test.ExpectVatzPort, cfg.Vatz.Port)
assert.Equal(t, test.ExpectDiscordSecret, cfg.Vatz.NotificationInfo.DiscordSecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)
assert.Equal(t, test.ExpectPagerDutySecret, cfg.Vatz.NotificationInfo.PagerDutySecret)

assert.Equal(t, test.ExpectDefaultVerifyInterval, cfg.PluginInfos.DefaultVerifyInterval)
assert.Equal(t, test.ExpectDefaultExecuteInterval, cfg.PluginInfos.DefaultExecuteInterval)
Expand Down
3 changes: 2 additions & 1 deletion manager/config/fixtures.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ vatz_protocol_info:
notification_info:
discord_secret: "XXXXX"
pager_duty_secret: "YYYYY"

health_checker_schedule:
- "* 1 * * *"
plugins_infos:
default_verify_interval: 15
default_execute_interval: 30
Expand Down
28 changes: 25 additions & 3 deletions manager/healthcheck/healthcheck.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ package healthcheck

import (
"context"
"time"

"github.com/robfig/cron/v3"
"github.com/rs/zerolog/log"

pluginpb "github.com/dsrvlabs/vatz-proto/plugin/v1"
"github.com/dsrvlabs/vatz/manager/config"
Expand All @@ -17,7 +21,7 @@ var (
type healthCheck struct {
}

func (h healthCheck) HealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
func (h *healthCheck) PluginHealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
// TODO: Magic value always wrong.
isAlive := "UP"
verify, err := gClient.Verify(context.Background(), new(emptypb.Empty))
Expand All @@ -28,7 +32,7 @@ func (h healthCheck) HealthCheck(gClient pluginpb.PluginClient, plugin config.Pl
FuncName: "is_plugin_up",
State: pluginpb.STATE_FAILURE,
Msg: "is Down !!",
Severity: pluginpb.SEVERITY_CRITICAL,
Severity: pluginpb.SEVERITY_INFO,
ResourceType: plugin.Name,
}

Expand All @@ -38,8 +42,26 @@ func (h healthCheck) HealthCheck(gClient pluginpb.PluginClient, plugin config.Pl
return isAlive, nil
}

func (v *healthCheck) VatzHealthCheck(HealthCheckerSchedule []string) error {
c := cron.New(cron.WithLocation(time.UTC))
jsonMessage := msg.ReqMsg{
FuncName: "vatz_healthcheck",
State: pluginpb.STATE_SUCCESS,
Msg: "Vatz is alive!",
Severity: pluginpb.SEVERITY_CRITICAL,
gnongs marked this conversation as resolved.
Show resolved Hide resolved
ResourceType: "Vatz",
}
for i := 0; i < len(HealthCheckerSchedule); i++ {
log.Info().Str("module", "VatzHealthCheck").Msgf("%d, %s", i, HealthCheckerSchedule[i])
c.AddFunc(HealthCheckerSchedule[i], func() { dispatchManager.SendNotification(jsonMessage) })
}
c.Start()
return nil
}

type HealthCheck interface {
HealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error)
PluginHealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error)
VatzHealthCheck(schedule []string) error
}

func NewHealthChecker() HealthCheck {
Expand Down
8 changes: 6 additions & 2 deletions manager/healthcheck/healthcheck_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ func init() {
type healthManager struct {
}

func (s *healthManager) HealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
return healthCheckInstance.HealthCheck(gClient, plugin)
func (s *healthManager) PluginHealthCheck(gClient pluginpb.PluginClient, plugin config.Plugin) (string, error) {
return healthCheckInstance.PluginHealthCheck(gClient, plugin)
}

func (s *healthManager) VatzHealthCheck(schedule []string) error {
return healthCheckInstance.VatzHealthCheck(schedule)
}