Skip to content

Commit

Permalink
Break glass target for gencred refresher
Browse files Browse the repository at this point in the history
This will help the case where gencred refresher failed more than 2 days
  • Loading branch information
chaodaiG committed May 17, 2022
1 parent 93745f3 commit 329db7a
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 4 deletions.
18 changes: 18 additions & 0 deletions config/prow/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,21 @@ deploy-monitoring: get-cluster-credentials
deploy-all: deploy-prow deploy-build deploy-monitoring

.PHONY: update-config update-plugins deploy-prow deploy-build deploy-monitoring deploy-all

# The targets below are for breaking glass usages, should not run on a regular
# basis

# Run fix-gencred-refresher will generate kubeconfig for "trusted" build cluster
# and store it in GSM, you will need "container.admin" and "secretManager.admin"
# permission on k8s-prow GCP project to perform this action.
# This target is for solving the bootstrapping problem of gencred refresher,
# where it runs in "trusted" build cluster, as a prow job to rotate the
# kubeconfig for "trusted" build cluster along with other build clusters. So
# when the refresher job failed for more than 2 days, prow will stop working
# with "trusted" build cluster and thus not able to schedule gencred refrsher
# prow jobs any more. And this is when the admin of this repo need to run this
# target manually.
fix-gencred-refresher:
../../hack/make-rules/go-run/arbitrary.sh run ./gencred --config=./config/prow/gencred-config/gencred-config.yaml --gke-filter=projects/k8s-prow/locations/us-central1-f/clusters/prow

.PHONY: fix-gencred-refresher
24 changes: 20 additions & 4 deletions gencred/cmd/gencred/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ type options struct {
overwrite bool

config string
filter filter
// RefreshInterval defines how frequently the secret is refreshed.
refreshInterval time.Duration
}
Expand Down Expand Up @@ -105,6 +106,11 @@ type GSMSecretConfig struct {
Name string `json:"name"`
}

type filter struct {
gkeConnection string
context string
}

// parseFlags parses the command-line flags.
func (o *options) parseFlags() {
flag.StringVar(&o.context, "context", "", "The name of the kubeconfig context to use.")
Expand All @@ -116,17 +122,23 @@ func (o *options) parseFlags() {
flag.BoolVar(&o.overwrite, "overwrite", false, "Overwrite (rather than merge) output file if exists.")

flag.StringVar(&o.config, "config", "", "Configurations for running gencred.")
flag.StringVar(&o.filter.context, "context-filter", "", "Once specified, gencred only works on this context from the config file, must be supplied together with --config.")
flag.StringVar(&o.filter.gkeConnection, "gke-filter", "", "Once specified, gencred only works on this gkeConn from the config file, must be supplied together with --config.")
flag.DurationVar(&o.refreshInterval, "refresh-interval", 0, "RefreshInterval defines how frequently the secret is refreshed, unit is second.")
flag.Parse()
}

// validateFlags validates the command-line flags.
func (o *options) defaultAndValidateFlags() (*config, error) {
// config is mutually exclusive from local cluster.
if len(o.config) > 0 && (len(o.context) > 0) {
if len(o.config) > 0 && len(o.context) > 0 {
return nil, &util.ExitError{Message: "--config option is mutually exclusive with other options.", Code: 1}
}

if (len(o.filter.context) > 0 || len(o.filter.gkeConnection) > 0) && len(o.config) == 0 {
return nil, &util.ExitError{Message: "--context-filter and --gke-filter can only be used when --config option is supplied.", Code: 1}
}

// Read value from yaml files
var c config
if len(o.config) > 0 {
Expand Down Expand Up @@ -289,19 +301,19 @@ func Main() {
}

if o.refreshInterval == 0 {
if err := runOnce(*c); err != nil {
if err := runOnce(*c, o.filter); err != nil {
util.PrintErrAndExit(err)
}
return
}

defer interrupts.WaitForGracefulShutdown()
interrupts.Tick(func() {
runOnce(*c)
runOnce(*c, o.filter)
}, func() time.Duration { return o.refreshInterval })
}

func runOnce(c config) error {
func runOnce(c config, filter filter) error {
// Make sure process everyone before crying.
var errs []error
var config *rest.Config
Expand All @@ -310,6 +322,10 @@ func runOnce(c config) error {
errs = append(errs, errors.New("gke and context are mutually exclusive"))
continue
}
if (filter.context != "" && cc.Context != nil && filter.context != *cc.Context) ||
(filter.gkeConnection != "" && cc.GKEConnection != nil && filter.gkeConnection != *cc.GKEConnection) {
continue
}
if cc.Duration.Duration == 0 {
cc.Duration = &metav1.Duration{Duration: defaultDuration}
}
Expand Down

0 comments on commit 329db7a

Please sign in to comment.