Skip to content
This repository has been archived by the owner on Nov 1, 2022. It is now read-only.

Commit

Permalink
Revise and uniform all Helm operator logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@hiddeco
hiddeco committed Apr 9, 2019
1 parent c333b25 commit 05cc815
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 62 deletions.
6 changes: 3 additions & 3 deletions cmd/helm-operator/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,19 +137,19 @@ func main() {

cfg, err := clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
if err != nil {
mainLogger.Log("error", fmt.Sprintf("Error building kubeconfig: %v", err))
mainLogger.Log("error", fmt.Sprintf("error building kubeconfig: %v", err))
os.Exit(1)
}

kubeClient, err := kubernetes.NewForConfig(cfg)
if err != nil {
mainLogger.Log("error", fmt.Sprintf("Error building kubernetes clientset: %v", err))
mainLogger.Log("error", fmt.Sprintf("error building kubernetes clientset: %v", err))
os.Exit(1)
}

ifClient, err := clientset.NewForConfig(cfg)
if err != nil {
mainLogger.Log("error", fmt.Sprintf("Error building integrations clientset: %v", err))
mainLogger.Log("error", fmt.Sprintf("error building integrations clientset: %v", err))
os.Exit(1)
}

Expand Down
59 changes: 33 additions & 26 deletions integrations/helm/chartsync/chartsync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ func New(logger log.Logger, clients Clients, release *release.Release, releaseQu
// Helm releases in the cluster, what HelmRelease declare, and
// changes in the git repos mentioned by any HelmRelease.
func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *sync.WaitGroup) {
chs.logger.Log("info", "Starting git chart sync loop")
chs.logger.Log("info", "starting git chart sync loop")

wg.Add(1)
go func() {
Expand All @@ -171,27 +171,32 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn
continue
}

// Retrieve the mirror we got a change signal for
repo, ok := chs.mirrors.Get(mirror)
if !ok {
// Then why .. did you say .. it had changed? It may have been removed. Add it back and let it signal again.
chs.logger.Log("warning", "mirrored git repo disappeared after signalling change", "repo", mirror)
for _, fhr := range resources {
// Then why .. did you say .. it had changed? It may have been removed. Add it back and let it signal again.
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionUnknown, ReasonGitNotReady, "git mirror missing; starting mirroring again")
chs.logger.Log("warning", "mirrored git repo disappeared after signalling change", "repo", mirror)
chs.maybeMirror(fhr)
}
continue
}

// Ensure the repo is ready
status, err := repo.Status()
if status != git.RepoReady {
chs.logger.Log("info", "repo not ready yet, while attempting chart sync", "repo", mirror, "status", string(status))
for _, fhr := range resources {
chs.logger.Log("info", "repo not ready yet, while attempting chart sync", "repo", mirror, "status", string(status))
// TODO(michael) log if there's a problem with the following?
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionUnknown, ReasonGitNotReady, err.Error())
}
continue
}

// Determine if we need to update the clone and
// schedule an upgrade for every HelmRelease that
// makes use of the mirror
for _, fhr := range resources {
ref := fhr.Spec.ChartSource.GitChartSource.RefOrDefault()
path := fhr.Spec.ChartSource.GitChartSource.Path
Expand All @@ -202,7 +207,7 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn
cancel()
if err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionFalse, ReasonGitNotReady, "problem cloning from local git mirror: "+err.Error())
chs.logger.Log("warning", "could not get revision for ref while checking for changes", "repo", mirror, "ref", ref, "err", err)
chs.logger.Log("warning", "could not get revision for ref while checking for changes", "resource", fhr.ResourceID().String(), "repo", mirror, "ref", ref, "err", err)
continue
}

Expand All @@ -218,7 +223,7 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn
cancel()
if err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionFalse, ReasonGitNotReady, "problem cloning from local git mirror: "+err.Error())
chs.logger.Log("warning", "could not get revision for ref while checking for changes", "repo", mirror, "ref", ref, "err", err)
chs.logger.Log("warning", "could not get revision for ref while checking for changes", "resource", fhr.ResourceID().String(), "repo", mirror, "ref", ref, "err", err)
continue
}
ok = len(commits) == 0
Expand All @@ -230,7 +235,7 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn
cancel()
if err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionFalse, ReasonGitNotReady, "problem cloning from local git mirror: "+err.Error())
chs.logger.Log("warning", "could not clone from mirror while checking for changes", "repo", mirror, "ref", ref, "err", err)
chs.logger.Log("warning", "could not clone from mirror while checking for changes", "resource", fhr.ResourceID().String(), "repo", mirror, "ref", ref, "err", err)
continue
}
newCloneForChart := clone{remote: mirror, ref: ref, head: refHead, export: newClone}
Expand All @@ -242,10 +247,12 @@ func (chs *ChartChangeSync) Run(stopCh <-chan struct{}, errc chan error, wg *syn
}
}

// Enqueue release
cacheKey, err := cache.MetaNamespaceKeyFunc(fhr.GetObjectMeta())
if err != nil {
continue
}
chs.logger.Log("info", "enqueing release upgrade due to change in git chart source", "resource", fhr.ResourceID().String())
chs.releaseQueue.AddRateLimited(cacheKey)
}
}
Expand Down Expand Up @@ -327,12 +334,12 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
if !ok {
chs.maybeMirror(fhr)
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionUnknown, ReasonGitNotReady, "git repo "+chartSource.GitURL+" not mirrored yet")
chs.logger.Log("info", "chart repo not cloned yet", "releaseName", releaseName, "resource", fmt.Sprintf("%s:%s/%s", fhr.Namespace, fhr.Kind, fhr.Name))
chs.logger.Log("info", "chart repo not cloned yet", "resource", fhr.ResourceID().String())
} else {
status, err := repo.Status()
if status != git.RepoReady {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionUnknown, ReasonGitNotReady, "git repo not mirrored yet: "+err.Error())
chs.logger.Log("info", "chart repo not ready yet", "releaseName", releaseName, "resource", fmt.Sprintf("%s:%s/%s", fhr.Namespace, fhr.Kind, fhr.Name), "status", string(status), "err", err)
chs.logger.Log("info", "chart repo not ready yet", "resource", fhr.ResourceID().String(), "status", string(status), "err", err)
}
}
return
Expand All @@ -344,7 +351,7 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
if chs.config.UpdateDeps && !fhr.Spec.ChartSource.GitChartSource.SkipDepUpdate {
if err := updateDependencies(chartPath, ""); err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionFalse, ReasonDependencyFailed, err.Error())
chs.logger.Log("warning", "Failed to update chart dependencies", "namespace", fhr.Namespace, "name", fhr.Name, "error", err)
chs.logger.Log("warning", "failed to update chart dependencies", "resource", fhr.ResourceID().String(), "err", err)
return
}
}
Expand All @@ -353,7 +360,7 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
path, err := ensureChartFetched(chs.config.ChartCache, chartSource)
if err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionFalse, ReasonDownloadFailed, "chart download failed: "+err.Error())
chs.logger.Log("info", "chart download failed", "releaseName", releaseName, "resource", fhr.ResourceID().String(), "err", err)
chs.logger.Log("info", "chart download failed", "resource", fhr.ResourceID().String(), "err", err)
return
}
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseChartFetched, v1.ConditionTrue, ReasonDownloaded, "chart fetched: "+filepath.Base(path))
Expand All @@ -365,7 +372,7 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
_, err := chs.release.Install(chartPath, releaseName, fhr, release.InstallAction, opts, &chs.kubeClient)
if err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionFalse, ReasonInstallFailed, err.Error())
chs.logger.Log("warning", "Failed to install chart", "namespace", fhr.Namespace, "name", fhr.Name, "error", err)
chs.logger.Log("warning", "failed to install chart", "resource", fhr.ResourceID().String(), "err", err)
return
}
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionTrue, ReasonSuccess, "helm install succeeded")
Expand All @@ -377,28 +384,28 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {

changed, err := chs.shouldUpgrade(chartPath, rel, fhr)
if err != nil {
chs.logger.Log("warning", "Unable to determine if release has changed", "namespace", fhr.Namespace, "name", fhr.Name, "error", err)
chs.logger.Log("warning", "unable to determine if release has changed", "resource", fhr.ResourceID().String(), "err", err)
return
}
if changed {
cFhr, err := chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace).Get(fhr.Name, metav1.GetOptions{})
if err != nil {
chs.logger.Log("warning", "Failed to retrieve HelmRelease scheduled for upgrade", "namespace", fhr.Namespace, "name", fhr.Name, "error", err)
chs.logger.Log("warning", "failed to retrieve HelmRelease scheduled for upgrade", "resource", fhr.ResourceID().String(), "err", err)
return
}
if diff := cmp.Diff(fhr.Spec, cFhr.Spec); diff != "" {
chs.logger.Log("warning", "HelmRelease spec has diverged since we calculated if we should upgrade, skipping upgrade", "namespace", fhr.Namespace, "name", fhr.Name)
chs.logger.Log("warning", "HelmRelease spec has diverged since we calculated if we should upgrade, skipping upgrade", "resource", fhr.ResourceID().String())
return
}
_, err = chs.release.Install(chartPath, releaseName, fhr, release.UpgradeAction, opts, &chs.kubeClient)
if err != nil {
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionFalse, ReasonUpgradeFailed, err.Error())
chs.logger.Log("warning", "Failed to upgrade chart", "namespace", fhr.Namespace, "name", fhr.Name, "error", err)
chs.logger.Log("warning", "failed to upgrade chart", "resource", fhr.ResourceID().String(), "err", err)
return
}
chs.setCondition(&fhr, fluxv1beta1.HelmReleaseReleased, v1.ConditionTrue, ReasonSuccess, "helm upgrade succeeded")
if err = status.UpdateReleaseRevision(chs.ifClient.FluxV1beta1().HelmReleases(fhr.Namespace), fhr, chartRevision); err != nil {
chs.logger.Log("warning", "could not update the release revision", "namespace", fhr.Namespace, "resource", fhr.Name, "err", err)
chs.logger.Log("warning", "could not update the release revision", "resource", fhr.ResourceID().String(), "err", err)
}
return
}
Expand All @@ -412,7 +419,7 @@ func (chs *ChartChangeSync) DeleteRelease(fhr fluxv1beta1.HelmRelease) {
name := release.GetReleaseName(fhr)
err := chs.release.Delete(name)
if err != nil {
chs.logger.Log("warning", "Chart release not deleted", "release", name, "error", err)
chs.logger.Log("warning", "chart release not deleted", "resource", fhr.ResourceID().String(), "release", name, "err", err)
}

// Remove the clone we may have for this HelmRelease
Expand All @@ -429,11 +436,11 @@ func (chs *ChartChangeSync) DeleteRelease(fhr fluxv1beta1.HelmRelease) {

// SyncMirrors instructs all mirrors to refresh from their upstream.
func (chs *ChartChangeSync) SyncMirrors() {
chs.logger.Log("info", "Starting mirror sync")
chs.logger.Log("info", "starting mirror sync")
for _, err := range chs.mirrors.RefreshAll(chs.config.GitTimeout) {
chs.logger.Log("error", fmt.Sprintf("Failure while syncing mirror: %s", err))
chs.logger.Log("error", fmt.Sprintf("failure while syncing mirror: %s", err))
}
chs.logger.Log("info", "Finished syncing mirrors")
chs.logger.Log("info", "finished syncing mirrors")
}

// getCustomResourcesForMirror retrieves all the resources that make
Expand Down Expand Up @@ -523,7 +530,7 @@ func sortChartFields(c *hapi_chart.Chart) *hapi_chart.Chart {
// doing a dry run install from the chart in the git repo.
func (chs *ChartChangeSync) shouldUpgrade(chartsRepo string, currRel *hapi_release.Release, fhr fluxv1beta1.HelmRelease) (bool, error) {
if currRel == nil {
return false, fmt.Errorf("No Chart release provided for %v", fhr.GetName())
return false, fmt.Errorf("no chart release provided for %v", fhr.GetName())
}

currVals := currRel.GetConfig()
Expand All @@ -542,18 +549,18 @@ func (chs *ChartChangeSync) shouldUpgrade(chartsRepo string, currRel *hapi_relea
// compare values && Chart
if diff := cmp.Diff(currVals, desVals); diff != "" {
if chs.config.LogDiffs {
chs.logger.Log("error", fmt.Sprintf("Release %s: values have diverged due to manual Chart release", currRel.GetName()), "diff", diff)
chs.logger.Log("error", fmt.Sprintf("release %s: values have diverged due to manual chart release", currRel.GetName()), "resource", fhr.ResourceID().String(), "diff", diff)
} else {
chs.logger.Log("error", fmt.Sprintf("Release %s: values have diverged due to manual Chart release", currRel.GetName()))
chs.logger.Log("error", fmt.Sprintf("release %s: values have diverged due to manual chart release", currRel.GetName()), "resource", fhr.ResourceID().String())
}
return true, nil
}

if diff := cmp.Diff(sortChartFields(currChart), sortChartFields(desChart)); diff != "" {
if chs.config.LogDiffs {
chs.logger.Log("error", fmt.Sprintf("Release %s: Chart has diverged due to manual Chart release", currRel.GetName()), "diff", diff)
chs.logger.Log("error", fmt.Sprintf("release %s: chart has diverged due to manual chart release", currRel.GetName()), "resource", fhr.ResourceID().String(), "diff", diff)
} else {
chs.logger.Log("error", fmt.Sprintf("Release %s: Chart has diverged due to manual Chart release", currRel.GetName()))
chs.logger.Log("error", fmt.Sprintf("release %s: chart has diverged due to manual chart release", currRel.GetName()), "resource", fhr.ResourceID().String())
}
return true, nil
}
Expand Down
2 changes: 1 addition & 1 deletion integrations/helm/helm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func ClientSetup(logger log.Logger, kubeClient *kubernetes.Clientset, tillerOpts
for {
helmClient, host, err = newClient(kubeClient, tillerOpts)
if err != nil {
logger.Log("error", fmt.Sprintf("Error creating helm client: %s", err.Error()))
logger.Log("error", fmt.Sprintf("error creating helm client: %s", err.Error()))
time.Sleep(20 * time.Second)
continue
}
Expand Down
9 changes: 5 additions & 4 deletions integrations/helm/http/daemon/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ package daemon
import (
"context"
"fmt"
"net/http"
"sync/atomic"
"time"

"github.com/go-kit/kit/log"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/weaveworks/flux/integrations/helm/api"
transport "github.com/weaveworks/flux/integrations/helm/http"
"net/http"
"sync/atomic"
"time"
)

// ListenAndServe starts a HTTP server instrumented with Prometheus metrics,
Expand All @@ -37,7 +38,7 @@ func ListenAndServe(listenAddr string, apiServer api.Server, logger log.Logger,
IdleTimeout: 15 * time.Second,
}

logger.Log("info", fmt.Sprintf("Starting HTTP server on %s", listenAddr))
logger.Log("info", fmt.Sprintf("starting HTTP server on %s", listenAddr))

// run server in background
go func() {
Expand Down
Loading

0 comments on commit 05cc815

Please sign in to comment.