diff --git a/Gopkg.lock b/Gopkg.lock index a7dfb7ff5..6d40e6165 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -155,6 +155,17 @@ packages = ["."] revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" +[[projects]] + name = "github.com/google/go-cmp" + packages = [ + "cmp", + "cmp/internal/diff", + "cmp/internal/function", + "cmp/internal/value" + ] + revision = "3af367b6b30c263d47e8895973edcca9a49cf029" + version = "v0.2.0" + [[projects]] branch = "master" name = "github.com/google/gofuzz" @@ -804,6 +815,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "18e1f9cb0f7c43cf67a0ee5b21699771d4c1474b521b3f9a41d863b0291e3a62" + inputs-digest = "3cbcf68d471634e600092225d2a648c2e1e1ec89f293cfee1e5751c5ee39e3c2" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 2c9499b23..414afebbb 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -32,3 +32,7 @@ required = ["k8s.io/code-generator/cmd/client-gen"] [[constraint]] name = "k8s.io/helm" version = "v2.8.1" + +[[constraint]] + name = "github.com/google/go-cmp" + version = "0.2.0" diff --git a/integrations/helm/release/release.go b/integrations/helm/release/release.go index 38d0d4bdd..df66a07cd 100644 --- a/integrations/helm/release/release.go +++ b/integrations/helm/release/release.go @@ -34,6 +34,21 @@ type Release struct { sync.RWMutex } +func (r *Release) ConfigSync() *helmgit.Checkout { + return r.Repo.ConfigSync +} + +type Releaser interface { + GetCurrent() (map[string][]DeployInfo, error) + GetDeployedRelease(name string) (*hapi_release.Release, error) + Install(checkout *helmgit.Checkout, + releaseName string, + fhr ifv1.FluxHelmRelease, + action Action, + opts InstallOptions) (hapi_release.Release, error) + ConfigSync() *helmgit.Checkout +} + type repo struct { ConfigSync *helmgit.Checkout } diff --git a/integrations/helm/releasesync/releasesync.go b/integrations/helm/releasesync/releasesync.go index 6872d3388..4aae14b6d 100644 --- a/integrations/helm/releasesync/releasesync.go +++ b/integrations/helm/releasesync/releasesync.go @@ -12,6 +12,8 @@ import ( "path/filepath" "strings" + "github.com/pkg/errors" + protobuf "github.com/golang/protobuf/ptypes/timestamp" hapi_release "k8s.io/helm/pkg/proto/hapi/release" @@ -25,28 +27,29 @@ import ( ) const ( - CustomResourceKind = "FluxHelmRelease" - syncDelay = 90 + syncDelay = 90 ) -type ReleaseFhr struct { +type releaseFhr struct { RelName string - FhrName string Fhr ifv1.FluxHelmRelease } +// ReleaseChangeSync implements DoReleaseChangeSync to return the cluster to the +// state dictated by Custom Resources after manual Chart release(s). type ReleaseChangeSync struct { logger log.Logger - release *chartrelease.Release + release chartrelease.Releaser } +// New creates a ReleaseChangeSync. func New( logger log.Logger, - release *chartrelease.Release) *ReleaseChangeSync { + releaser chartrelease.Releaser) *ReleaseChangeSync { return &ReleaseChangeSync{ logger: logger, - release: release, + release: releaser, } } @@ -63,13 +66,14 @@ type chartRelease struct { } // DoReleaseChangeSync returns the cluster to the state dictated by Custom Resources -// after manual Chart release(s) +// after manual Chart release(s). func (rs *ReleaseChangeSync) DoReleaseChangeSync(ifClient ifclientset.Clientset, ns []string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), helmgit.DefaultCloneTimeout) relsToSync, err := rs.releasesToSync(ctx, ifClient, ns) cancel() if err != nil { - err := fmt.Errorf("Failure to get info about manual chart release changes: %#v", err) + err = errors.Wrap(err, "getting info about manual chart release changes") + rs.logger.Log("error", err) return false, err } if len(relsToSync) == 0 { @@ -80,29 +84,32 @@ func (rs *ReleaseChangeSync) DoReleaseChangeSync(ifClient ifclientset.Clientset, err = rs.sync(ctx, relsToSync) cancel() if err != nil { - err := fmt.Errorf("Failure to sync cluster after manual chart release changes: %#v", err) - return false, err + return false, errors.Wrap(err, "syncing cluster after manual chart release changes") } return true, nil } // getCustomResources retrieves FluxHelmRelease resources -// and outputs them organised by namespace and: Chart release name or Custom Resource name -// map[namespace] = []ReleaseFhr -func (rs *ReleaseChangeSync) getCustomResources(ifClient ifclientset.Clientset, namespaces []string) (map[string][]ReleaseFhr, error) { - relInfo := make(map[string][]ReleaseFhr) +// and returns them organised by namespace and chart release name. +// map[namespace] = []releaseFhr. +func (rs *ReleaseChangeSync) getCustomResources( + ifClient ifclientset.Clientset, + namespaces []string) (map[string][]releaseFhr, error) { + + relInfo := make(map[string][]releaseFhr) for _, ns := range namespaces { list, err := customresource.GetNSCustomResources(ifClient, ns) if err != nil { - rs.logger.Log("error", fmt.Errorf("Failure while retrieving FluxHelmReleases in namespace %s: %v", ns, err)) - return nil, err + return nil, errors.Wrap(err, + fmt.Sprintf("retrieving FluxHelmReleases in namespace %s", ns)) } - rf := []ReleaseFhr{} + + rf := []releaseFhr{} for _, fhr := range list.Items { relName := chartrelease.GetReleaseName(fhr) - rf = append(rf, ReleaseFhr{RelName: relName, Fhr: fhr}) + rf = append(rf, releaseFhr{RelName: relName, Fhr: fhr}) } if len(rf) > 0 { relInfo[ns] = rf @@ -111,7 +118,13 @@ func (rs *ReleaseChangeSync) getCustomResources(ifClient ifclientset.Clientset, return relInfo, nil } -func (rs *ReleaseChangeSync) shouldUpgrade(currRel *hapi_release.Release, fhr ifv1.FluxHelmRelease) (bool, error) { +// shouldUpgrade returns true if the current running values or chart +// don't match what the repo says we ought to be running, based on +// doing a dry run install from the chart in the git repo. +func (rs *ReleaseChangeSync) shouldUpgrade( + currRel *hapi_release.Release, + fhr ifv1.FluxHelmRelease) (bool, error) { + if currRel == nil { return false, fmt.Errorf("No Chart release provided for %v", fhr.GetName()) } @@ -122,7 +135,7 @@ func (rs *ReleaseChangeSync) shouldUpgrade(currRel *hapi_release.Release, fhr if // Get the desired release state opts := chartrelease.InstallOptions{DryRun: true} tempRelName := strings.Join([]string{currRel.GetName(), "temp"}, "-") - desRel, err := rs.release.Install(rs.release.Repo.ConfigSync, tempRelName, fhr, "CREATE", opts) + desRel, err := rs.release.Install(rs.release.ConfigSync(), tempRelName, fhr, "CREATE", opts) if err != nil { return false, err } @@ -142,8 +155,10 @@ func (rs *ReleaseChangeSync) shouldUpgrade(currRel *hapi_release.Release, fhr if return false, nil } -// existingReleasesToSync determines which Chart releases need to be deleted/upgraded -// to bring the cluster to the desired state +// addExistingReleasesToSync populates relsToSync (map from namespace +// to chartRelease) with the members of currentReleases that need +// updating because they're diverged from the desired state. Desired +// state is specified by customResources and what's in our git checkout. func (rs *ReleaseChangeSync) addExistingReleasesToSync( relsToSync map[string][]chartRelease, currentReleases map[string]map[string]struct{}, @@ -185,8 +200,9 @@ func (rs *ReleaseChangeSync) addExistingReleasesToSync( return nil } -// deletedReleasesToSync determines which Chart releases need to be installed -// to bring the cluster to the desired state +// addDeletedReleasesToSync populates relsToSync (map from namespace +// to chartRelease) with chartReleases based on the charts referenced +// in customResources that are absent from currentReleases. func (rs *ReleaseChangeSync) addDeletedReleasesToSync( relsToSync map[string][]chartRelease, currentReleases map[string]map[string]struct{}, @@ -224,35 +240,53 @@ func (rs *ReleaseChangeSync) addDeletedReleasesToSync( return nil } -func (rs *ReleaseChangeSync) releasesToSync(ctx context.Context, ifClient ifclientset.Clientset, ns []string) (map[string][]chartRelease, error) { +// releasesToSync queries Tiller to get all current Helm releases, queries k8s +// custom resources to get all FluxHelmRelease(s), and returns a map from +// namespace to chartRelease(s) that need to be synced. +func (rs *ReleaseChangeSync) releasesToSync( + ctx context.Context, + ifClient ifclientset.Clientset, + ns []string) (map[string][]chartRelease, error) { + relDepl, err := rs.release.GetCurrent() if err != nil { return nil, err } - curRels := MappifyDeployInfo(relDepl) + curRels := mappifyDeployInfo(relDepl) relCrs, err := rs.getCustomResources(ifClient, ns) if err != nil { return nil, err } - crs := MappifyReleaseFhrInfo(relCrs) + crs := mappifyReleaseFhrInfo(relCrs) relsToSync := make(map[string][]chartRelease) - rs.addDeletedReleasesToSync(relsToSync, curRels, crs) - rs.addExistingReleasesToSync(relsToSync, curRels, crs) + + // FIXME: we probably shouldn't be throwing away errors + _ = rs.addDeletedReleasesToSync(relsToSync, curRels, crs) + _ = rs.addExistingReleasesToSync(relsToSync, curRels, crs) return relsToSync, nil } -func (rs *ReleaseChangeSync) sync(ctx context.Context, releases map[string][]chartRelease) error { +// sync takes a map from namespace to list of chartRelease(s) +// that need to be applied, and attempts to apply them. +// It returns the first error encountered. A chart missing +// from the repo doesn't count as an error, but will be logged. +func (rs *ReleaseChangeSync) sync( + ctx context.Context, + releases map[string][]chartRelease) error { + + // TODO it's weird that we do a pull here, after we've already decided + // what to do. Ask why. ctx, cancel := context.WithTimeout(ctx, helmgit.DefaultPullTimeout) - err := rs.release.Repo.ConfigSync.Pull(ctx) + err := rs.release.ConfigSync().Pull(ctx) cancel() if err != nil { return fmt.Errorf("Failure while pulling repo: %#v", err) } - checkout := rs.release.Repo.ConfigSync + checkout := rs.release.ConfigSync() chartPathBase := filepath.Join(checkout.Dir, checkout.Config.Path) opts := chartrelease.InstallOptions{DryRun: false} @@ -282,6 +316,8 @@ func (rs *ReleaseChangeSync) sync(ctx context.Context, releases map[string][]cha if err != nil { return err } + default: + panic(fmt.Sprintf("invalid action %q", chr.action)) } } } diff --git a/integrations/helm/releasesync/releasesync_test.go b/integrations/helm/releasesync/releasesync_test.go new file mode 100644 index 000000000..db8099ea8 --- /dev/null +++ b/integrations/helm/releasesync/releasesync_test.go @@ -0,0 +1,353 @@ +package releasesync + +import ( + "fmt" + "os" + "testing" + + proto "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "k8s.io/helm/pkg/helm" + hapi_chart "k8s.io/helm/pkg/proto/hapi/chart" + hapi_release "k8s.io/helm/pkg/proto/hapi/release" + + "github.com/go-kit/kit/log" + ifv1 "github.com/weaveworks/flux/apis/helm.integrations.flux.weave.works/v1alpha2" + helmgit "github.com/weaveworks/flux/integrations/helm/git" + "github.com/weaveworks/flux/integrations/helm/release" + chartrelease "github.com/weaveworks/flux/integrations/helm/release" +) + +type installReq struct { + checkoutDir string + releaseName string + fhr ifv1.FluxHelmRelease + action chartrelease.Action + opts chartrelease.InstallOptions +} + +type installResult struct { + release hapi_release.Release + err error +} + +type install struct { + installReq + installResult +} + +type mockReleaser struct { + current map[string][]chartrelease.DeployInfo + deployed map[string]*hapi_release.Release + configSync *helmgit.Checkout + installs []install +} + +func (r *mockReleaser) GetCurrent() (map[string][]chartrelease.DeployInfo, error) { + if r.current == nil { + return nil, fmt.Errorf("failed to fetch current releases") + } + return r.current, nil +} + +func (r *mockReleaser) GetDeployedRelease(name string) (*hapi_release.Release, error) { + if _, present := r.deployed[name]; !present { + return nil, fmt.Errorf("no release hamed %q", name) + } + return r.deployed[name], nil +} + +func (r *mockReleaser) ConfigSync() *helmgit.Checkout { + return r.configSync +} + +func (r *mockReleaser) Install(checkout *helmgit.Checkout, + releaseName string, + fhr ifv1.FluxHelmRelease, + action chartrelease.Action, + opts chartrelease.InstallOptions) (hapi_release.Release, error) { + req := installReq{ + checkoutDir: checkout.Dir, + releaseName: releaseName, + fhr: fhr, + action: action, + opts: opts} + cmpopts := cmp.AllowUnexported(installReq{}) + for _, i := range r.installs { + if cmp.Equal(i.installReq, req, cmpopts) { + return i.installResult.release, i.installResult.err + } + } + return hapi_release.Release{}, fmt.Errorf("unexpected request: %+v", req) +} + +func makeCurRel(ns string, relNames ...string) map[string]map[string]struct{} { + m := make(map[string]map[string]struct{}) + m[ns] = make(map[string]struct{}) + for _, relName := range relNames { + m[ns][relName] = struct{}{} + } + return m +} + +func mergeCurRels(a, b map[string]map[string]struct{}) map[string]map[string]struct{} { + m := make(map[string]map[string]struct{}) + for ns := range a { + m[ns] = a[ns] + } + for ns := range b { + if _, present := m[ns]; present { + panic("ns '" + ns + "' present in both a and b") + } + m[ns] = b[ns] + } + return m +} + +func makeCustRes(ns string, relNames ...string) map[string]map[string]ifv1.FluxHelmRelease { + m := make(map[string]map[string]ifv1.FluxHelmRelease) + m[ns] = make(map[string]ifv1.FluxHelmRelease) + for _, relName := range relNames { + m[ns][relName] = ifv1.FluxHelmRelease{} + } + return m +} + +func mergeCustRes(a, b map[string]map[string]ifv1.FluxHelmRelease) map[string]map[string]ifv1.FluxHelmRelease { + m := make(map[string]map[string]ifv1.FluxHelmRelease) + for ns := range a { + m[ns] = a[ns] + } + for ns := range b { + if _, present := m[ns]; present { + panic("ns '" + ns + "' present in both a and b") + } + m[ns] = b[ns] + } + return m +} + +func TestAddDeletedReleasesToSync(t *testing.T) { + var zeromap = make(map[string][]chartRelease) + var tests = []struct { + msg string + currentReleases map[string]map[string]struct{} + customResources map[string]map[string]ifv1.FluxHelmRelease + want map[string][]chartRelease + }{ + { + msg: "no-op, zero resources", + currentReleases: makeCurRel("ns1", "r1"), + customResources: make(map[string]map[string]ifv1.FluxHelmRelease), + want: zeromap, + }, + { + msg: "no-op, equality", + currentReleases: makeCurRel("ns1", "r1"), + customResources: makeCustRes("ns1", "r1"), + want: zeromap, + }, + { + msg: "add missing release", + currentReleases: makeCurRel("ns1"), + customResources: makeCustRes("ns1", "r1"), + want: map[string][]chartRelease{"ns1": []chartRelease{ + chartRelease{releaseName: "r1", action: release.InstallAction}}}, + }, + { + msg: "add missing release new namespace", + currentReleases: makeCurRel("ns1"), + customResources: makeCustRes("ns2", "r1"), + want: map[string][]chartRelease{"ns2": []chartRelease{ + chartRelease{releaseName: "r1", action: release.InstallAction}}}, + }, + { + msg: "add missing releases multi namespace", + currentReleases: mergeCurRels(makeCurRel("ns1"), + makeCurRel("ns2", "r2")), + customResources: mergeCustRes(makeCustRes("ns1", "r1"), + makeCustRes("ns2", "r2", "r3")), + want: map[string][]chartRelease{ + "ns1": []chartRelease{chartRelease{releaseName: "r1", action: release.InstallAction}}, + "ns2": []chartRelease{chartRelease{releaseName: "r3", action: release.InstallAction}}, + }, + }, + } + + opts := cmp.AllowUnexported(chartRelease{}) + rs := New(log.NewLogfmtLogger(os.Stdout), nil) + for i, test := range tests { + var got = make(map[string][]chartRelease) + err := rs.addDeletedReleasesToSync(got, test.currentReleases, test.customResources) + if err != nil { + t.Errorf("%d %s: got error: %v", i, test.msg, err) + } + if diff := cmp.Diff(got, test.want, opts); diff != "" { + t.Errorf("%d %s: diff (-got +want)\n%s", i, test.msg, diff) + } + + } +} + +func config(vals map[string]string) *hapi_chart.Config { + pv := make(map[string]*hapi_chart.Value) + for k, v := range vals { + pv[k] = &hapi_chart.Value{Value: v} + } + + c := &hapi_chart.Config{Values: pv} + // Marshalling to get c.Raw populated + data, _ := proto.Marshal(c) + _ = proto.Unmarshal(data, c) + return c +} + +func relvals(name string, vals string) *hapi_release.Release { + rel := helm.ReleaseMock(&helm.MockReleaseOptions{Name: name}) + rel.Config.Raw = vals + return rel +} + +func relchart(name string, chartname string, chartver string, tmplname string) *hapi_release.Release { + return helm.ReleaseMock(&helm.MockReleaseOptions{Name: name, Chart: &hapi_chart.Chart{ + Metadata: &hapi_chart.Metadata{ + Name: chartname, + Version: chartver, + }, + Templates: []*hapi_chart.Template{ + {Name: tmplname, Data: []byte(helm.MockManifest)}, + }, + }}) +} + +func TestAddExistingReleasesToSync(t *testing.T) { + var zeromap = make(map[string][]chartRelease) + var tests = []struct { + msg string + currentReleases map[string]map[string]struct{} + customResources map[string]map[string]ifv1.FluxHelmRelease + want map[string][]chartRelease + releaser chartrelease.Releaser + wanterror error + }{ + { + msg: "no-op, zero resources", + currentReleases: makeCurRel("ns1", "r1"), + customResources: make(map[string]map[string]ifv1.FluxHelmRelease), + want: zeromap, + }, + { + msg: "no-op, no overlap", + currentReleases: makeCurRel("ns1", "r1"), + customResources: mergeCustRes( + makeCustRes("ns1", "r2"), + makeCustRes("ns2", "r1")), + want: zeromap, + }, + { + msg: "get deployed release fails", + currentReleases: makeCurRel("ns1", "r1"), + customResources: makeCustRes("ns1", "r1"), + releaser: &mockReleaser{}, + wanterror: fmt.Errorf("no release hamed %q", "r1"), + }, + { + msg: "dry-run install fails", + currentReleases: makeCurRel("ns1", "r1"), + customResources: makeCustRes("ns1", "r1"), + releaser: &mockReleaser{ + configSync: &helmgit.Checkout{Dir: "dir"}, + deployed: map[string]*hapi_release.Release{ + "r1": relvals("r1", `k1: "v1"`), + }, + installs: []install{ + dryinst("r1", *relvals("", ""), fmt.Errorf("dry-run failed")), + }, + }, + wanterror: fmt.Errorf("dry-run failed"), + }, + { + msg: "r1 vals changed, r2 unchanged", + currentReleases: mergeCurRels( + makeCurRel("ns1", "r1"), + makeCurRel("ns2", "r2")), + customResources: mergeCustRes( + makeCustRes("ns1", "r1"), + makeCustRes("ns2", "r2")), + releaser: &mockReleaser{ + configSync: &helmgit.Checkout{Dir: "dir"}, + deployed: map[string]*hapi_release.Release{ + "r1": relvals("r1", `k1: "v1"`), + "r2": relvals("r2", `k1: "v1"`), + }, + installs: []install{ + dryinst("r1", *relvals("r1", `k1: "v2"`), nil), + dryinst("r2", *relvals("r2", `k1: "v1"`), nil), + }, + }, + want: map[string][]chartRelease{"ns1": []chartRelease{ + chartRelease{releaseName: "r1", action: release.Action("UPDATE")}, + }}, + }, + { + msg: "r1/r2/r3 charts changed, r4 unchanged", + currentReleases: mergeCurRels(mergeCurRels( + makeCurRel("ns1", "r1"), + makeCurRel("ns2", "r2")), + makeCurRel("ns3", "r3", "r4")), + customResources: mergeCustRes(mergeCustRes( + makeCustRes("ns1", "r1"), + makeCustRes("ns2", "r2")), + makeCustRes("ns3", "r3", "r4")), + releaser: &mockReleaser{ + configSync: &helmgit.Checkout{Dir: "dir"}, + deployed: map[string]*hapi_release.Release{ + "r1": relchart("r1", "c1", "v0.1.0", "templates/foo.tpl"), + "r2": relchart("r2", "c2", "v0.1.0", "templates/bar.tpl"), + "r3": relchart("r3", "c3", "v0.1.0", "templates/baz.tpl"), + "r4": relchart("r4", "c4", "v0.1.0", "templates/qux.tpl"), + }, + installs: []install{ + dryinst("r1", *relchart("r1", "c-", "v0.1.0", "templates/foo.tpl"), nil), + dryinst("r2", *relchart("r2", "c2", "v0.1.1", "templates/bar.tpl"), nil), + dryinst("r3", *relchart("r3", "c3", "v0.1.0", "templates/foo.tpl"), nil), + dryinst("r4", *relchart("r4", "c4", "v0.1.0", "templates/qux.tpl"), nil), + }, + }, + want: map[string][]chartRelease{ + "ns1": []chartRelease{chartRelease{releaseName: "r1", action: release.Action("UPDATE")}}, + "ns2": []chartRelease{chartRelease{releaseName: "r2", action: release.Action("UPDATE")}}, + "ns3": []chartRelease{chartRelease{releaseName: "r3", action: release.Action("UPDATE")}}, + }, + }, + } + + opts := cmp.AllowUnexported(chartRelease{}) + for i, test := range tests { + rs := New(log.NewLogfmtLogger(os.Stdout), test.releaser) + var got = make(map[string][]chartRelease) + err := rs.addExistingReleasesToSync(got, test.currentReleases, test.customResources) + if fmt.Sprintf("%v", err) != fmt.Sprintf("%v", test.wanterror) { + t.Errorf("%d %s: got error %q, want error %q", i, test.msg, err, test.wanterror) + } + if test.wanterror != nil { + continue + } + if diff := cmp.Diff(got, test.want, opts); diff != "" { + t.Errorf("%d %s: diff (-got +want)\n%s", i, test.msg, diff) + } + + } +} + +func dryinst(relname string, rel hapi_release.Release, err error) install { + return install{ + installReq{ + checkoutDir: "dir", + releaseName: relname + "-temp", + action: "CREATE", + opts: chartrelease.InstallOptions{DryRun: true}, + }, + installResult{rel, err}, + } +} diff --git a/integrations/helm/releasesync/utils.go b/integrations/helm/releasesync/utils.go index 61ac77d1e..3b77b5b07 100644 --- a/integrations/helm/releasesync/utils.go +++ b/integrations/helm/releasesync/utils.go @@ -5,7 +5,10 @@ import ( "github.com/weaveworks/flux/integrations/helm/release" ) -func MappifyDeployInfo(releases map[string][]release.DeployInfo) map[string]map[string]struct{} { +// mappifyDeployInfo takes a map of namespace -> []DeployInfo, +// returning a map whose keys are the same namespaces +// and whose values are key-only maps holding the DeployInfo names. +func mappifyDeployInfo(releases map[string][]release.DeployInfo) map[string]map[string]struct{} { deployM := make(map[string]map[string]struct{}) for ns, nsRels := range releases { @@ -18,7 +21,10 @@ func MappifyDeployInfo(releases map[string][]release.DeployInfo) map[string]map[ return deployM } -func MappifyReleaseFhrInfo(fhrs map[string][]ReleaseFhr) map[string]map[string]ifv1.FluxHelmRelease { +// mappifyReleaseFhrInfo takes a map of namespace -> []releaseFhr, +// returning a map whose keys are the same namespaces +// and whose values are maps of releaseName -> FluxHelmRelease. +func mappifyReleaseFhrInfo(fhrs map[string][]releaseFhr) map[string]map[string]ifv1.FluxHelmRelease { relFhrM := make(map[string]map[string]ifv1.FluxHelmRelease) for ns, nsFhrs := range fhrs {