kio.go

// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

// Package kio contains low-level libraries for reading, modifying and writing
// Resource Configuration and packages.
package kio

import (
	"fmt"
	"strconv"

	"sigs.k8s.io/kustomize/kyaml/errors"
	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

// Reader reads ResourceNodes. Analogous to io.Reader.
type Reader interface {
	Read() ([]*yaml.RNode, error)
}

// ResourceNodeSlice is a collection of ResourceNodes.
// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
// Readers, Filters or Writers may have constraints.
type ResourceNodeSlice []*yaml.RNode

var _ Reader = ResourceNodeSlice{}

func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
	return o, nil
}

// Writer writes ResourceNodes. Analogous to io.Writer.
type Writer interface {
	Write([]*yaml.RNode) error
}

// WriterFunc implements a Writer as a function.
type WriterFunc func([]*yaml.RNode) error

func (fn WriterFunc) Write(o []*yaml.RNode) error {
	return fn(o)
}

// ReaderWriter implements both Reader and Writer interfaces
type ReaderWriter interface {
	Reader
	Writer
}

// Filter modifies a collection of Resource Configuration by returning the modified slice.
// When possible, Filters should be serializable to yaml so that they can be described
// as either data or code.
//
// Analogous to http://www.linfo.org/filters.html
type Filter interface {
	Filter([]*yaml.RNode) ([]*yaml.RNode, error)
}

// TrackableFilter is an extension of Filter which is also capable of tracking
// which fields were mutated by the filter.
type TrackableFilter interface {
	Filter
	WithMutationTracker(func(key, value, tag string, node *yaml.RNode))
}

// FilterFunc implements a Filter as a function.
type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)

func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
	return fn(o)
}

// Pipeline reads Resource Configuration from a set of Inputs, applies some
// transformation filters, and writes the results to a set of Outputs.
//
// Analogous to http://www.linfo.org/pipes.html
type Pipeline struct {
	// Inputs provide sources for Resource Configuration to be read.
	Inputs []Reader `yaml:"inputs,omitempty"`

	// Filters are transformations applied to the Resource Configuration.
	// They are applied in the order they are specified.
	// Analogous to http://www.linfo.org/filters.html
	Filters []Filter `yaml:"filters,omitempty"`

	// Outputs are where the transformed Resource Configuration is written.
	Outputs []Writer `yaml:"outputs,omitempty"`

	// ContinueOnEmptyResult configures what happens when a filter in the pipeline
	// returns an empty result.
	// If it is false (default), subsequent filters will be skipped and the result
	// will be returned immediately. This is useful as an optimization when you
	// know that subsequent filters will not alter the empty result.
	// If it is true, the empty result will be provided as input to the next
	// filter in the list. This is useful when subsequent functions in the
	// pipeline may generate new resources.
	ContinueOnEmptyResult bool `yaml:"continueOnEmptyResult,omitempty"`
}

// Execute executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline.
func (p Pipeline) Execute() error {
	return p.ExecuteWithCallback(nil)
}

// PipelineExecuteCallbackFunc defines a callback function that will be called each time a step in the pipeline succeeds.
type PipelineExecuteCallbackFunc = func(op Filter)

// ExecuteWithCallback executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline. The callback will be called each time a step succeeds.
func (p Pipeline) ExecuteWithCallback(callback PipelineExecuteCallbackFunc) error {
	var result []*yaml.RNode

	// read from the inputs
	for _, i := range p.Inputs {
		nodes, err := i.Read()
		if err != nil {
			return errors.Wrap(err)
		}
		result = append(result, nodes...)
	}

	// apply operations
	for i := range p.Filters {
		// Not all RNodes passed through kio.Pipeline have metadata nor should
		// they all be required to.
		nodeAnnos, err := PreprocessResourcesForInternalAnnotationMigration(result)
		if err != nil {
			return err
		}

		op := p.Filters[i]
		if callback != nil {
			callback(op)
		}
		result, err = op.Filter(result)
		// TODO (issue 2872): This len(result) == 0 should be removed and empty result list should be
		// handled by outputs. However currently some writer like LocalPackageReadWriter
		// will clear the output directory and which will cause unpredictable results
		if len(result) == 0 && !p.ContinueOnEmptyResult || err != nil {
			return errors.Wrap(err)
		}

		// If either the internal annotations for path, index, and id OR the legacy
		// annotations for path, index, and id are changed, we have to update the other.
		err = ReconcileInternalAnnotations(result, nodeAnnos)
		if err != nil {
			return err
		}
	}

	// write to the outputs
	for _, o := range p.Outputs {
		if err := o.Write(result); err != nil {
			return errors.Wrap(err)
		}
	}
	return nil
}

// FilterAll runs the yaml.Filter against all inputs
func FilterAll(filter yaml.Filter) Filter {
	return FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
		for i := range nodes {
			_, err := filter.Filter(nodes[i])
			if err != nil {
				return nil, errors.Wrap(err)
			}
		}
		return nodes, nil
	})
}

// PreprocessResourcesForInternalAnnotationMigration returns a mapping from id to all
// internal annotations, so that we can use it to reconcile the annotations
// later. This is necessary because currently both internal-prefixed annotations
// and legacy annotations are currently supported, and a change to one must be
// reflected in the other if needed.
func PreprocessResourcesForInternalAnnotationMigration(result []*yaml.RNode) (map[string]map[string]string, error) {
	idToAnnosMap := make(map[string]map[string]string)
	for i := range result {
		idStr := strconv.Itoa(i)
		err := result[i].PipeE(yaml.SetAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation, idStr))
		if err != nil {
			return nil, err
		}
		idToAnnosMap[idStr] = kioutil.GetInternalAnnotations(result[i])
		if err = kioutil.CopyLegacyAnnotations(result[i]); err != nil {
			return nil, err
		}
		meta, _ := result[i].GetMeta()
		if err = checkMismatchedAnnos(meta.Annotations); err != nil {
			return nil, err
		}
	}
	return idToAnnosMap, nil
}

func checkMismatchedAnnos(annotations map[string]string) error {
	path := annotations[kioutil.PathAnnotation]
	index := annotations[kioutil.IndexAnnotation]
	id := annotations[kioutil.IdAnnotation]

	legacyPath := annotations[kioutil.LegacyPathAnnotation]
	legacyIndex := annotations[kioutil.LegacyIndexAnnotation]
	legacyId := annotations[kioutil.LegacyIdAnnotation]

	// if prior to running the functions, the legacy and internal annotations differ,
	// throw an error as we cannot infer the user's intent.
	if path != "" && legacyPath != "" && path != legacyPath {
		return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
	}
	if index != "" && legacyIndex != "" && index != legacyIndex {
		return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
	}
	if id != "" && legacyId != "" && id != legacyId {
		return fmt.Errorf("resource input to function has mismatched legacy and internal id annotations")
	}
	return nil
}

type nodeAnnotations struct {
	path  string
	index string
	id    string
}

// ReconcileInternalAnnotations reconciles the annotation format for path, index and id annotations.
// It will ensure the output annotation format matches the format in the input. e.g. if the input
// format uses the legacy format and the output will be converted to the legacy format if it's not.
func ReconcileInternalAnnotations(result []*yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
	useInternal, useLegacy, err := determineAnnotationsFormat(nodeAnnosMap)
	if err != nil {
		return err
	}

	for i := range result {
		// if only one annotation is set, set the other.
		err = missingInternalOrLegacyAnnotations(result[i])
		if err != nil {
			return err
		}
		// we must check to see if the function changed either the new internal annotations
		// or the old legacy annotations. If one is changed, the change must be reflected
		// in the other.
		err = checkAnnotationsAltered(result[i], nodeAnnosMap)
		if err != nil {
			return err
		}
		// We invoke determineAnnotationsFormat to find out if the original annotations
		// use the internal or (and) the legacy format. We format the resources to
		// make them consistent with original format.
		err = formatInternalAnnotations(result[i], useInternal, useLegacy)
		if err != nil {
			return err
		}
		// if the annotations are still somehow out of sync, throw an error
		meta, _ := result[i].GetMeta()
		err = checkMismatchedAnnos(meta.Annotations)
		if err != nil {
			return err
		}

		if _, err = result[i].Pipe(yaml.ClearAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation)); err != nil {
			return err
		}
	}
	return nil
}

// determineAnnotationsFormat determines if the resources are using one of the internal and legacy annotation format or both of them.
func determineAnnotationsFormat(nodeAnnosMap map[string]map[string]string) (bool, bool, error) {
	var useInternal, useLegacy bool
	var err error

	if len(nodeAnnosMap) == 0 {
		return true, true, nil
	}

	var internal, legacy *bool
	for _, annos := range nodeAnnosMap {
		_, foundPath := annos[kioutil.PathAnnotation]
		_, foundIndex := annos[kioutil.IndexAnnotation]
		_, foundId := annos[kioutil.IdAnnotation]
		_, foundLegacyPath := annos[kioutil.LegacyPathAnnotation]
		_, foundLegacyIndex := annos[kioutil.LegacyIndexAnnotation]
		_, foundLegacyId := annos[kioutil.LegacyIdAnnotation]

		if !(foundPath || foundIndex || foundId || foundLegacyPath || foundLegacyIndex || foundLegacyId) {
			continue
		}

		foundOneOf := foundPath || foundIndex || foundId
		if internal == nil {
			f := foundOneOf
			internal = &f
		}
		if (foundOneOf && !*internal) || (!foundOneOf && *internal) {
			err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
			return useInternal, useLegacy, err
		}

		foundOneOf = foundLegacyPath || foundLegacyIndex || foundLegacyId
		if legacy == nil {
			f := foundOneOf
			legacy = &f
		}
		if (foundOneOf && !*legacy) || (!foundOneOf && *legacy) {
			err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
			return useInternal, useLegacy, err
		}
	}
	if internal != nil {
		useInternal = *internal
	}
	if legacy != nil {
		useLegacy = *legacy
	}
	return useInternal, useLegacy, err
}

func missingInternalOrLegacyAnnotations(rn *yaml.RNode) error {
	if err := missingInternalOrLegacyAnnotation(rn, kioutil.PathAnnotation, kioutil.LegacyPathAnnotation); err != nil {
		return err
	}
	if err := missingInternalOrLegacyAnnotation(rn, kioutil.IndexAnnotation, kioutil.LegacyIndexAnnotation); err != nil {
		return err
	}
	if err := missingInternalOrLegacyAnnotation(rn, kioutil.IdAnnotation, kioutil.LegacyIdAnnotation); err != nil {
		return err
	}
	return nil
}

func missingInternalOrLegacyAnnotation(rn *yaml.RNode, newKey string, legacyKey string) error {
	meta, _ := rn.GetMeta()
	annotations := meta.Annotations
	value := annotations[newKey]
	legacyValue := annotations[legacyKey]

	if value == "" && legacyValue == "" {
		// do nothing
		return nil
	}

	if value == "" {
		// new key is not set, copy from legacy key
		if err := rn.PipeE(yaml.SetAnnotation(newKey, legacyValue)); err != nil {
			return err
		}
	} else if legacyValue == "" {
		// legacy key is not set, copy from new key
		if err := rn.PipeE(yaml.SetAnnotation(legacyKey, value)); err != nil {
			return err
		}
	}
	return nil
}

func checkAnnotationsAltered(rn *yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
	meta, _ := rn.GetMeta()
	annotations := meta.Annotations
	// get the resource's current path, index, and ids from the new annotations
	internal := nodeAnnotations{
		path:  annotations[kioutil.PathAnnotation],
		index: annotations[kioutil.IndexAnnotation],
		id:    annotations[kioutil.IdAnnotation],
	}

	// get the resource's current path, index, and ids from the legacy annotations
	legacy := nodeAnnotations{
		path:  annotations[kioutil.LegacyPathAnnotation],
		index: annotations[kioutil.LegacyIndexAnnotation],
		id:    annotations[kioutil.LegacyIdAnnotation],
	}

	rid := annotations[kioutil.InternalAnnotationsMigrationResourceIDAnnotation]
	originalAnnotations, found := nodeAnnosMap[rid]
	if !found {
		return nil
	}
	originalPath, found := originalAnnotations[kioutil.PathAnnotation]
	if !found {
		originalPath = originalAnnotations[kioutil.LegacyPathAnnotation]
	}
	if originalPath != "" {
		switch {
		case originalPath != internal.path && originalPath != legacy.path && internal.path != legacy.path:
			return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
		case originalPath != internal.path:
			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyPathAnnotation, internal.path)); err != nil {
				return err
			}
		case originalPath != legacy.path:
			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, legacy.path)); err != nil {
				return err
			}
		}
	}

	originalIndex, found := originalAnnotations[kioutil.IndexAnnotation]
	if !found {
		originalIndex = originalAnnotations[kioutil.LegacyIndexAnnotation]
	}
	if originalIndex != "" {
		switch {
		case originalIndex != internal.index && originalIndex != legacy.index && internal.index != legacy.index:
			return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
		case originalIndex != internal.index:
			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyIndexAnnotation, internal.index)); err != nil {
				return err
			}
		case originalIndex != legacy.index:
			if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.IndexAnnotation, legacy.index)); err != nil {
				return err
			}
		}
	}
	return nil
}

func formatInternalAnnotations(rn *yaml.RNode, useInternal, useLegacy bool) error {
	if !useInternal {
		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IdAnnotation)); err != nil {
			return err
		}
		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.PathAnnotation)); err != nil {
			return err
		}
		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IndexAnnotation)); err != nil {
			return err
		}
	}
	if !useLegacy {
		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIdAnnotation)); err != nil {
			return err
		}
		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyPathAnnotation)); err != nil {
			return err
		}
		if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIndexAnnotation)); err != nil {
			return err
		}
	}
	return nil
}