Skip to content

Commit

Permalink
support updating primary labels/annotations after first time rollout
Browse files Browse the repository at this point in the history 
Signed-off-by: hei4290 <[email protected]>
  • Loading branch information
@kh34
kh34 committed Oct 27, 2021
1 parent c02477a commit e712c6b
Show file tree
Hide file tree
Showing 8 changed files with 283 additions and 46 deletions.
33 changes: 33 additions & 0 deletions artifacts/flagger/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,38 @@ spec:
type: object
additionalProperties:
type: string
deployment:
description: Kubernetes Deployment spec
type: object
properties:
primary:
description: Metadata to add to the primary deployment
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
autoscaler:
description: Kubernetes Autoscaler spec
type: object
properties:
primary:
description: Metadata to add to the primary autoscaler
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
skipAnalysis:
description: Skip analysis and promote canary
type: boolean
Expand Down Expand Up @@ -1108,6 +1140,7 @@ spec:
- cloudwatch
- newrelic
- graphite
- dynatrace
address:
description: API address of this provider
type: string
Expand Down
32 changes: 32 additions & 0 deletions charts/flagger/crds/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,38 @@ spec:
type: object
additionalProperties:
type: string
deployment:
description: Kubernetes Deployment spec
type: object
properties:
primary:
description: Metadata to add to the primary deployment
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
autoscaler:
description: Kubernetes Autoscaler spec
type: object
properties:
primary:
description: Metadata to add to the primary autoscaler
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
skipAnalysis:
description: Skip analysis and promote canary
type: boolean
Expand Down
32 changes: 32 additions & 0 deletions kustomize/base/flagger/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,38 @@ spec:
type: object
additionalProperties:
type: string
deployment:
description: Kubernetes Deployment spec
type: object
properties:
primary:
description: Metadata to add to the primary deployment
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
autoscaler:
description: Kubernetes Autoscaler spec
type: object
properties:
primary:
description: Metadata to add to the primary autoscaler
type: object
properties:
labels:
type: object
additionalProperties:
type: string
annotations:
type: object
additionalProperties:
type: string
skipAnalysis:
description: Skip analysis and promote canary
type: boolean
Expand Down
22 changes: 22 additions & 0 deletions pkg/apis/flagger/v1beta1/canary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ type CanarySpec struct {
// Service defines how ClusterIP services, service mesh or ingress routing objects are generated
Service CanaryService `json:"service"`

// Service defines how deployments are generated
Deployment CanaryDeployment `json:"deployment"`

// Service defines how autoscaler are generated
Autoscaler CanaryAutoscaler `json:"autoscaler"`

// Analysis defines the validation process of a release
Analysis *CanaryAnalysis `json:"analysis,omitempty"`

Expand Down Expand Up @@ -193,6 +199,22 @@ type CanaryService struct {
Canary *CustomMetadata `json:"canary,omitempty"`
}

// CanaryDeployment defines how deployments are generated
type CanaryDeployment struct {

// Primary is the metadata to add to the primary deployment
// +optional
Primary *CustomMetadata `json:"primary,omitempty"`
}

// CanaryDeployment defines how deployments are generated
type CanaryAutoscaler struct {

// Primary is the metadata to add to the primary autoscaler
// +optional
Primary *CustomMetadata `json:"primary,omitempty"`
}

// CanaryAnalysis is used to describe how the analysis should be done
type CanaryAnalysis struct {
// Schedule interval for this canary analysis
Expand Down
44 changes: 44 additions & 0 deletions pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

138 changes: 92 additions & 46 deletions pkg/canary/deployment_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"

flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
Expand Down Expand Up @@ -84,54 +85,72 @@ func (c *DeploymentController) Promote(cd *flaggerv1.Canary) error {
targetName := cd.Spec.TargetRef.Name
primaryName := fmt.Sprintf("%s-primary", targetName)

canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
}
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
}

label, labelValue, err := c.getSelectorLabel(canary)
primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
if err != nil {
return fmt.Errorf("getSelectorLabel failed: %w", err)
}
label, labelValue, err := c.getSelectorLabel(canary)
primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
if err != nil {
return fmt.Errorf("getSelectorLabel failed: %w", err)
}

primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
}
primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
}

// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return fmt.Errorf("GetTargetConfigs failed: %w", err)
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
}
// promote secrets and config maps
configRefs, err := c.configTracker.GetTargetConfigs(cd)
if err != nil {
return fmt.Errorf("GetTargetConfigs failed: %w", err)
}
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
}

primaryCopy := primary.DeepCopy()
primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.Strategy = canary.Spec.Strategy
primaryCopy := primary.DeepCopy()
primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
primaryCopy.Spec.Strategy = canary.Spec.Strategy

// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)
// update spec with primary secrets and config maps
primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)

// update pod annotations to ensure a rolling update
annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
if err != nil {
return fmt.Errorf("makeAnnotations failed: %w", err)
}
// update pod annotations to ensure a rolling update
annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
if err != nil {
return fmt.Errorf("makeAnnotations failed: %w", err)
}

primaryCopy.Spec.Template.Annotations = annotations
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
primaryCopy.Spec.Template.Annotations = annotations
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)

// apply update
_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
if cd.Spec.Deployment.Primary != nil {
if len(primaryCopy.ObjectMeta.Annotations) == 0 {
primaryCopy.ObjectMeta.Annotations = make(map[string]string)
}
for k, v := range cd.Spec.Deployment.Primary.Annotations {
primaryCopy.ObjectMeta.Annotations[k] = v
}
if len(primaryCopy.ObjectMeta.Labels) == 0 {
primaryCopy.ObjectMeta.Labels = make(map[string]string)
}
for k, v := range cd.Spec.Deployment.Primary.Labels {
primaryCopy.ObjectMeta.Labels[k] = v
}
}

// apply update
_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
return err
})
if err != nil {
return fmt.Errorf("updating deployment %s.%s template spec failed: %w",
primaryCopy.GetName(), primaryCopy.Namespace, err)
primaryName, cd.Namespace, err)
}

// update HPA
Expand Down Expand Up @@ -364,18 +383,45 @@ func (c *DeploymentController) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bo
if !init && primaryHpa != nil {
diffMetrics := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics)
diffBehavior := cmp.Diff(hpaSpec.Behavior, primaryHpa.Spec.Behavior)
if diffMetrics != "" || diffBehavior != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
diffLabels := cmp.Diff(hpa.ObjectMeta.Labels, primaryHpa.ObjectMeta.Labels)
diffAnnotations := cmp.Diff(hpa.ObjectMeta.Annotations, primaryHpa.ObjectMeta.Annotations)
if diffMetrics != "" || diffBehavior != "" || diffLabels != "" || diffAnnotations != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
fmt.Println(diffMetrics, diffBehavior, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas)
hpaClone := primaryHpa.DeepCopy()
hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
hpaClone.Spec.Metrics = hpaSpec.Metrics
hpaClone.Spec.Behavior = hpaSpec.Behavior

_, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{})
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
primaryHpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
if err != nil {
return err
}

hpaClone := primaryHpa.DeepCopy()
hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
hpaClone.Spec.Metrics = hpaSpec.Metrics
hpaClone.Spec.Behavior = hpaSpec.Behavior

if cd.Spec.Autoscaler.Primary != nil {
if len(hpaClone.ObjectMeta.Annotations) == 0 {
hpaClone.ObjectMeta.Annotations = make(map[string]string)
}
for k, v := range cd.Spec.Autoscaler.Primary.Annotations {
hpaClone.ObjectMeta.Annotations[k] = v
}
if len(hpaClone.ObjectMeta.Labels) == 0 {
hpaClone.ObjectMeta.Labels = make(map[string]string)
}
for k, v := range cd.Spec.Autoscaler.Primary.Labels {
hpaClone.ObjectMeta.Labels[k] = v
}
}

_, err = c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{})
return err
})

if err != nil {
return fmt.Errorf("updating HorizontalPodAutoscaler %s.%s failed: %w",
hpaClone.Name, hpaClone.Namespace, err)
primaryHpa.Name, primaryHpa.Namespace, err)
}
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
Infof("HorizontalPodAutoscaler %s.%s updated", primaryHpa.GetName(), cd.Namespace)
Expand Down
Loading

0 comments on commit e712c6b

Please sign in to comment.