diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 96dc77271ff76..fc02ad756646a 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -217,6 +217,10 @@ spec: image: description: Image is the instance (ami etc) we should use type: string + instanceInterruptionBehavior: + description: InstanceInterruptionBehavior defines if a spot instance + should be terminated, hibernated, or stopped after interruption + type: string instanceProtection: description: InstanceProtection makes new instances in an autoscaling group protected from scale in diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 37afbc4891ac9..58700c8f346d1 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -161,6 +161,9 @@ type InstanceGroupSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the rolling-update behavior RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + // InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated, + // or stopped after interruption + InstanceInterruptionBehavior *string `json:"instanceInterruptionBehavior,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 6812d58194f60..d3a055814c7b8 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -157,6 +157,9 @@ type InstanceGroupSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the rolling-update behavior RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + // InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated, + // or stopped after interruption + InstanceInterruptionBehavior *string `json:"instanceInterruptionBehavior,omitempty"` } const ( diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 8b20fa1783bfd..41466502c6be5 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -3366,6 +3366,7 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan } else { out.RollingUpdate = nil } + out.InstanceInterruptionBehavior = in.InstanceInterruptionBehavior return nil } @@ -3504,6 +3505,7 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I } else { out.RollingUpdate = nil } + out.InstanceInterruptionBehavior = in.InstanceInterruptionBehavior return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 7343504c69766..1cf06326f65f5 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1787,6 +1787,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(RollingUpdate) (*in).DeepCopyInto(*out) } + if in.InstanceInterruptionBehavior != nil { + in, out := &in.InstanceInterruptionBehavior, &out.InstanceInterruptionBehavior + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go index 00267b89310d9..e9a64867e8de3 100644 --- a/pkg/apis/kops/validation/aws.go +++ b/pkg/apis/kops/validation/aws.go @@ -51,6 +51,8 @@ func awsValidateInstanceGroup(ig *kops.InstanceGroup) field.ErrorList { allErrs = append(allErrs, awsValidateSpotDurationInMinute(field.NewPath(ig.GetName(), "spec", "spotDurationInMinutes"), ig)...) + allErrs = append(allErrs, awsValidateInstanceInterruptionBehavior(field.NewPath(ig.GetName(), "spec", "instanceInterruptionBehavior"), ig)...) + return allErrs } @@ -120,3 +122,13 @@ func awsValidateSpotDurationInMinute(fieldPath *field.Path, ig *kops.InstanceGro } return allErrs } + +func awsValidateInstanceInterruptionBehavior(fieldPath *field.Path, ig *kops.InstanceGroup) field.ErrorList { + allErrs := field.ErrorList{} + if ig.Spec.InstanceInterruptionBehavior != nil { + validInterruptionBehaviors := []string{"terminate", "hibernate", "stop"} + instanceInterruptionBehavior := *ig.Spec.InstanceInterruptionBehavior + allErrs = append(allErrs, IsValidValue(fieldPath, &instanceInterruptionBehavior, validInterruptionBehaviors)...) + } + return allErrs +} diff --git a/pkg/apis/kops/validation/aws_test.go b/pkg/apis/kops/validation/aws_test.go index 20cd30636c59f..9b496015c1dd1 100644 --- a/pkg/apis/kops/validation/aws_test.go +++ b/pkg/apis/kops/validation/aws_test.go @@ -134,6 +134,32 @@ func TestValidateInstanceGroupSpec(t *testing.T) { }, ExpectedErrors: []string{}, }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("invalidValue"), + }, + ExpectedErrors: []string{ + "Unsupported value::test-nodes.spec.instanceInterruptionBehavior", + }, + }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("terminate"), + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("hibernate"), + }, + ExpectedErrors: []string{}, + }, + { + Input: kops.InstanceGroupSpec{ + InstanceInterruptionBehavior: fi.String("stop"), + }, + ExpectedErrors: []string{}, + }, } for _, g := range grid { ig := &kops.InstanceGroup{ diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 629784685e544..8c7b4f4551994 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1953,6 +1953,11 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) { *out = new(RollingUpdate) (*in).DeepCopyInto(*out) } + if in.InstanceInterruptionBehavior != nil { + in, out := &in.InstanceInterruptionBehavior, &out.InstanceInterruptionBehavior + *out = new(string) + **out = **in + } return } diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index dcc611b4a6c61..ab2438ce8c514 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -134,6 +134,9 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde if ig.Spec.SpotDurationInMinutes != nil { lt.SpotDurationInMinutes = ig.Spec.SpotDurationInMinutes } + if ig.Spec.InstanceInterruptionBehavior != nil { + lt.InstanceInterruptionBehavior = ig.Spec.InstanceInterruptionBehavior + } return lt, nil } diff --git a/tests/integration/update_cluster/launch_templates/cloudformation.json b/tests/integration/update_cluster/launch_templates/cloudformation.json index cfb4a0e8418e3..2033fc77f1a33 100644 --- a/tests/integration/update_cluster/launch_templates/cloudformation.json +++ b/tests/integration/update_cluster/launch_templates/cloudformation.json @@ -606,6 +606,7 @@ "MarketType": "spot", "SpotOptions": { "BlockDurationMinutes": 120, + "InstanceInterruptionBehavior": "hibernate", "MaxPrice": "0.1" } }, diff --git a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml index 44f663c28ffaa..8d58f039a9c7f 100644 --- a/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/launch_templates/in-v1alpha2.yaml @@ -73,6 +73,7 @@ spec: instanceProtection: true maxPrice: "0.1" spotDurationInMinutes: 120 + instanceInterruptionBehavior: "hibernate" subnets: - us-test-1b --- diff --git a/tests/integration/update_cluster/launch_templates/kubernetes.tf b/tests/integration/update_cluster/launch_templates/kubernetes.tf index f7abc21cc6001..e7cefff781c04 100644 --- a/tests/integration/update_cluster/launch_templates/kubernetes.tf +++ b/tests/integration/update_cluster/launch_templates/kubernetes.tf @@ -530,8 +530,9 @@ resource "aws_launch_template" "nodes-launchtemplates-example-com" { instance_market_options { market_type = "spot" spot_options { - block_duration_minutes = 120 - max_price = "0.1" + block_duration_minutes = 120 + instance_interruption_behavior = "hibernate" + max_price = "0.1" } } instance_type = "t3.medium" diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate.go index a55907e0d59af..664a5ec9d6c40 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate.go @@ -70,6 +70,9 @@ type LaunchTemplate struct { Tenancy *string // UserData is the user data configuration UserData *fi.ResourceHolder + // InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated, + // or stopped after interruption + InstanceInterruptionBehavior *string } var ( diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_api.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_api.go index 7932f97fe8c86..d5f638994bc22 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_api.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_api.go @@ -128,7 +128,15 @@ func (t *LaunchTemplate) RenderAWS(c *awsup.AWSAPITarget, a, ep, changes *Launch } lc.UserData = aws.String(base64.StdEncoding.EncodeToString(d)) } - + // @step: add instanceInterruptionBehavior + if t.InstanceInterruptionBehavior != nil { + s := &ec2.LaunchTemplateSpotMarketOptionsRequest{ + InstanceInterruptionBehavior: t.InstanceInterruptionBehavior, + } + lc.InstanceMarketOptions = &ec2.LaunchTemplateInstanceMarketOptionsRequest{ + SpotOptions: s, + } + } // @step: attempt to create the launch template err = func() error { for attempt := 0; attempt < 10; attempt++ { @@ -223,6 +231,10 @@ func (t *LaunchTemplate) Find(c *fi.Context) (*LaunchTemplate, error) { if lt.LaunchTemplateData.IamInstanceProfile != nil { actual.IAMInstanceProfile = &IAMInstanceProfile{Name: lt.LaunchTemplateData.IamInstanceProfile.Name} } + // @step: add instanceInterruptionBehavior if there is one + if lt.LaunchTemplateData.InstanceMarketOptions != nil && lt.LaunchTemplateData.InstanceMarketOptions.SpotOptions != nil { + actual.InstanceInterruptionBehavior = lt.LaunchTemplateData.InstanceMarketOptions.SpotOptions.InstanceInterruptionBehavior + } // @step: get the image is order to find out the root device name as using the index // is not variable, under conditions they move diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go index 39f3a0d75fd0a..c621f75c2b7f2 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation.go @@ -64,8 +64,8 @@ type cloudformationLaunchTemplateIAMProfile struct { type cloudformationLaunchTemplateMarketOptionsSpotOptions struct { // BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60. BlockDurationMinutes *int64 `json:"BlockDurationMinutes,omitempty"` - // InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate - InstancesInterruptionBehavior *string `json:"InstancesInterruptionBehavior,omitempty"` + // InstanceInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate + InstanceInterruptionBehavior *string `json:"InstanceInterruptionBehavior,omitempty"` // MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances MaxPrice *string `json:"MaxPrice,omitempty"` // SpotInstanceType is the Spot Instance request type. Can be one-time, or persistent @@ -185,6 +185,9 @@ func (t *LaunchTemplate) RenderCloudformation(target *cloudformation.Cloudformat if e.SpotDurationInMinutes != nil { marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes } + if e.InstanceInterruptionBehavior != nil { + marketSpotOptions.InstanceInterruptionBehavior = e.InstanceInterruptionBehavior + } launchTemplateData.MarketOptions = &cloudformationLaunchTemplateMarketOptions{MarketType: fi.String("spot"), SpotOptions: &marketSpotOptions} } diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go index 427f96a583f91..aa14b6587ea8a 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_cloudformation_test.go @@ -31,14 +31,15 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) { IAMInstanceProfile: &IAMInstanceProfile{ Name: fi.String("nodes"), }, - ID: fi.String("test-11"), - InstanceMonitoring: fi.Bool(true), - InstanceType: fi.String("t2.medium"), - RootVolumeOptimization: fi.Bool(true), - RootVolumeIops: fi.Int64(100), - RootVolumeSize: fi.Int64(64), - SpotPrice: "10", - SpotDurationInMinutes: fi.Int64(120), + ID: fi.String("test-11"), + InstanceMonitoring: fi.Bool(true), + InstanceType: fi.String("t2.medium"), + RootVolumeOptimization: fi.Bool(true), + RootVolumeIops: fi.Int64(100), + RootVolumeSize: fi.Int64(64), + SpotPrice: "10", + SpotDurationInMinutes: fi.Int64(120), + InstanceInterruptionBehavior: fi.String("hibernate"), SSHKey: &SSHKey{ Name: fi.String("mykey"), }, @@ -67,6 +68,7 @@ func TestLaunchTemplateCloudformationRender(t *testing.T) { "MarketType": "spot", "SpotOptions": { "BlockDurationMinutes": 120, + "InstanceInterruptionBehavior": "hibernate", "MaxPrice": "10" } }, diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go index ebe60d4d7af14..276a8d27c7e2a 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go @@ -61,8 +61,8 @@ type terraformLaunchTemplateIAMProfile struct { type terraformLaunchTemplateMarketOptionsSpotOptions struct { // BlockDurationMinutes is required duration in minutes. This value must be a multiple of 60. BlockDurationMinutes *int64 `json:"block_duration_minutes,omitempty" cty:"block_duration_minutes"` - // InstancesInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate - InstancesInterruptionBehavior *string `json:"instances_interruption_behavior,omitempty" cty:"instances_interruption_behavior"` + // InstanceInterruptionBehavior is the behavior when a Spot Instance is interrupted. Can be hibernate, stop, or terminate + InstanceInterruptionBehavior *string `json:"instance_interruption_behavior,omitempty" cty:"instance_interruption_behavior"` // MaxPrice is the maximum hourly price you're willing to pay for the Spot Instances MaxPrice *string `json:"max_price,omitempty" cty:"max_price"` // SpotInstanceType is the Spot Instance request type. Can be one-time, or persistent @@ -183,6 +183,9 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e if e.SpotDurationInMinutes != nil { marketSpotOptions.BlockDurationMinutes = e.SpotDurationInMinutes } + if e.InstanceInterruptionBehavior != nil { + marketSpotOptions.InstanceInterruptionBehavior = e.InstanceInterruptionBehavior + } tf.MarketOptions = []*terraformLaunchTemplateMarketOptions{ { MarketType: fi.String("spot"), diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go index e0016a4e666a6..3d2c76a4092fe 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform_test.go @@ -31,14 +31,15 @@ func TestLaunchTemplateTerraformRender(t *testing.T) { IAMInstanceProfile: &IAMInstanceProfile{ Name: fi.String("nodes"), }, - ID: fi.String("test-11"), - InstanceMonitoring: fi.Bool(true), - InstanceType: fi.String("t2.medium"), - SpotPrice: "0.1", - SpotDurationInMinutes: fi.Int64(60), - RootVolumeOptimization: fi.Bool(true), - RootVolumeIops: fi.Int64(100), - RootVolumeSize: fi.Int64(64), + ID: fi.String("test-11"), + InstanceMonitoring: fi.Bool(true), + InstanceType: fi.String("t2.medium"), + SpotPrice: "0.1", + SpotDurationInMinutes: fi.Int64(60), + InstanceInterruptionBehavior: fi.String("hibernate"), + RootVolumeOptimization: fi.Bool(true), + RootVolumeIops: fi.Int64(100), + RootVolumeSize: fi.Int64(64), SSHKey: &SSHKey{ Name: fi.String("newkey"), PublicKey: fi.WrapResource(fi.NewStringResource("newkey")), @@ -61,8 +62,9 @@ resource "aws_launch_template" "test" { instance_market_options { market_type = "spot" spot_options { - block_duration_minutes = 60 - max_price = "0.1" + block_duration_minutes = 60 + instance_interruption_behavior = "hibernate" + max_price = "0.1" } } instance_type = "t2.medium"