diff --git a/pkg/cluster-runtime/installer.go b/pkg/cluster-runtime/installer.go index 5c4c2567eb0..234fcf7a251 100644 --- a/pkg/cluster-runtime/installer.go +++ b/pkg/cluster-runtime/installer.go @@ -188,6 +188,19 @@ func (i *Installer) Install() error { return err } + for _, ip := range all { + taints := i.infraDriver.GetHostTaint(ip) + for _, taint := range taints { + newTaint := NewTaint() + if err := newTaint.FormatData(ip, taint); err != nil { + return err + } + if err := UpdateTaint(newTaint, ip); err != nil { + return err + } + } + } + appInstaller := NewAppInstaller(i.infraDriver, i.Distributor, extension, i.RegistryConfig) if err = appInstaller.LaunchClusterImage(master0, cmds); err != nil { diff --git a/pkg/cluster-runtime/taint.go b/pkg/cluster-runtime/taint.go new file mode 100644 index 00000000000..7fb79060700 --- /dev/null +++ b/pkg/cluster-runtime/taint.go @@ -0,0 +1,196 @@ +// Copyright © 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterruntime + +import ( + "fmt" + "net" + "strings" + + "github.com/sealerio/sealer/pkg/client/k8s" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" +) + +const ( + DelSymbol = "-" + EqualSymbol = "=" + ColonSymbol = ":" +) + +var TaintEffectValues = []v1.TaintEffect{v1.TaintEffectNoSchedule, v1.TaintEffectNoExecute, v1.TaintEffectPreferNoSchedule} + +type TaintList map[string]*taintList //map[ip]taint + +type Taint struct { + IPList []string + TaintList +} + +type taintList struct { + DelTaintList []v1.Taint + AddTaintList []v1.Taint +} + +func newTaintStruct(key, value, effect string) v1.Taint { + return v1.Taint{Key: key, Value: value, Effect: v1.TaintEffect(effect)} +} + +// FormatData key1=value1:NoSchedule;key1=value1:NoSchedule-; +//key1:NoSchedule;key1:NoSchedule-; +//key1=:NoSchedule-;key1=value1:NoSchedule +func (l *Taint) FormatData(ip net.IP, data string) error { + var key, value, effect string + items := strings.Split(data, "\n") + if l.TaintList == nil { + l.TaintList = make(map[string]*taintList) + } + for _, v := range items { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "#") || v == "" { + continue + } + if strings.Contains(v, EqualSymbol) && !strings.Contains(v, EqualSymbol+ColonSymbol) { + temps := strings.Split(v, EqualSymbol) + if len(temps) != 2 { + return fmt.Errorf("faild to split taint argument: %s", v) + } + key = temps[0] + taintArgs := strings.Split(temps[1], ColonSymbol) + if len(taintArgs) != 2 { + return fmt.Errorf("error: invalid taint data: %s", v) + } + value, effect = taintArgs[0], taintArgs[1] + effect = strings.TrimSuffix(effect, DelSymbol) + } + + if !strings.Contains(v, EqualSymbol) { + temps := strings.Split(v, ColonSymbol) + if len(temps) != 2 { + return fmt.Errorf("faild to split taint argument: %s", v) + } + key, value, effect = temps[0], "", temps[1] + } + + if strings.Contains(v, EqualSymbol+ColonSymbol) { + temps := strings.Split(v, EqualSymbol+ColonSymbol) + if len(temps) != 2 { + return fmt.Errorf("faild to split taint argument: %s", v) + } + key, value, effect = temps[0], "", temps[1] + } + + //kubectl taint nodes xxx key- : remove all key related taints + if l.TaintList[ip.String()] == nil { + l.TaintList[ip.String()] = &taintList{} + } + if strings.HasSuffix(v, DelSymbol) && !strings.Contains(v, ColonSymbol) && !strings.Contains(v, EqualSymbol) { + l.TaintList[ip.String()].DelTaintList = append(l.TaintList[ip.String()].DelTaintList, newTaintStruct(strings.TrimSuffix(v, DelSymbol), "", "")) + } + + effect = strings.TrimSuffix(effect, DelSymbol) + if NotInEffect(v1.TaintEffect(effect), TaintEffectValues) { + return fmt.Errorf("taint effect %s need in %v", v, TaintEffectValues) + } + + taint := newTaintStruct(key, value, effect) + if _, ok := l.TaintList[ip.String()]; !ok { + l.TaintList[ip.String()] = &taintList{} + } + if strings.HasSuffix(v, DelSymbol) { + l.TaintList[ip.String()].DelTaintList = append(l.TaintList[ip.String()].DelTaintList, taint) + continue + } + l.TaintList[ip.String()].AddTaintList = append(l.TaintList[ip.String()].AddTaintList, taint) + } + return nil +} + +// UpdateTaints return nil mean's needn't update taint +func (l *Taint) UpdateTaints(taints []v1.Taint, ip string) []v1.Taint { + needDel := false + var updateTaints []v1.Taint + for k, v := range taints { + l.removePresenceTaint(v, ip) + if l.isDelTaint(v, ip) { + needDel = true + continue + } + updateTaints = append(updateTaints, taints[k]) + } + if len(l.TaintList[ip].AddTaintList) == 0 && !needDel { + return nil + } + return append(updateTaints, l.TaintList[ip].AddTaintList...) +} + +//Remove existing taint +func (l *Taint) removePresenceTaint(taint v1.Taint, ip string) { + for k, v := range l.TaintList[ip].AddTaintList { + if v.Key == taint.Key && v.Value == taint.Value && v.Effect == taint.Effect { + logrus.Infof("taint %s already exist", l.TaintList[ip].AddTaintList[k].String()) + l.TaintList[ip].AddTaintList = append(l.TaintList[ip].AddTaintList[:k], l.TaintList[ip].AddTaintList[k+1:]...) + break + } + } +} + +func (l *Taint) isDelTaint(taint v1.Taint, ip string) bool { + for _, v := range l.TaintList[ip].DelTaintList { + if v.Key == taint.Key && (v.Effect == taint.Effect || v.Effect == "") { + return true + } + } + return false +} + +func NotInEffect(effect v1.TaintEffect, effects []v1.TaintEffect) bool { + for _, e := range effects { + if e == effect { + return false + } + } + return true +} + +func NewTaint() *Taint { + return &Taint{} +} + +func UpdateTaint(newTaint *Taint, ip net.IP) error { + k8sClient, err := k8s.NewK8sClient() + if err != nil { + return err + } + nodeList, err := k8sClient.ListNodes() + if err != nil { + return err + } + + for _, node := range nodeList.Items { + for _, v := range node.Status.Addresses { + updateTaints := newTaint.UpdateTaints(node.Spec.Taints, ip.String()) + if updateTaints != nil { + node.Spec.Taints = updateTaints + _, err := k8sClient.UpdateNode(node) + if err != nil { + return err + } + logrus.Infof("succeed in updating node(%s) taints to %v", v.Address, updateTaints) + } + } + } + return nil +} diff --git a/pkg/infradriver/infradriver.go b/pkg/infradriver/infradriver.go index bfd4999c3fa..12fd674a7c2 100644 --- a/pkg/infradriver/infradriver.go +++ b/pkg/infradriver/infradriver.go @@ -23,6 +23,9 @@ import ( // InfraDriver treat the entire cluster as an operating system kernel, // interface function here is the target system call. type InfraDriver interface { + // GetHostTaint return host taint + GetHostTaint(host net.IP) []string + GetHostIPList() []net.IP GetHostIPListByRole(role string) []net.IP diff --git a/pkg/infradriver/ssh_infradriver.go b/pkg/infradriver/ssh_infradriver.go index f1b4525d3ee..dfd57d08516 100644 --- a/pkg/infradriver/ssh_infradriver.go +++ b/pkg/infradriver/ssh_infradriver.go @@ -35,6 +35,7 @@ import ( type SSHInfraDriver struct { sshConfigs map[string]ssh.Interface hosts []net.IP + hostTaint map[string][]string roleHostsMap map[string][]net.IP hostEnvMap map[string]map[string]interface{} clusterEnv map[string]interface{} @@ -105,6 +106,7 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) { roleHostsMap: map[string][]net.IP{}, // todo need to separate env into app render data and sys render data hostEnvMap: map[string]map[string]interface{}{}, + hostTaint: map[string][]string{}, clusterHostAliases: cluster.Spec.HostAliases, } @@ -157,12 +159,17 @@ func NewInfraDriver(cluster *v2.Cluster) (InfraDriver, error) { for _, host := range cluster.Spec.Hosts { for _, ip := range host.IPS { ret.hostEnvMap[ip.String()] = mergeList(ConvertEnv(host.Env), ret.clusterEnv) + ret.hostTaint[ip.String()] = host.Taints } } return ret, err } +func (d *SSHInfraDriver) GetHostTaint(host net.IP) []string { + return d.hostTaint[host.String()] +} + func (d *SSHInfraDriver) GetHostIPList() []net.IP { return d.hosts } diff --git a/types/api/v2/cluster_types.go b/types/api/v2/cluster_types.go index 46771790fcf..35da5b00b01 100644 --- a/types/api/v2/cluster_types.go +++ b/types/api/v2/cluster_types.go @@ -52,7 +52,8 @@ type Host struct { //overwrite SSH config SSH v1.SSH `json:"ssh,omitempty"` //overwrite env - Env []string `json:"env,omitempty"` + Env []string `json:"env,omitempty"` + Taints []string `json:"taints,omitempty"` } // HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the