diff --git a/images/chaos-daemon/Dockerfile b/images/chaos-daemon/Dockerfile index 05c24a26ae..c5b65ec871 100644 --- a/images/chaos-daemon/Dockerfile +++ b/images/chaos-daemon/Dockerfile @@ -3,6 +3,6 @@ FROM alpine:3.10 ARG HTTPS_PROXY ARG HTTP_PROXY -RUN apk add --no-cache tzdata iptables ipset stress-ng iproute2 +RUN apk add --no-cache tzdata iptables ipset stress-ng iproute2 util-linux COPY --from=pingcap/binary /src/bin/chaos-daemon /usr/local/bin/chaos-daemon \ No newline at end of file diff --git a/pkg/chaosdaemon/ipset_server.go b/pkg/chaosdaemon/ipset_server.go index 9601d450aa..4cf47b898b 100644 --- a/pkg/chaosdaemon/ipset_server.go +++ b/pkg/chaosdaemon/ipset_server.go @@ -38,7 +38,7 @@ func (s *daemonServer) FlushIpSet(ctx context.Context, req *pb.IpSetRequest) (*e return nil, err } - nsPath := GenNetnsPath(pid) + nsPath := GetNsPath(pid, netNS) // TODO: lock every ipset when working on it diff --git a/pkg/chaosdaemon/iptables_server.go b/pkg/chaosdaemon/iptables_server.go index 24a9461876..c8293d67ad 100644 --- a/pkg/chaosdaemon/iptables_server.go +++ b/pkg/chaosdaemon/iptables_server.go @@ -40,7 +40,7 @@ func (s *daemonServer) FlushIptables(ctx context.Context, req *pb.IpTablesReques return nil, err } - nsPath := GenNetnsPath(pid) + nsPath := GetNsPath(pid, netNS) rule := req.Rule format := "" diff --git a/pkg/chaosdaemon/netlink_linux.go b/pkg/chaosdaemon/netlink_linux.go index 02965ffca0..429e8e3ddb 100644 --- a/pkg/chaosdaemon/netlink_linux.go +++ b/pkg/chaosdaemon/netlink_linux.go @@ -25,7 +25,7 @@ type toQdiscFunc func(*netlink.Handle, netlink.Link) netlink.Qdisc func applyQdisc(pid uint32, toQdisc toQdiscFunc) error { log.Info("Apply qdisc on PID", "pid", pid) - ns, err := netns.GetFromPath(GenNetnsPath(pid)) + ns, err := netns.GetFromPath(GetNsPath(pid, netNS)) if err != nil { log.Error(err, "failed to find network namespace", "pid", pid) return err @@ -60,7 +60,7 @@ func applyQdisc(pid uint32, toQdisc toQdiscFunc) error { func deleteQdisc(pid uint32, toQdisc toQdiscFunc) error { log.Info("Delete qdisc on PID", "pid", pid) - ns, err := netns.GetFromPath(GenNetnsPath(pid)) + ns, err := netns.GetFromPath(GetNsPath(pid, netNS)) if err != nil { log.Error(err, "failed to find network namespace", "pid", pid) return err diff --git a/pkg/chaosdaemon/stress_server_linux.go b/pkg/chaosdaemon/stress_server_linux.go index e34066652e..4a40726219 100644 --- a/pkg/chaosdaemon/stress_server_linux.go +++ b/pkg/chaosdaemon/stress_server_linux.go @@ -63,10 +63,14 @@ func (s *daemonServer) ExecStressors(ctx context.Context, if err != nil { return nil, err } - cmd := exec.Command("stress-ng", strings.Fields(req.Stressors)...) + + cmd := withPidNS(context.Background(), GetNsPath(pid, pidNS), "stress-ng", strings.Fields(req.Stressors)...) + if err := cmd.Start(); err != nil { return nil, err } + log.Info("Start process successfully") + procState, err := process.NewProcess(int32(cmd.Process.Pid)) if err != nil { return nil, err @@ -108,15 +112,25 @@ func (s *daemonServer) CancelStressors(ctx context.Context, return nil, err } log.Info("Canceling stressors", "request", req) + ins, err := process.NewProcess(int32(pid)) if err != nil { return &empty.Empty{}, nil } if ct, err := ins.CreateTime(); err == nil && ct == req.StartTime { - if err := ins.Kill(); err != nil { + children, err := ins.Children() + if err != nil { return nil, err } + for _, child := range children { + log.Info("killing children for nsenter", "pid", child.Pid) + if err := child.Kill(); err != nil { + return nil, err + } + } } + + log.Info("Successfully canceled stressors") return &empty.Empty{}, nil } diff --git a/pkg/chaosdaemon/tc_linux.go b/pkg/chaosdaemon/tc_linux.go index d10785cec1..d3d5b4d5b5 100644 --- a/pkg/chaosdaemon/tc_linux.go +++ b/pkg/chaosdaemon/tc_linux.go @@ -30,7 +30,7 @@ func applyTc(ctx context.Context, pid uint32, args ...string) error { } } - nsPath := GenNetnsPath(pid) + nsPath := GetNsPath(pid, netNS) cmd := withNetNS(ctx, nsPath, "tc", args...) log.Info("tc command", "command", cmd.String(), "args", args) diff --git a/pkg/chaosdaemon/util.go b/pkg/chaosdaemon/util.go index 156b8ddd4e..87b813f829 100755 --- a/pkg/chaosdaemon/util.go +++ b/pkg/chaosdaemon/util.go @@ -21,6 +21,7 @@ import ( "os" "os/exec" "strconv" + "strings" "sync" "syscall" @@ -184,9 +185,29 @@ func CreateContainerRuntimeInfoClient(containerRuntime string) (ContainerRuntime return cli, nil } -// GetNetnsPath returns network namespace path -func GenNetnsPath(pid uint32) string { - return fmt.Sprintf("%s/%d/ns/net", defaultProcPrefix, pid) +type nsType string + +const ( + mountNS nsType = "mnt" + utsNS nsType = "uts" + ipcNS nsType = "ipc" + netNS nsType = "net" + pidNS nsType = "pid" + userNS nsType = "user" +) + +var nsArgMap = map[nsType]string{ + mountNS: "m", + utsNS: "u", + ipcNS: "i", + netNS: "n", + pidNS: "p", + userNS: "U", +} + +// GetNsPath returns corresponding namespace path +func GetNsPath(pid uint32, typ nsType) string { + return fmt.Sprintf("%s/%d/ns/%s", defaultProcPrefix, pid, string(typ)) } func withNetNS(ctx context.Context, nsPath string, cmd string, args ...string) *exec.Cmd { @@ -196,8 +217,43 @@ func withNetNS(ctx context.Context, nsPath string, cmd string, args ...string) * return f(ctx, nsPath, cmd, args...) } - // BusyBox's nsenter is very confusing. This usage is found by several attempts - args = append([]string{"-n" + nsPath, "--", cmd}, args...) + return withNS(ctx, []nsOption{{ + Typ: netNS, + Path: nsPath, + }}, cmd, args...) +} + +func withPidNS(ctx context.Context, nsPath string, cmd string, args ...string) *exec.Cmd { + // Mock point to return mock Cmd in unit test + if c := mock.On("MockWithPidNs"); c != nil { + f := c.(func(context.Context, string, string, ...string) *exec.Cmd) + return f(ctx, nsPath, cmd, args...) + } + + return withNS(ctx, []nsOption{{ + Typ: pidNS, + Path: nsPath, + }}, cmd, args...) +} + +type nsOption struct { + Typ nsType + Path string +} + +func withNS(ctx context.Context, options []nsOption, cmd string, args ...string) *exec.Cmd { + // Mock point to return mock Cmd in unit test + if c := mock.On("MockWithNs"); c != nil { + f := c.(func(context.Context, []nsOption, string, ...string) *exec.Cmd) + return f(ctx, options, cmd, args...) + } + + args = append([]string{"--", cmd}, args...) + for _, option := range options { + args = append([]string{"-" + nsArgMap[option.Typ] + option.Path}, args...) + } + + log.Info("running command", "command", "nsenter "+strings.Join(args, " ")) return exec.CommandContext(ctx, "nsenter", args...) }