ebpf: syscall: Add a new tool for syscall trace

Signed-off-by: Tao Chen <[email protected]>

Makefile

@@ -51,6 +51,7 @@ ebpf.o: libbpf
 	$(CLANG_COMPILE) -c $(EBPF_SRC)/cpu/offcpu/offcpu.bpf.c -o $(EBPF_SRC)/cpu/offcpu/offcpu.bpf.o
 	$(CLANG_COMPILE) -c $(EBPF_SRC)/cpu/oncpu/oncpu.bpf.c -o $(EBPF_SRC)/cpu/oncpu/oncpu.bpf.o
 	$(CLANG_COMPILE) -c $(EBPF_SRC)/cpu/futexsnoop/futexsnoop.bpf.c -o $(EBPF_SRC)/cpu/futexsnoop/futexsnoop.bpf.o
+	$(CLANG_COMPILE) -c $(EBPF_SRC)/cpu/syscall/syscall.bpf.c -o $(EBPF_SRC)/cpu/syscall/syscall.bpf.o
 
 all: generate ebpf.o
 	@echo "go build $(TARGET)"

pkg/app/cmd.go

@@ -62,6 +62,7 @@ func SubCmdInit(cmd *Cmd) {
 		newOffCpuSpyCmd(&cmd.cfg.OFFCPU),
 		newOnCpuSpyCmd(&cmd.cfg.ONCPU),
 		newFutexSnoopSpyCmd(&cmd.cfg.FUTEXSNOOP),
+		newSyscallSpyCmd(&cmd.cfg.SYSCALL),
 		newVersionCmd(),
 	}
 
@@ -207,6 +208,33 @@ func newOffCpuSpyCmd(cfg *config.OFFCPU) *cobra.Command {
 	return connectCmd
 }
 
+func newSyscallSpyCmd(cfg *config.SYSCALL) *cobra.Command {
+	vpr := newViper()
+
+	connectCmd := &cobra.Command{
+		Use:   "syscall [flags]",
+		Short: "eBPF snoop syscall",
+		Args:  cobra.NoArgs,
+
+		RunE: cli.CreateCmdRunFn(cfg, vpr, func(_ *cobra.Command, _ []string) error {
+			conf := &appspy.Config{
+				Exporter: cfg.Exporter,
+				Server:   cfg.Server,
+			}
+			return RunSpy(cfg, conf, func(cfg interface{}, buf chan *model.SpyEvent) core.BpfSpyer {
+				config, ok := cfg.(*config.SYSCALL)
+				if ok {
+					return cpu.NewSyscallSession(model.Syscall, config, buf)
+				}
+				return nil
+			})
+		}),
+	}
+
+	cli.PopulateFlagSet(cfg, connectCmd.Flags(), vpr)
+	return connectCmd
+}
+
 func NewRootCmd(cfg *config.Config) *cobra.Command {
 	vpr := newViper()
 	rootCmd := &cobra.Command{

pkg/app/config/config.go

@@ -9,7 +9,8 @@ type Config struct {
 
 	ONCPU      ONCPU      `skip:"true" mapstructure:",squash"`
 	OFFCPU     OFFCPU     `skip:"true" mapstructure:",squash"`
-	FUTEXSNOOP FUTEXSNOOP `skip:"true" mapstructire:",squash"`
+	FUTEXSNOOP FUTEXSNOOP `skip:"true" mapstructure:",squash"`
+	SYSCALL    SYSCALL    `skip:"true" mapstructure:",squash"`
 }
 
 type FUTEXSNOOP struct {
@@ -28,6 +29,20 @@ type FUTEXSNOOP struct {
 	Exporter         string `def:"" desc:"data exporter: loki,pyroscoe,prometheus,disk,etc." mapstructure:"exporter"`
 }
 
+type SYSCALL struct {
+	LogLevel        string `def:"info" desc:"log level: debug|info|warn|error" mapstructure:"log-level"`
+	AppName         string `def:"" desc:"application name used when uploading profiling data" mapstructure:"app-name"`
+	Server          string `def:"http://localhost:4040" desc:"the server address" mapstructure:"server"`
+	Pid             int    `def:"0" desc:"pid to trace, 0 to trace all pids" mapstructure:"pid"`
+	Tid             int    `def:"0" desc:"tid to trace, 0 to trace all tids" mapstructure:"tid"`
+	MaxDurMs        uint   `def:"10000" desc:"max time(ms) wait unlock" mapstructure:"max_dur_ms"`
+	MinDurMs        uint   `def:"1" desc:"min time(ms) wait unlock" mapstructure:"min_dur_ms"`
+	SymbolCacheSize int    `def:"256" desc:"max size of symbols cache" mapstructure:"symbol-cache-size"`
+	Stack           bool   `def:"false" desc:"get stack info or not" mapstructure:"stack"`
+	BtfPath         string `def:"" desc:"btf file path" mapstructure:"btf-path"`
+	Exporter        string `def:"" desc:"data exporter: loki,pyroscoe,prometheus,disk,etc." mapstructure:"exporter"`
+}
+
 type OFFCPU struct {
 	LogLevel        string `def:"info" desc:"log level: debug|info|warn|error" mapstructure:"log-level"`
 	AppName         string `def:"" desc:"application name used when uploading profiling data" mapstructure:"app-name"`

pkg/component/detector/cpudetector/cpudetector.go

@@ -204,6 +204,8 @@ func (c *CpuDetector) ProcessEvent(e *model.SpyEvent) error {
 		dataBlock, err = c.irqoffHandler(e)
 	case model.FutexSnoop:
 		dataBlock, err = c.futexsnoopHandler(e)
+	case model.Syscall:
+		dataBlock, err = c.syscallHandler(e)
 	default:
 		return nil
 	}
@@ -248,6 +250,24 @@ func (c *CpuDetector) irqoffHandler(e *model.SpyEvent) (*model.DataBlock, error)
 	return nil, nil
 }
 
+func (c *CpuDetector) formatSyscallLabels(e *model.SpyEvent) (*model.AttributeMap, error) {
+	labels := model.NewAttributeMap()
+	for i := 0; i < int(e.ParamsCnt); i++ {
+		userAttributes := e.UserAttributes[i]
+		switch {
+		case userAttributes.GetKey() == "dur_ms":
+			labels.AddIntValue(model.DurMs, int64(userAttributes.GetUintValue()))
+		case userAttributes.GetKey() == "syscall":
+			labels.AddStringValue(model.Syscall, string(userAttributes.GetValue()))
+		case userAttributes.GetKey() == "stack":
+			labels.AddStringValue(model.Stack, string(userAttributes.GetValue()))
+		}
+	}
+	labels.AddIntValue(model.Pid, int64(e.Task.Pid))
+	labels.AddStringValue(model.Comm, strings.Replace(string(e.Task.Comm), "\u0000", "", -1))
+	return labels, nil
+}
+
 func (c *CpuDetector) formatFutexSnoopLabels(e *model.SpyEvent) (*model.AttributeMap, error) {
 	labels := model.NewAttributeMap()
 	for i := 0; i < int(e.ParamsCnt); i++ {
@@ -329,6 +349,13 @@ func (c *CpuDetector) futexsnoopHandler(e *model.SpyEvent) (*model.DataBlock, er
 	return model.NewDataBlock(model.FutexSnoop, labels, e.TimeStamp, metric), nil
 }
 
+func (c *CpuDetector) syscallHandler(e *model.SpyEvent) (*model.DataBlock, error) {
+	labels, _ := c.formatSyscallLabels(e)
+	val := e.GetUintUserAttribute("dur_ms")
+	metric := model.NewIntMetric(model.Syscall, int64(val))
+	return model.NewDataBlock(model.Syscall, labels, e.TimeStamp, metric), nil
+}
+
 func (c *CpuDetector) offcpuHandler(e *model.SpyEvent) (*model.DataBlock, error) {
 	labels, _ := c.formatOffcpuLabels(e)
 	// util.PrintStructFields(*ev)
@@ -368,5 +395,5 @@ func (c *CpuDetector) ConsumeEvent(e *model.SpyEvent) error {
 
 // Notice: hard code, new tool added update this function
 func (c *CpuDetector) OwnedEvents() []string {
-	return []string{model.OffCpu, model.IrqOff, model.OnCpu, model.FutexSnoop}
+	return []string{model.OffCpu, model.IrqOff, model.OnCpu, model.FutexSnoop, model.Syscall}
 }

pkg/component/processor/processor.go

@@ -153,6 +153,8 @@ func (d *DefaultProcessor) Consume(data *model.DataBlock) error {
 	// offcpu事件不聚合数据直接输出给spyexporter
 	case model.OffCpu:
 		fallthrough
+	case model.Syscall:
+		fallthrough
 	case model.FutexSnoop:
 		err := d.consumer.Consume(data)
 		if err != nil {

pkg/core/model/const.go

@@ -14,6 +14,7 @@ const (
 	IrqOff     = "irqoff"
 	OnCpu      = "oncpu"
 	FutexSnoop = "futexsnoop"
+	Syscall    = "syscall"
 	OtherEvent = "other_event"
 )
 
@@ -62,6 +63,9 @@ const (
 	AvgDur     = "avg_dur"
 	LockCnt    = "lock_cnt"
 	Stack      = "stack"
+
+	// for syscall
+	DurMs = "dur_ms"
 )
 
 const (