From 8a7090435e9b3c590d25c5c3f3bb90f8326a2e09 Mon Sep 17 00:00:00 2001 From: Mark Hansen Date: Wed, 22 Sep 2021 15:50:41 +1000 Subject: [PATCH] Add -tagroot and -tagleaf options These add synthetic stack frames to samples. Example usage: $ pprof -tagroot thread pprof.proto Will add synthetic stack frames at the root like "thread:UIThread" and "thread:BackgroundPool-1". Closes #558 --- internal/driver/commands.go | 8 + internal/driver/config.go | 4 + internal/driver/driver.go | 23 +++ internal/driver/tagroot.go | 125 ++++++++++++ internal/driver/tagroot_test.go | 351 ++++++++++++++++++++++++++++++++ 5 files changed, 511 insertions(+) create mode 100644 internal/driver/tagroot.go create mode 100644 internal/driver/tagroot_test.go diff --git a/internal/driver/commands.go b/internal/driver/commands.go index 4397e253e..c9edf10bb 100644 --- a/internal/driver/commands.go +++ b/internal/driver/commands.go @@ -189,6 +189,14 @@ var configHelp = map[string]string{ "Drops functions above the highest matched frame.", "If set, all frames above the highest match are dropped from every sample.", "Matching includes the function name, filename or object name."), + "tagroot": helpText( + "Adds pseudo stack frames for labels key/value pairs at the callstack root.", + "A comma-separated list of label keys.", + "The first key creates frames at the new root."), + "tagleaf": helpText( + "Adds pseudo stack frames for labels key/value pairs at the callstack leaf.", + "A comma-separated list of label keys.", + "The last key creates frames at the new leaf."), "tagfocus": helpText( "Restricts to samples with tags in range or matched by regexp", "Use name=value syntax to limit the matching to a specific tag.", diff --git a/internal/driver/config.go b/internal/driver/config.go index b3f82f22c..9fcdd459b 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -30,6 +30,10 @@ type config struct { Normalize bool `json:"normalize,omitempty"` Sort string `json:"sort,omitempty"` + // Label pseudo stack frame generation options + TagRoot string `json:"tagroot,omitempty"` + TagLeaf string `json:"tagleaf,omitempty"` + // Filtering options DropNegative bool `json:"drop_negative,omitempty"` NodeCount int `json:"nodecount,omitempty"` diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 3967a12d4..6a1e64c60 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -73,6 +73,10 @@ func generateRawReport(p *profile.Profile, cmd []string, cfg config, o *plugin.O cfg = applyCommandOverrides(cmd[0], c.format, cfg) + // Create label pseudo nodes before filtering, in case the filters use + // the generated nodes. + generateTagRootsLeaves(p, cfg, o.UI) + // Delay focus after configuring report to get percentages on all samples. relative := cfg.RelativePercentages if relative { @@ -208,6 +212,25 @@ func applyCommandOverrides(cmd string, outputFormat int, cfg config) config { return cfg } +// generateTagRootsLeaves generates extra nodes from the tagroot and tagleaf options. +func generateTagRootsLeaves(prof *profile.Profile, cfg config, ui plugin.UI) { + tagRootLabelKeys := dropEmptyStrings(strings.Split(cfg.TagRoot, ",")) + tagLeafLabelKeys := dropEmptyStrings(strings.Split(cfg.TagLeaf, ",")) + rootm, leafm := addLabelNodes(prof, tagRootLabelKeys, tagLeafLabelKeys, cfg.Unit) + warnNoMatches(cfg.TagRoot == "" || rootm, "TagRoot", ui) + warnNoMatches(cfg.TagLeaf == "" || leafm, "TagLeaf", ui) +} + +// dropEmptyStrings filters a slice to only non-empty strings +func dropEmptyStrings(in []string) (out []string) { + for _, s := range in { + if s != "" { + out = append(out, s) + } + } + return +} + func aggregate(prof *profile.Profile, cfg config) error { var function, filename, linenumber, address bool inlines := !cfg.NoInlines diff --git a/internal/driver/tagroot.go b/internal/driver/tagroot.go new file mode 100644 index 000000000..9580f3469 --- /dev/null +++ b/internal/driver/tagroot.go @@ -0,0 +1,125 @@ +package driver + +import ( + "strings" + + "github.com/google/pprof/internal/measurement" + "github.com/google/pprof/profile" +) + +// addLabelNodes adds pseudo stack frames "label:value" to each Sample with +// labels matching the supplied keys. +// +// rootKeys adds frames at the root of the callgraph (first key becomes new root). +// leafKeys adds frames at the leaf of the callgraph (last key becomes new leaf). +// +// Returns whether there were matches found for the label keys. +func addLabelNodes(p *profile.Profile, rootKeys, leafKeys []string, outputUnit string) (rootm, leafm bool) { + // Find where to insert the new locations and functions at the end of + // their ID spaces. + var maxLocID uint64 + var maxFunctionID uint64 + for _, loc := range p.Location { + if loc.ID > maxLocID { + maxLocID = loc.ID + } + } + for _, f := range p.Function { + if f.ID > maxFunctionID { + maxFunctionID = f.ID + } + } + nextLocID := maxLocID + 1 + nextFuncID := maxFunctionID + 1 + + // Intern the new locations and functions we are generating. + locsByName := map[string]*profile.Location{} + functionsByName := map[string]*profile.Function{} + + internFunction := func(functionName string) *profile.Function { + function, found := functionsByName[functionName] + if found { + return function + } + function = &profile.Function{ + ID: nextFuncID, + Name: functionName, + } + nextFuncID++ + p.Function = append(p.Function, function) + functionsByName[functionName] = function + return function + } + + internLocation := func(functionName string, function *profile.Function) *profile.Location { + location, found := locsByName[functionName] + if found { + return location + } + location = &profile.Location{ + ID: nextLocID, + Line: []profile.Line{ + profile.Line{ + Function: function, + }, + }, + } + nextLocID++ + p.Location = append(p.Location, location) + locsByName[functionName] = location + return location + } + + makeLabelLocations := func(s *profile.Sample, keys []string) ([]*profile.Location, bool) { + var locations []*profile.Location + var match bool + for i := range keys { + // Loop backwards, ensuring the first tag is closest to the root, + // and the last tag is closest to the leaves. + k := keys[len(keys)-1-i] + values := formatLabelValues(s, k, outputUnit) + if len(values) > 0 { + match = true + } + functionName := k + ":" + strings.Join(values, ",") + location := internLocation(functionName, internFunction(functionName)) + locations = append(locations, location) + } + return locations, match + } + + for _, s := range p.Sample { + rootsToAdd, sampleMatchedRoot := makeLabelLocations(s, rootKeys) + if sampleMatchedRoot { + rootm = true + } + leavesToAdd, sampleMatchedLeaf := makeLabelLocations(s, leafKeys) + if sampleMatchedLeaf { + leafm = true + } + + var newLocations []*profile.Location + newLocations = append(newLocations, leavesToAdd...) + newLocations = append(newLocations, s.Location...) + newLocations = append(newLocations, rootsToAdd...) + s.Location = newLocations + } + return +} + +// formatLabelValues returns all the string and numeric labels in Sample, with +// the numeric labels formatted according to outputUnit. +func formatLabelValues(s *profile.Sample, k string, outputUnit string) []string { + var values []string + values = append(values, s.Label[k]...) + numLabels := s.NumLabel[k] + numUnits := s.NumUnit[k] + if len(numLabels) != len(numUnits) { + return values + } + for i, numLabel := range numLabels { + unit := numUnits[i] + values = append(values, measurement.ScaledLabel(numLabel, unit, outputUnit)) + } + return values +} diff --git a/internal/driver/tagroot_test.go b/internal/driver/tagroot_test.go new file mode 100644 index 000000000..afebcb598 --- /dev/null +++ b/internal/driver/tagroot_test.go @@ -0,0 +1,351 @@ +package driver + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/pprof/internal/proftest" + "github.com/google/pprof/profile" +) + +const mainBinary = "/bin/main" + +var cpuF = []*profile.Function{ + {ID: 1, Name: "main", SystemName: "main", Filename: "main.c"}, + {ID: 2, Name: "foo", SystemName: "foo", Filename: "foo.c"}, + {ID: 3, Name: "foo_caller", SystemName: "foo_caller", Filename: "foo.c"}, +} + +var cpuM = []*profile.Mapping{ + { + ID: 1, + Start: 0x10000, + Limit: 0x40000, + File: mainBinary, + HasFunctions: true, + HasFilenames: true, + HasLineNumbers: true, + HasInlineFrames: true, + }, + { + ID: 2, + Start: 0x1000, + Limit: 0x4000, + File: "/lib/lib.so", + HasFunctions: true, + HasFilenames: true, + HasLineNumbers: true, + HasInlineFrames: true, + }, + { + ID: 3, + Start: 0x4000, + Limit: 0x5000, + File: "/lib/lib2_c.so.6", + HasFunctions: true, + HasFilenames: true, + HasLineNumbers: true, + HasInlineFrames: true, + }, + { + ID: 4, + Start: 0x5000, + Limit: 0x9000, + File: "/lib/lib.so_6 (deleted)", + HasFunctions: true, + HasFilenames: true, + HasLineNumbers: true, + HasInlineFrames: true, + }, +} + +var cpuL = []*profile.Location{ + { + ID: 1000, + Mapping: cpuM[1], + Address: 0x1000, + Line: []profile.Line{ + {Function: cpuF[0], Line: 1}, + }, + }, + { + ID: 2000, + Mapping: cpuM[0], + Address: 0x2000, + Line: []profile.Line{ + {Function: cpuF[1], Line: 2}, + {Function: cpuF[2], Line: 1}, + }, + }, + { + ID: 3000, + Mapping: cpuM[0], + Address: 0x3000, + Line: []profile.Line{ + {Function: cpuF[1], Line: 2}, + {Function: cpuF[2], Line: 1}, + }, + }, + { + ID: 3001, + Mapping: cpuM[0], + Address: 0x3001, + Line: []profile.Line{ + {Function: cpuF[2], Line: 2}, + }, + }, + { + ID: 3002, + Mapping: cpuM[0], + Address: 0x3002, + Line: []profile.Line{ + {Function: cpuF[2], Line: 3}, + }, + }, +} + +var testProfile1 = &profile.Profile{ + TimeNanos: 10000, + PeriodType: &profile.ValueType{Type: "cpu", Unit: "milliseconds"}, + Period: 1, + DurationNanos: 10e9, + SampleType: []*profile.ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "milliseconds"}, + }, + Sample: []*profile.Sample{ + { + Location: []*profile.Location{cpuL[0]}, + Value: []int64{1000, 1000}, + Label: map[string][]string{ + "key1": {"tag1"}, + "key2": {"tag1"}, + }, + }, + { + Location: []*profile.Location{cpuL[1], cpuL[0]}, + Value: []int64{100, 100}, + Label: map[string][]string{ + "key1": {"tag2"}, + "key3": {"tag2"}, + }, + }, + { + Location: []*profile.Location{cpuL[2], cpuL[0]}, + Value: []int64{10, 10}, + Label: map[string][]string{ + "key1": {"tag3"}, + "key2": {"tag2"}, + }, + NumLabel: map[string][]int64{ + "allocations": {1024}, + }, + NumUnit: map[string][]string{ + "allocations": {""}, + }, + }, + { + Location: []*profile.Location{cpuL[3], cpuL[0]}, + Value: []int64{10000, 10000}, + Label: map[string][]string{ + "key1": {"tag4"}, + "key2": {"tag1"}, + }, + NumLabel: map[string][]int64{ + "allocations": {1024, 2048}, + }, + NumUnit: map[string][]string{ + "allocations": {"bytes", "b"}, + }, + }, + { + Location: []*profile.Location{cpuL[4], cpuL[0]}, + Value: []int64{1, 1}, + Label: map[string][]string{ + "key1": {"tag4"}, + "key2": {"tag1", "tag5"}, + }, + NumLabel: map[string][]int64{ + "allocations": {1024, 1}, + }, + NumUnit: map[string][]string{ + "allocations": {"byte", "kilobyte"}, + }, + }, + }, + Location: cpuL, + Function: cpuF, + Mapping: cpuM, +} + +func TestAddLabelNodesMatchBooleans(t *testing.T) { + type addLabelNodesTestcase struct { + tagroot, tagleaf []string + outputUnit string + rootm, leafm bool + // wantSampleFuncs contains expected stack functions and sample value after + // adding nodes, in the same order as in the profile. The format is as + // returned by sampleFuncs function, which is "callee caller: ". + wantSampleFuncs []string + } + for tx, tc := range []addLabelNodesTestcase{ + { + wantSampleFuncs: []string{ + "main: 1000", + "foo foo_caller main: 100", + "foo foo_caller main: 10", + "foo_caller main: 10000", + "foo_caller main: 1", + }, + }, + { + tagroot: []string{"key404"}, + tagleaf: []string{"key404"}, + wantSampleFuncs: []string{ + "key404: main key404:: 1000", + "key404: foo foo_caller main key404:: 100", + "key404: foo foo_caller main key404:: 10", + "key404: foo_caller main key404:: 10000", + "key404: foo_caller main key404:: 1", + }, + }, + { + tagroot: []string{"key1"}, + rootm: true, + wantSampleFuncs: []string{ + "main key1:tag1: 1000", + "foo foo_caller main key1:tag2: 100", + "foo foo_caller main key1:tag3: 10", + "foo_caller main key1:tag4: 10000", + "foo_caller main key1:tag4: 1", + }, + }, + { + tagroot: []string{"key2"}, + rootm: true, + wantSampleFuncs: []string{ + "main key2:tag1: 1000", + "foo foo_caller main key2:: 100", + "foo foo_caller main key2:tag2: 10", + "foo_caller main key2:tag1: 10000", + "foo_caller main key2:tag1,tag5: 1", + }, + }, + { + tagleaf: []string{"key1"}, + leafm: true, + wantSampleFuncs: []string{ + "key1:tag1 main: 1000", + "key1:tag2 foo foo_caller main: 100", + "key1:tag3 foo foo_caller main: 10", + "key1:tag4 foo_caller main: 10000", + "key1:tag4 foo_caller main: 1", + }, + }, + { + tagleaf: []string{"key3"}, + leafm: true, + wantSampleFuncs: []string{ + "key3: main: 1000", + "key3:tag2 foo foo_caller main: 100", + "key3: foo foo_caller main: 10", + "key3: foo_caller main: 10000", + "key3: foo_caller main: 1", + }, + }, + { + tagroot: []string{"key1", "key2"}, + rootm: true, + wantSampleFuncs: []string{ + "main key2:tag1 key1:tag1: 1000", + "foo foo_caller main key2: key1:tag2: 100", + "foo foo_caller main key2:tag2 key1:tag3: 10", + "foo_caller main key2:tag1 key1:tag4: 10000", + "foo_caller main key2:tag1,tag5 key1:tag4: 1", + }, + }, + { + tagleaf: []string{"key1", "key2"}, + leafm: true, + wantSampleFuncs: []string{ + "key2:tag1 key1:tag1 main: 1000", + "key2: key1:tag2 foo foo_caller main: 100", + "key2:tag2 key1:tag3 foo foo_caller main: 10", + "key2:tag1 key1:tag4 foo_caller main: 10000", + "key2:tag1,tag5 key1:tag4 foo_caller main: 1", + }, + }, + { + tagleaf: []string{"allocations"}, + leafm: true, + wantSampleFuncs: []string{ + "allocations: main: 1000", + "allocations: foo foo_caller main: 100", + // TODO: bug, this line should say allocations:1024 + "allocations: foo foo_caller main: 10", + "allocations:1024B,2048B foo_caller main: 10000", + "allocations:1024B,1024B foo_caller main: 1", + }, + }, + { + tagroot: []string{"allocations"}, + rootm: true, + wantSampleFuncs: []string{ + "main allocations:: 1000", + "foo foo_caller main allocations:: 100", + // TODO: bug #651, this line should say allocations:1024 + "foo foo_caller main allocations:: 10", + "foo_caller main allocations:1024B,2048B: 10000", + "foo_caller main allocations:1024B,1024B: 1", + }, + }, + { + outputUnit: "kB", + tagleaf: []string{"allocations"}, + leafm: true, + wantSampleFuncs: []string{ + "allocations: main: 1000", + "allocations: foo foo_caller main: 100", + // TODO: bug #651, this line should say allocations:1024 + "allocations: foo foo_caller main: 10", + "allocations:1kB,2kB foo_caller main: 10000", + "allocations:1kB,1kB foo_caller main: 1", + }, + }, + } { + p := testProfile1.Copy() + rootm, leafm := addLabelNodes(p, tc.tagroot, tc.tagleaf, tc.outputUnit) + if rootm != tc.rootm { + t.Errorf("Test #%d, got rootm=%v, want=%v", tx, rootm, tc.rootm) + } + if leafm != tc.leafm { + t.Errorf("Test #%d, got leafm=%v, want=%v", tx, leafm, tc.leafm) + } + if got, want := strings.Join(sampleFuncs(p), "\n")+"\n", strings.Join(tc.wantSampleFuncs, "\n")+"\n"; got != want { + diff, err := proftest.Diff([]byte(want), []byte(got)) + if err != nil { + t.Fatalf("failed to get diff: %v", err) + } + t.Errorf("Test #%d, profile samples got diff(want->got):\n%s", tx, diff) + } + } +} + +// sampleFuncs returns a slice of strings where each string represents one +// profile sample in the format " : ". This allows +// the expected values for test cases to be specified in human-readable +// strings. +func sampleFuncs(p *profile.Profile) []string { + var ret []string + for _, s := range p.Sample { + var funcs []string + for _, loc := range s.Location { + for _, line := range loc.Line { + funcs = append(funcs, line.Function.Name) + } + } + ret = append(ret, fmt.Sprintf("%s: %d", strings.Join(funcs, " "), s.Value[0])) + } + return ret +}