Add environment variables to the system process metricset (#3337)

This PR adds the environment variables that were used to start the process to the data reported in the system process metricset. The data is added as a dictionary under the `system.process.env` key. Environment variables must be whitelisted using an array of regular expressions specified using `process.env.whitelist: []` in the module config.

This feature implemented for FreeBSD, Linux, and OS X.

This gets the ProcArgs implementation.
* Update gosigar version to b49e01eb1e5c68c469392a63feaccae9352ceb12

glide.yaml

@@ -25,7 +25,7 @@ import:
   subpackages:
   - /difflib
 - package: github.com/elastic/gosigar
-  version: 171a3c9e31dde9688c154ba94be6cd5d8a78bf64
+  version: b49e01eb1e5c68c469392a63feaccae9352ceb12
 - package: github.com/elastic/procfs
   version: abf152e5f3e97f2fafac028d2cc06c1feb87ffa5
 - package: github.com/samuel/go-parser

metricbeat/_meta/beat.full.yml

@@ -67,6 +67,10 @@ metricbeat.modules:
   # EXPERIMENTAL: cgroups can be enabled for the process metricset.
   #cgroups: false
 
+  # A list of regular expressions used to whitelist environment variables
+  # reported with the process metricset's events. Defaults to empty.
+  #process.env.whitelist: []
+
   # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
   #socket.reverse_lookup.enabled: false
   #socket.reverse_lookup.success_ttl: 60s

metricbeat/docs/fields.asciidoc

@@ -5797,6 +5797,14 @@ type: keyword
 The username of the user that created the process. If the username cannot be determined, the field will contain the user's numeric identifier (UID). On Windows, this field includes the user's domain and is formatted as `domain\username`.
 
 
+[float]
+=== system.process.env
+
+type: dict
+
+The environment variables used to start the process. The data is available on FreeBSD, Linux, and OS X.
+
+
 [float]
 == cpu Fields
 

metricbeat/metricbeat.full.yml

@@ -67,6 +67,10 @@ metricbeat.modules:
   # EXPERIMENTAL: cgroups can be enabled for the process metricset.
   #cgroups: false
 
+  # A list of regular expressions used to whitelist environment variables
+  # reported with the process metricset's events. Defaults to empty.
+  #process.env.whitelist: []
+
   # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
   #socket.reverse_lookup.enabled: false
   #socket.reverse_lookup.success_ttl: 60s

metricbeat/module/system/_meta/config.full.yml

@@ -39,6 +39,10 @@
   # EXPERIMENTAL: cgroups can be enabled for the process metricset.
   #cgroups: false
 
+  # A list of regular expressions used to whitelist environment variables
+  # reported with the process metricset's events. Defaults to empty.
+  #process.env.whitelist: []
+
   # Configure reverse DNS lookup on remote IP addresses in the socket metricset.
   #socket.reverse_lookup.enabled: false
   #socket.reverse_lookup.success_ttl: 60s

metricbeat/module/system/process/_meta/fields.yml

@@ -35,6 +35,12 @@
         cannot be determined, the field will contain the user's
         numeric identifier (UID). On Windows, this field includes the user's
         domain and is formatted as `domain\username`.
+    - name: env
+      type: dict
+      dict-type: keyword
+      description: >
+        The environment variables used to start the process. The data is
+        available on FreeBSD, Linux, and OS X.
     - name: cpu
       type: group
       prefix: "[float]"

metricbeat/module/system/process/helper.go

@@ -31,18 +31,23 @@ type Process struct {
 	Cpu      sigar.ProcTime
 	Ctime    time.Time
 	FD       sigar.ProcFDUsage
+	Env      common.MapStr
 }
 
 type ProcStats struct {
-	Procs    []string
-	regexps  []*regexp.Regexp
-	ProcsMap ProcsMap
-	CpuTicks bool
-}
+	Procs        []string
+	ProcsMap     ProcsMap
+	CpuTicks     bool
+	EnvWhitelist []string
 
-// newProcess creates a new Process object based on the state information.
-func newProcess(pid int) (*Process, error) {
+	procRegexps []*regexp.Regexp // List of regular expressions used to whitelist processes.
+	envRegexps  []*regexp.Regexp // List of regular expressions used to whitelist env vars.
+}
 
+// newProcess creates a new Process object and initializes it with process
+// state information. If the process's command line and environment variables
+// are known they should be passed in to avoid re-fetching the information.
+func newProcess(pid int, cmdline string, env common.MapStr) (*Process, error) {
 	state := sigar.ProcState{}
 	if err := state.Get(pid); err != nil {
 		return nil, fmt.Errorf("error getting process state for pid=%d: %v", pid, err)
@@ -53,17 +58,22 @@ func newProcess(pid int) (*Process, error) {
 		Ppid:     state.Ppid,
 		Pgid:     state.Pgid,
 		Name:     state.Name,
-		State:    getProcState(byte(state.State)),
 		Username: state.Username,
+		State:    getProcState(byte(state.State)),
+		CmdLine:  cmdline,
 		Ctime:    time.Now(),
+		Env:      env,
 	}
 
 	return &proc, nil
 }
 
-// getDetails fills in CPU, memory, FD usage, and command line details for the process.
-func (proc *Process) getDetails(cmdline string) error {
-
+// getDetails fetches CPU, memory, FD usage, command line arguments, and
+// environment variables for the process. The envPredicate parameter is an
+// optional predicate function that should return true if an environment
+// variable should be saved with the process. If the argument is nil then all
+// environment variables are stored.
+func (proc *Process) getDetails(envPredicate func(string) bool) error {
 	proc.Mem = sigar.ProcMem{}
 	if err := proc.Mem.Get(proc.Pid); err != nil {
 		return fmt.Errorf("error getting process mem for pid=%d: %v", proc.Pid, err)
@@ -74,14 +84,12 @@ func (proc *Process) getDetails(cmdline string) error {
 		return fmt.Errorf("error getting process cpu time for pid=%d: %v", proc.Pid, err)
 	}
 
-	if cmdline == "" {
+	if proc.CmdLine == "" {
 		args := sigar.ProcArgs{}
 		if err := args.Get(proc.Pid); err != nil && !sigar.IsNotImplemented(err) {
 			return fmt.Errorf("error getting process arguments for pid=%d: %v", proc.Pid, err)
 		}
 		proc.CmdLine = strings.Join(args.List, " ")
-	} else {
-		proc.CmdLine = cmdline
 	}
 
 	if fd, err := getProcFDUsage(proc.Pid); err != nil {
@@ -90,6 +98,13 @@ func (proc *Process) getDetails(cmdline string) error {
 		proc.FD = *fd
 	}
 
+	if proc.Env == nil {
+		proc.Env = common.MapStr{}
+		if err := getProcEnv(proc.Pid, proc.Env, envPredicate); err != nil {
+			return fmt.Errorf("error getting process environment variables for pid=%d: %v", proc.Pid, err)
+		}
+	}
+
 	return nil
 }
 
@@ -120,6 +135,35 @@ func getProcFDUsage(pid int) (*sigar.ProcFDUsage, error) {
 	return &fd, nil
 }
 
+// getProcEnv gets the process's environment variables and writes them to the
+// out parameter. It handles ErrNotImplemented and permission errors. Any other
+// errors are returned.
+//
+// The filter function should return true if a given environment variable should
+// be added to the out parameter.
+//
+// On Linux you must be root to read other processes' environment variables.
+func getProcEnv(pid int, out common.MapStr, filter func(v string) bool) error {
+	env := &sigar.ProcEnv{}
+	if err := env.Get(pid); err != nil {
+		switch {
+		case sigar.IsNotImplemented(err):
+			return nil
+		case os.IsPermission(err):
+			return nil
+		default:
+			return err
+		}
+	}
+
+	for k, v := range env.Vars {
+		if filter == nil || filter(k) {
+			out[k] = v
+		}
+	}
+	return nil
+}
+
 func GetProcMemPercentage(proc *Process, totalPhyMem uint64) float64 {
 
 	// in unit tests, total_phymem is set to a value greater than zero
@@ -186,6 +230,10 @@ func (procStats *ProcStats) GetProcessEvent(process *Process, last *Process) com
 		proc["cmdline"] = process.CmdLine
 	}
 
+	if len(process.Env) > 0 {
+		proc["env"] = process.Env
+	}
+
 	if procStats.CpuTicks {
 		proc["cpu"] = common.MapStr{
 			"user":   process.Cpu.User,
@@ -233,7 +281,7 @@ func GetProcCpuPercentage(last *Process, current *Process) float64 {
 
 func (procStats *ProcStats) MatchProcess(name string) bool {
 
-	for _, reg := range procStats.regexps {
+	for _, reg := range procStats.procRegexps {
 		if reg.MatchString(name) {
 			return true
 		}
@@ -249,13 +297,22 @@ func (procStats *ProcStats) InitProcStats() error {
 		return nil
 	}
 
-	procStats.regexps = []*regexp.Regexp{}
+	procStats.procRegexps = []*regexp.Regexp{}
 	for _, pattern := range procStats.Procs {
 		reg, err := regexp.Compile(pattern)
 		if err != nil {
 			return fmt.Errorf("Failed to compile regexp [%s]: %v", pattern, err)
 		}
-		procStats.regexps = append(procStats.regexps, reg)
+		procStats.procRegexps = append(procStats.procRegexps, reg)
+	}
+
+	procStats.envRegexps = make([]*regexp.Regexp, 0, len(procStats.EnvWhitelist))
+	for _, pattern := range procStats.EnvWhitelist {
+		reg, err := regexp.Compile(pattern)
+		if err != nil {
+			return fmt.Errorf("failed to compile env whitelist regexp [%v]: %v", pattern, err)
+		}
+		procStats.envRegexps = append(procStats.envRegexps, reg)
 	}
 
 	return nil
@@ -278,26 +335,28 @@ func (procStats *ProcStats) GetProcStats() ([]common.MapStr, error) {
 
 	for _, pid := range pids {
 		var cmdline string
+		var env common.MapStr
 		if previousProc := procStats.ProcsMap[pid]; previousProc != nil {
 			cmdline = previousProc.CmdLine
+			env = previousProc.Env
 		}
 
-		process, err := newProcess(pid)
+		process, err := newProcess(pid, cmdline, env)
 		if err != nil {
 			logp.Debug("metricbeat", "Skip process pid=%d: %v", pid, err)
 			continue
 		}
 
 		if procStats.MatchProcess(process.Name) {
-			err = process.getDetails(cmdline)
+			err = process.getDetails(procStats.isWhitelistedEnvVar)
 			if err != nil {
 				logp.Err("Error getting process details. pid=%d: %v", process.Pid, err)
 				continue
 			}
 
 			newProcs[process.Pid] = process
 
-			last, _ := procStats.ProcsMap[process.Pid]
+			last := procStats.ProcsMap[process.Pid]
 			proc := procStats.GetProcessEvent(process, last)
 
 			processes = append(processes, proc)
@@ -308,6 +367,21 @@ func (procStats *ProcStats) GetProcStats() ([]common.MapStr, error) {
 	return processes, nil
 }
 
+// isWhitelistedEnvVar returns true if the given variable name is a match for
+// the whitelist. If the whitelist is empty it returns false.
+func (p ProcStats) isWhitelistedEnvVar(varName string) bool {
+	if len(p.envRegexps) == 0 {
+		return false
+	}
+
+	for _, p := range p.envRegexps {
+		if p.MatchString(varName) {
+			return true
+		}
+	}
+	return false
+}
+
 // unixTimeMsToTime converts a unix time given in milliseconds since Unix epoch
 // to a common.Time value.
 func unixTimeMsToTime(unixTimeMs uint64) common.Time {