Add kernel.audit_rules config option to set audit rules (#4482)

* Update go-libaudit to get the ability to set audit rules

* Add kernel.audit_rules config option to set audit rules

CHANGELOG.asciidoc

@@ -64,6 +64,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 *Metricbeat*
 - Add random startup delay to each metricset to avoid the thundering herd problem. {issue}4010[4010]
 
+- Add the ability to configure audit rules to the kernel module. {pull}4482[4482]
+
 *Packetbeat*
 
 *Winlogbeat*

NOTICE

@@ -270,7 +270,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-libaudit
 Version: v0.0.3
-Revision: cce87232cabb992866d94d6b6b6f810adcf13f9b
+Revision: b2d37f9d37d8e2a81bf69b5fae3c1f49bf6a14bf
 License type (autodetected): Apache License 2.0
 ./vendor/github.com/elastic/go-libaudit/LICENSE:
 --------------------------------------------------------------------

metricbeat/metricbeat.full.yml

@@ -130,6 +130,11 @@ metricbeat.modules:
   kernel.rate_limit: 0
   kernel.include_raw_message: false
   kernel.include_warnings: false
+  kernel.audit_rules: |
+    # Define audit rules here.
+    # Create file watches (-w) or syscall audits (-a or -A). For example:
+    -w /etc/passwd -p wa -k identity
+    -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access
 
 #-------------------------------- ceph Module --------------------------------
 - module: ceph

metricbeat/module/audit/_meta/config.full.yml

@@ -6,3 +6,8 @@
   kernel.rate_limit: 0
   kernel.include_raw_message: false
   kernel.include_warnings: false
+  kernel.audit_rules: |
+    # Define audit rules here.
+    # Create file watches (-w) or syscall audits (-a or -A). For example:
+    -w /etc/passwd -p wa -k identity
+    -a always,exit -F arch=b32 -S open,creat,truncate,ftruncate,openat,open_by_handle_at -F exit=-EPERM -k access

metricbeat/module/audit/kernel/_meta/docs.asciidoc

@@ -71,21 +71,15 @@ as warnings any issues that were encountered while parsing the raw messages.
 The default value is false. This setting is primarily used for development and
 debugging purposes.
 
-////
-TODO (andrewkroh): Uncomment when rules can be installed by Metricbeat.
 *`kernel.audit_rules`*:: A string containing the audit rules that should be
 installed to the kernel. There should be one rule per line. Comments can be
 embedded in the string using `#` as a prefix. The format for rules is the same
-used by the Linux `auditctl` utility.
-////
+used by the Linux `auditctl` utility. Metricbeat supports adding file watches
+(`-w`) and syscall rules (`-a` or `-A`).
 
 [float]
 === Audit Rules
 
-// TODO (andrewkroh): Uncomment when rules can be installed by Metricbeat.
-WARNING: This feature is not complete yet. Currently you must manually install
-audit rules using `auditctl`.
-
 The audit rules are where you configure the activities that are audited. These
 rules are configured as either syscalls or files that should be monitored. For
 example you can track all `connect` syscalls or file system writes to
@@ -101,6 +95,10 @@ the most active rules first in order to speed up evaluation.
 You can assign keys to each rule for better identification of the rule that
 triggered an event and easier filtering later in Elasticsearch.
 
+Defining any audit rules in the config causes Metricbeat to purge all existing
+audit rules prior to adding the rules specified in the config. Therefore it is
+unnecessary and unsupported to include a `-D` (delete all) rule.
+
 [source,yaml]
 ----
 metricbeat.modules:

metricbeat/module/audit/kernel/audit_linux.go

@@ -79,6 +79,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 func (ms *MetricSet) Run(reporter mb.PushReporter) {
 	defer ms.client.Close()
 
+	if err := ms.addRules(reporter); err != nil {
+		reporter.Error(err)
+		logp.Err("%v %v", logPrefix, err)
+		return
+	}
+
 	out, err := ms.receiveEvents(reporter.Done())
 	if err != nil {
 		reporter.Error(err)
@@ -101,6 +107,37 @@ func (ms *MetricSet) Run(reporter mb.PushReporter) {
 	}
 }
 
+func (ms *MetricSet) addRules(reporter mb.PushReporter) error {
+	rules, err := ms.config.rules()
+	if err != nil {
+		return errors.Wrap(err, "failed to add rules")
+	}
+
+	if len(rules) == 0 {
+		logp.Info("%v No audit kernel.rules were specified.", logPrefix)
+		return nil
+	}
+
+	// Delete existing rules.
+	n, err := ms.client.DeleteRules()
+	if err != nil {
+		return errors.Wrap(err, "failed to delete existing rules")
+	}
+	logp.Info("%v Deleted %v pre-existing audit rules.", logPrefix, n)
+
+	// Add rules from config.
+	for _, rule := range rules {
+		if err = ms.client.AddRule(rule.data); err != nil {
+			// Treat rule add errors as warnings and continue.
+			err = errors.Wrapf(err, "failed to add kernel rule '%v'", rule.flags)
+			reporter.Error(err)
+			logp.Warn("%v %v", err)
+		}
+	}
+	logp.Info("%v Successfully added %d kernel audit rules.", logPrefix, len(rules))
+	return nil
+}
+
 func (ms *MetricSet) initClient() error {
 	status, err := ms.client.GetStatus()
 	if err != nil {

metricbeat/module/audit/kernel/audit_linux_test.go

@@ -11,8 +11,18 @@ import (
 var userLoginMsg = `type=USER_LOGIN msg=audit(1492896301.818:19955): pid=12635 uid=0 auid=4294967295 ses=4294967295 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=179.38.151.221 terminal=sshd res=failed'`
 
 func TestData(t *testing.T) {
-	// Create a mock netlink client.
-	mock := NewMock().returnACK().returnStatus().returnMessage(userLoginMsg)
+	// Create a mock netlink client that provides the expected responses.
+	mock := NewMock().
+		// GetRules response with zero rules. Used by DeleteAll rules.
+		returnACK().returnDone().
+		// AddRule response.
+		returnACK().
+		// AddRule response.
+		returnACK().
+		// Get Status response for initClient
+		returnACK().returnStatus().
+		// Send a single audit message from the kernel.
+		returnMessage(userLoginMsg)
 
 	// Replace the default AuditClient with a mock.
 	ms := mbtest.NewPushMetricSet(t, getConfig())
@@ -22,7 +32,7 @@ func TestData(t *testing.T) {
 
 	events, errs := mbtest.RunPushMetricSet(time.Second, ms)
 	if len(errs) > 0 {
-		t.Fatal("received errors:", errs)
+		t.Fatalf("received errors: %+v", errs)
 	}
 	if len(events) == 0 {
 		t.Fatal("received no events")
@@ -36,5 +46,9 @@ func getConfig() map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "audit",
 		"metricsets": []string{"kernel"},
+		"kernel.audit_rules": `
+		   -w /etc/passwd -p wa -k auth
+		   -a always,exit -F arch=b64 -S execve -k exec
+		`,
 	}
 }

metricbeat/module/audit/kernel/config.go

@@ -1,12 +1,80 @@
 package kernel
 
+import (
+	"bufio"
+	"bytes"
+	"strings"
+
+	"github.com/joeshaw/multierror"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/go-libaudit/rule"
+	"github.com/elastic/go-libaudit/rule/flags"
+)
+
 // Config defines the kernel metricset's possible configuration options.
 type Config struct {
 	ResolveIDs   bool   `config:"kernel.resolve_ids"`         // Resolve UID/GIDs to names.
 	BacklogLimit uint32 `config:"kernel.backlog_limit"`       // Max number of message to buffer in the kernel.
 	RateLimit    uint32 `config:"kernel.rate_limit"`          // Rate limit in messages/sec of messages from kernel.
 	RawMessage   bool   `config:"kernel.include_raw_message"` // Include the list of raw audit messages in the event.
 	Warnings     bool   `config:"kernel.include_warnings"`    // Include warnings in the event (for dev/debug purposes only).
+	RulesBlob    string `config:"kernel.audit_rules"`         // Audit rules. One rule per line.
+}
+
+type auditRule struct {
+	flags string
+	data  []byte
+}
+
+// Validate validates the rules specified in the config.
+func (c Config) Validate() error {
+	_, err := c.rules()
+	return err
+}
+
+// Rules returns a list of rules specified in the config.
+func (c Config) rules() ([]auditRule, error) {
+	var errs multierror.Errors
+	var auditRules []auditRule
+	ruleSet := map[string]auditRule{}
+	s := bufio.NewScanner(bytes.NewBufferString(c.RulesBlob))
+	for s.Scan() {
+		line := strings.TrimSpace(s.Text())
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+
+		// Parse the CLI flags into an intermediate rule specification.
+		r, err := flags.Parse(line)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "failed on rule '%v'", line))
+			continue
+		}
+
+		// Convert rule specification to a binary rule representation.
+		data, err := rule.Build(r)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "failed on rule '%v'", line))
+			continue
+		}
+
+		// Detect duplicates based on the normalized binary rule representation.
+		existingRule, found := ruleSet[string(data)]
+		if found {
+			errs = append(errs, errors.Errorf("failed on rule '%v' because its a duplicate of '%v'", line, existingRule.flags))
+			continue
+		}
+		auditRule := auditRule{flags: line, data: []byte(data)}
+		ruleSet[string(data)] = auditRule
+
+		auditRules = append(auditRules, auditRule)
+	}
+
+	if len(errs) > 0 {
+		return nil, errors.Wrap(errs.Err(), "invalid kernel.audit_rules")
+	}
+	return auditRules, nil
 }
 
 var defaultConfig = Config{

metricbeat/module/audit/kernel/config_linux_test.go

@@ -0,0 +1,76 @@
+package kernel
+
+import (
+	"testing"
+
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigValidate(t *testing.T) {
+	data := `
+kernel.audit_rules: |
+  # Comments and empty lines are ignored.
+  -w /etc/passwd -p wa -k auth
+
+  -a always,exit -F arch=b64 -S execve -k exec`
+
+	config, err := parseConfig(t, data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rules, err := config.rules()
+	if err != nil {
+		t.Fatal()
+	}
+	assert.EqualValues(t, []string{
+		"-w /etc/passwd -p wa -k auth",
+		"-a always,exit -F arch=b64 -S execve -k exec",
+	}, commands(rules))
+}
+
+func TestConfigValidateWithError(t *testing.T) {
+	data := `
+kernel.audit_rules: |
+  -x bad -F flag
+  -a always,exit -w /etc/passwd
+  -a always,exit -F arch=b64 -S fake -k exec`
+
+	_, err := parseConfig(t, data)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	t.Log(err)
+}
+
+func TestConfigValidateWithDuplicates(t *testing.T) {
+	data := `
+kernel.audit_rules: |
+  -w /etc/passwd -p rwxa -k auth
+  -w /etc/passwd -k auth`
+
+	_, err := parseConfig(t, data)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	t.Log(err)
+}
+
+func parseConfig(t testing.TB, yaml string) (Config, error) {
+	c, err := common.NewConfigWithYAML([]byte(yaml), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := defaultConfig
+	err = c.Unpack(&config)
+	return config, err
+}
+
+func commands(rules []auditRule) []string {
+	var cmds []string
+	for _, r := range rules {
+		cmds = append(cmds, r.flags)
+	}
+	return cmds
+}

metricbeat/module/audit/kernel/mock_linux_test.go

@@ -30,6 +30,16 @@ func (n *MockNetlinkSendReceiver) returnACK() *MockNetlinkSendReceiver {
 	return n
 }
 
+func (n *MockNetlinkSendReceiver) returnDone() *MockNetlinkSendReceiver {
+	n.messages = append(n.messages, syscall.NetlinkMessage{
+		Header: syscall.NlMsghdr{
+			Type:  syscall.NLMSG_DONE,
+			Flags: syscall.NLM_F_ACK,
+		},
+	})
+	return n
+}
+
 func (n *MockNetlinkSendReceiver) returnStatus() *MockNetlinkSendReceiver {
 	status := libaudit.AuditStatus{}
 	buf := new(bytes.Buffer)