macOS recursive file monitoring (#5575)

* auditbeat: macOS recursive file monitoring (#5421)

This patch adds a new file monitoring to the integrity module that
uses the FSEvents facility from macOS. Uses fsnotify's implementation
of fsevents library. Non-recursive behavior is emulated by filtering events.

* Use os.SameFile instead of comparing inodes

* Persist Action as a bit mask

auditbeat/module/audit/file/action.go

@@ -0,0 +1,154 @@
+package file
+
+import (
+	"math/bits"
+	"strings"
+)
+
+// Action is a description of the changes described by an event.
+type Action uint8
+
+// ActionArray is just syntactic sugar to invoke methods on []Action receiver
+type ActionArray []Action
+
+// List of possible Actions.
+const (
+	None               Action = 0
+	AttributesModified        = 1 << (iota - 1)
+	Created
+	Deleted
+	Updated
+	Moved
+	ConfigChange
+)
+
+var actionNames = map[Action]string{
+	None:               "none",
+	AttributesModified: "attributes_modified",
+	Created:            "created",
+	Deleted:            "deleted",
+	Updated:            "updated",
+	Moved:              "moved",
+	ConfigChange:       "config_change",
+}
+
+type actionOrderKey struct {
+	ExistsBefore, ExistsNow bool
+	Action                  Action
+}
+
+// Given the previous and current state of the file, and an action mask
+// returns a meaningful ordering for the actions in the mask
+var actionOrderMap = map[actionOrderKey]ActionArray{
+	{false, false, Created | Deleted}:                  {Created, Deleted},
+	{true, true, Created | Deleted}:                    {Deleted, Created},
+	{false, false, Moved | Created}:                    {Created, Moved},
+	{true, true, Moved | Created}:                      {Moved, Created},
+	{true, true, Moved | Deleted}:                      {Deleted, Moved},
+	{false, false, Moved | Deleted}:                    {Moved, Deleted},
+	{false, true, Updated | Created}:                   {Created, Updated},
+	{true, false, Updated | Deleted}:                   {Updated, Deleted},
+	{false, true, Updated | Moved}:                     {Moved, Updated},
+	{true, false, Updated | Moved}:                     {Updated, Moved},
+	{false, true, Moved | Created | Deleted}:           {Created, Deleted, Moved},
+	{true, false, Moved | Created | Deleted}:           {Deleted, Created, Moved},
+	{false, false, Updated | Moved | Created}:          {Created, Updated, Moved},
+	{true, true, Updated | Moved | Created}:            {Moved, Created, Updated},
+	{false, false, Updated | Moved | Deleted}:          {Moved, Updated, Deleted},
+	{true, true, Updated | Moved | Deleted}:            {Deleted, Moved, Updated},
+	{false, false, Updated | Created | Deleted}:        {Created, Updated, Deleted},
+	{true, true, Updated | Created | Deleted}:          {Deleted, Created, Updated},
+	{false, true, Updated | Moved | Created | Deleted}: {Created, Deleted, Moved, Updated},
+	{true, false, Updated | Moved | Created | Deleted}: {Deleted, Created, Updated, Moved},
+}
+
+func (action Action) isMultiple() bool {
+	return bits.OnesCount8(uint8(action)) > 1
+}
+
+func (action Action) String() string {
+	if name, found := actionNames[action]; found {
+		return name
+	}
+	var list []string
+	for flag, name := range actionNames {
+		if action&flag != 0 {
+			action ^= flag
+			list = append(list, name)
+		}
+	}
+	if action != 0 {
+		list = append(list, "unknown")
+	}
+	return strings.Join(list, "|")
+}
+
+func resolveActionOrder(action Action, existedBefore, existsNow bool) ActionArray {
+	if action == None {
+		return nil
+	}
+	if !action.isMultiple() {
+		return []Action{action}
+	}
+	key := actionOrderKey{existedBefore, existsNow, action}
+	if result, ok := actionOrderMap[key]; ok {
+		return result
+	}
+
+	// Can't resolve a meaningful order for the actions, usually the file
+	// has received further actions after the event being processed
+	return action.InAnyOrder()
+}
+
+func (action Action) InOrder(existedBefore, existsNow bool) ActionArray {
+	hasConfigChange := 0 != action&ConfigChange
+	hasUpdate := 0 != action&Updated
+	hasAttrMod := 0 != action&AttributesModified
+	action = Action(int(action) & int(^(ConfigChange | AttributesModified)))
+	if hasAttrMod {
+		action |= Updated
+	}
+
+	result := resolveActionOrder(action, existedBefore, existsNow)
+
+	if hasConfigChange {
+		result = append(result, ConfigChange)
+	}
+
+	if hasAttrMod {
+		for idx, value := range result {
+			if value == Updated {
+				if !hasUpdate {
+					result[idx] = AttributesModified
+				} else {
+					result = append(result, None)
+					copy(result[idx+2:], result[idx+1:])
+					result[idx+1] = AttributesModified
+				}
+				break
+			}
+		}
+	}
+	return result
+}
+
+func (action Action) InAnyOrder() ActionArray {
+	if !action.isMultiple() {
+		return []Action{action}
+	}
+	var result []Action
+	for k := range actionNames {
+		if 0 != action&k {
+			result = append(result, k)
+		}
+	}
+	return result
+}
+
+func (actions ActionArray) StringArray() []string {
+	result := make([]string, len(actions))
+	for index, value := range actions {
+		result[index] = value.String()
+	}
+	return result
+}

auditbeat/module/audit/file/event.go

@@ -25,37 +25,6 @@ import (
 	"github.com/elastic/beats/metricbeat/mb"
 )
 
-// Action is a description of the change that occurred.
-type Action uint8
-
-func (a Action) String() string {
-	if name, found := actionNames[a]; found {
-		return name
-	}
-	return "unknown"
-}
-
-// List of possible Actions.
-const (
-	None = iota << 1
-	AttributesModified
-	Created
-	Deleted
-	Updated
-	Moved
-	ConfigChange
-)
-
-var actionNames = map[Action]string{
-	None:               "none",
-	AttributesModified: "attributes_modified",
-	Created:            "created",
-	Deleted:            "deleted",
-	Updated:            "updated",
-	Moved:              "moved",
-	ConfigChange:       "config_change",
-}
-
 // Source identifies the source of an event (i.e. what triggered it).
 type Source uint8
 
@@ -198,11 +167,11 @@ func NewEvent(
 	maxFileSize uint64,
 	hashTypes []HashType,
 ) Event {
-	if action == Deleted {
-		return NewEventFromFileInfo(path, nil, nil, action, source, maxFileSize, hashTypes)
-	}
-
 	info, err := os.Lstat(path)
+	if err != nil && os.IsNotExist(err) {
+		// deleted file is signaled by info == nil
+		err = nil
+	}
 	err = errors.Wrap(err, "failed to lstat")
 	return NewEventFromFileInfo(path, info, err, action, source, maxFileSize, hashTypes)
 }
@@ -212,7 +181,7 @@ func (e *Event) String() string {
 	return string(data)
 }
 
-func buildMapStr(e *Event) common.MapStr {
+func buildMapStr(e *Event, existedBefore bool) common.MapStr {
 	m := common.MapStr{
 		"@timestamp": e.Timestamp,
 		"path":       e.Path,
@@ -221,7 +190,7 @@ func buildMapStr(e *Event) common.MapStr {
 	}
 
 	if e.Action > 0 {
-		m["action"] = e.Action.String()
+		m["action"] = e.Action.InOrder(existedBefore, e.Info != nil).StringArray()
 	}
 
 	if e.TargetPath != "" {
@@ -287,9 +256,12 @@ func diffEvents(old, new *Event) (Action, bool) {
 		return Moved, true
 	}
 
+	result := None
+
 	// Test if new.Hashes is a subset of old.Hashes.
 	hasAllHashes := true
 	for hashType, newValue := range new.Hashes {
+
 		oldValue, found := old.Hashes[hashType]
 		if !found {
 			hasAllHashes = false
@@ -298,37 +270,37 @@ func diffEvents(old, new *Event) (Action, bool) {
 
 		// The Updated action takes precedence over a new hash type being configured.
 		if !bytes.Equal(oldValue, newValue) {
-			return Updated, true
+			result |= Updated
+			break
 		}
 	}
 
 	if old.TargetPath != new.TargetPath ||
 		(old.Info == nil && new.Info != nil) ||
 		(old.Info != nil && new.Info == nil) {
-		return AttributesModified, true
+		result |= AttributesModified
 	}
 
 	// Test if metadata has changed.
 	if o, n := old.Info, new.Info; o != nil && n != nil {
 		// The owner and group names are ignored (they aren't persisted).
 		if o.Inode != n.Inode || o.UID != n.UID || o.GID != n.GID || o.SID != n.SID ||
 			o.Mode != n.Mode || o.Type != n.Type {
-			return AttributesModified, true
+			result |= AttributesModified
 		}
 
 		// For files consider mtime and size.
 		if n.Type == FileType && (!o.MTime.Equal(n.MTime) || o.Size != n.Size) {
-			return AttributesModified, true
+			result |= AttributesModified
 		}
 	}
 
 	// The old event didn't have all the requested hash types.
 	if !hasAllHashes {
-		return ConfigChange, true
+		result |= ConfigChange
 	}
 
-	// No change.
-	return None, false
+	return result, result != None
 }
 
 func hashFile(name string, hashType ...HashType) (map[HashType][]byte, error) {

auditbeat/module/audit/file/event_test.go

@@ -126,7 +126,7 @@ func TestDiffEvents(t *testing.T) {
 
 		action, changed := diffEvents(testEvent(), e)
 		assert.True(t, changed)
-		assert.EqualValues(t, Updated, action)
+		assert.EqualValues(t, Updated|AttributesModified, action)
 	})
 }