physical/cache: Add a list of prefixes to not cache (#4515)

* physical/cache: Add a list of prefixes to not cache

* Rename the pathmanager

* Move cache back to the beggining of postUnseal

* Fix comment

helper/pathmanager/pathmanager.go

@@ -0,0 +1,136 @@
+package pathmanager
+
+import (
+	"strings"
+	"sync"
+
+	iradix "github.com/hashicorp/go-immutable-radix"
+)
+
+// PathManager is a prefix searchable index of paths
+type PathManager struct {
+	l     sync.RWMutex
+	paths *iradix.Tree
+}
+
+// New creates a new path manager
+func New() *PathManager {
+	return &PathManager{
+		paths: iradix.New(),
+	}
+}
+
+// AddPaths adds path to the paths list
+func (p *PathManager) AddPaths(paths []string) {
+	p.l.Lock()
+	defer p.l.Unlock()
+
+	txn := p.paths.Txn()
+	for _, prefix := range paths {
+		if len(prefix) == 0 {
+			continue
+		}
+
+		var exception bool
+		if strings.HasPrefix(prefix, "!") {
+			prefix = strings.TrimPrefix(prefix, "!")
+			exception = true
+		}
+
+		// We trim any trailing *, but we don't touch whether it is a trailing
+		// slash or not since we want to be able to ignore prefixes that fully
+		// specify a file
+		txn.Insert([]byte(strings.TrimSuffix(prefix, "*")), exception)
+	}
+	p.paths = txn.Commit()
+}
+
+// RemovePaths removes paths from the paths list
+func (p *PathManager) RemovePaths(paths []string) {
+	p.l.Lock()
+	defer p.l.Unlock()
+
+	txn := p.paths.Txn()
+	for _, prefix := range paths {
+		if len(prefix) == 0 {
+			continue
+		}
+
+		// Exceptions aren't stored with the leading ! so strip it
+		if strings.HasPrefix(prefix, "!") {
+			prefix = strings.TrimPrefix(prefix, "!")
+		}
+
+		// We trim any trailing *, but we don't touch whether it is a trailing
+		// slash or not since we want to be able to ignore prefixes that fully
+		// specify a file
+		txn.Delete([]byte(strings.TrimSuffix(prefix, "*")))
+	}
+	p.paths = txn.Commit()
+}
+
+// RemovePathPrefix removes all paths with the given prefix
+func (p *PathManager) RemovePathPrefix(prefix string) {
+	p.l.Lock()
+	defer p.l.Unlock()
+
+	// We trim any trailing *, but we don't touch whether it is a trailing
+	// slash or not since we want to be able to ignore prefixes that fully
+	// specify a file
+	p.paths, _ = p.paths.DeletePrefix([]byte(strings.TrimSuffix(prefix, "*")))
+}
+
+// Len returns the number of paths
+func (p *PathManager) Len() int {
+	return p.paths.Len()
+}
+
+// Paths returns the path list
+func (p *PathManager) Paths() []string {
+	p.l.RLock()
+	defer p.l.RUnlock()
+
+	paths := make([]string, 0, p.paths.Len())
+	walkFn := func(k []byte, v interface{}) bool {
+		paths = append(paths, string(k))
+		return false
+	}
+	p.paths.Root().Walk(walkFn)
+	return paths
+}
+
+// HasPath returns if the prefix for the path exists regardless if it is a path
+// (ending with /) or a prefix for a leaf node
+func (p *PathManager) HasPath(path string) bool {
+	p.l.RLock()
+	defer p.l.RUnlock()
+
+	if _, exceptionRaw, ok := p.paths.Root().LongestPrefix([]byte(path)); ok {
+		var exception bool
+		if exceptionRaw != nil {
+			exception = exceptionRaw.(bool)
+		}
+		return !exception
+	}
+	return false
+}
+
+// HasExactPath returns if the longest match is an exact match for the
+// full path
+func (p *PathManager) HasExactPath(path string) bool {
+	p.l.RLock()
+	defer p.l.RUnlock()
+
+	if val, exceptionRaw, ok := p.paths.Root().LongestPrefix([]byte(path)); ok {
+		var exception bool
+		if exceptionRaw != nil {
+			exception = exceptionRaw.(bool)
+		}
+
+		strVal := string(val)
+		if strings.HasSuffix(strVal, "/") || strVal == path {
+			return !exception
+		}
+	}
+	return false
+}

helper/pathmanager/pathmanager_test.go

@@ -0,0 +1,143 @@
+package pathmanager
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestPathManager(t *testing.T) {
+	m := New()
+
+	if m.Len() != 0 {
+		t.Fatalf("bad: path length expect 0, got %d", len(m.Paths()))
+	}
+
+	paths := []string{
+		"path1/",
+		"path2/",
+		"path3/",
+	}
+
+	for _, path := range paths {
+		if m.HasPath(path) {
+			t.Fatalf("path should not exist in filtered paths '%s'", path)
+		}
+	}
+
+	// add paths
+	m.AddPaths(paths)
+	if m.Len() != 3 {
+		t.Fatalf("bad: path length expect 3, got %d", len(m.Paths()))
+	}
+	if !reflect.DeepEqual(paths, m.Paths()) {
+		t.Fatalf("mismatch in paths")
+	}
+	for _, path := range paths {
+		if !m.HasPath(path) {
+			t.Fatalf("path should exist in filtered paths '%s'", path)
+		}
+	}
+
+	// remove the paths
+	m.RemovePaths(paths)
+
+	for _, path := range paths {
+		if m.HasPath(path) {
+			t.Fatalf("path should not exist in filtered paths '%s'", path)
+		}
+	}
+}
+
+func TestPathManager_RemovePrefix(t *testing.T) {
+	m := New()
+
+	if m.Len() != 0 {
+		t.Fatalf("bad: path length expect 0, got %d", len(m.Paths()))
+	}
+
+	paths := []string{
+		"path1/",
+		"path2/",
+		"path3/",
+	}
+
+	for _, path := range paths {
+		if m.HasPath(path) {
+			t.Fatalf("path should not exist in filtered paths '%s'", path)
+		}
+	}
+
+	// add paths
+	m.AddPaths(paths)
+	if m.Len() != 3 {
+		t.Fatalf("bad: path length expect 3, got %d", len(m.Paths()))
+	}
+	if !reflect.DeepEqual(paths, m.Paths()) {
+		t.Fatalf("mismatch in paths")
+	}
+	for _, path := range paths {
+		if !m.HasPath(path) {
+			t.Fatalf("path should exist in filtered paths '%s'", path)
+		}
+	}
+
+	// remove the paths
+	m.RemovePathPrefix("path")
+
+	if m.Len() != 0 {
+		t.Fatalf("bad: path length expect 0, got %d", len(m.Paths()))
+	}
+
+	for _, path := range paths {
+		if m.HasPath(path) {
+			t.Fatalf("path should not exist in filtered paths '%s'", path)
+		}
+	}
+}
+
+func TestPathManager_HasExactPath(t *testing.T) {
+	m := New()
+	paths := []string{
+		"path1/key1",
+		"path1/key1/subkey1",
+		"path1/key1/subkey2",
+		"path1/key1/subkey3",
+		"path2/*",
+		"path3/",
+		"!path4/key1",
+		"!path5/*",
+	}
+	m.AddPaths(paths)
+	if m.Len() != len(paths) {
+		t.Fatalf("path count does not match: expected %d, got %d", len(paths), m.Len())
+	}
+
+	type tCase struct {
+		key    string
+		expect bool
+	}
+
+	tcases := []tCase{
+		tCase{"path1/key1", true},
+		tCase{"path2/key1", true},
+		tCase{"path3/key1", true},
+		tCase{"path1/key1/subkey1", true},
+		tCase{"path1/key1/subkey99", false},
+		tCase{"path2/key1/subkey1", true},
+		tCase{"path1/key1/subkey1/subkey1", false},
+		tCase{"nonexistentpath/key1", false},
+		tCase{"path4/key1", false},
+		tCase{"path5/key1/subkey1", false},
+	}
+
+	for _, tc := range tcases {
+		if match := m.HasExactPath(tc.key); match != tc.expect {
+			t.Fatalf("incorrect match: key %q", tc.key)
+		}
+	}
+
+	m.RemovePaths(paths)
+	if len(m.Paths()) != 0 {
+		t.Fatalf("removing all paths did not clear manager: paths %v", m.Paths())
+	}
+}