many: apply new home directory rules to data copy (#13145)

* Apply new home directory rules to data copy
* Cleanups and adding caching of home directories
* optimizing home directory caching
* Adding tests for new home directory behavior
* improvements and enhancements to Get/SetSnapHomeDirs and tests
* snapdata_test and copydata_test fixes
* Adding tests/main/user-data-handling-homedirs/task.yaml
It also performs the verification for the root user as well as the test user present in /home

* Adding Mock functions for snapDataDirs and snapCommonDataDirs and creating unit tests
* Adding data copy between revisions test to snapd-homedirs spread test
This commit also deletes the test-snapd-sh snap files and the user-data-handling-homedirs spread test

* Change return behavior of GetSnapHomeDirs to be more in line with other functions and TestDataHomeDirs rename and fixes
* Adding unit tests and renaming GetSnapHomeDirs
* Adding unit tests and doing some cleanups
* go.sum cleanups
* cleanups, globbing expressions and root directory improvements
* more s.tempdir cleanups in unit tests, removing debug remnants
* unit test fixups, dirs.go fixups, typo fixups
* SetSnapHomeDirs optimizations, fixups and unit test improvements
* make globs only accessible via DataHomeGlobs
* fixups and conflict resolution
* comment fixup
* dirs.go fixups and improvements
* tests/main/snapd-homedirs: revision improvements and spread fix
* tests/main/snapd-homedirs: adding details section to fix static-checks
* tests/main/snapd-homedirs-vendored: fixing AppArmor tunable check
* tests/main/snapd-homedirs: reverting rev read to using snap list
This fixes issues with FHS compliant distros such as arch linux where the /snap directory is located in /var/lib/snapd/snap

* fixing comments, small dirs.go and homedirs.go improvements
* dirs/dirs.go: SeSnapHomeDirs comment fixup

dirs/dirs.go

@@ -25,25 +25,27 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/snapcore/snapd/release"
+	"github.com/snapcore/snapd/strutil"
 )
 
 // the various file paths
 var (
-	GlobalRootDir string
+	GlobalRootDir string = "/"
 
 	RunDir string
 
 	SnapMountDir string
 
 	DistroLibExecDir string
 
-	HiddenSnapDataHomeGlob string
+	hiddenSnapDataHomeGlob []string
 
 	SnapBlobDir          string
 	SnapDataDir          string
-	SnapDataHomeGlob     string
+	snapDataHomeGlob     []string
 	SnapDownloadCacheDir string
 	SnapAppArmorDir      string
 	SnapSeccompBase      string
@@ -140,6 +142,13 @@ var (
 	FeaturesDir string
 )
 
+// User defined home directory variables
+// Not exported, use SnapHomeDirs() and SetSnapHomeDirs() instead
+var (
+	snapHomeDirsMu sync.Mutex
+	snapHomeDirs   []string
+)
+
 const (
 	defaultSnapMountDir = "/snap"
 
@@ -190,6 +199,67 @@ func init() {
 	SetRootDir(root)
 }
 
+// SnapHomeDirs returns a slice of the currently configured home directories.
+func SnapHomeDirs() []string {
+	snapHomeDirsMu.Lock()
+	defer snapHomeDirsMu.Unlock()
+	dirs := make([]string, len(snapHomeDirs))
+	copy(dirs, snapHomeDirs)
+	// Should never be true since SetSnapHomeDirs is run on init and on SetRootDir calls.
+	// Useful for unit tests.
+	if len(dirs) == 0 {
+		return []string{filepath.Join(GlobalRootDir, "/home")}
+	}
+	return dirs
+}
+
+// SetSnapHomeDirs sets SnapHomeDirs to the user defined values and appends /home if needed.
+// homedirs must be a comma separated list of paths to home directories.
+// If homedirs is empty, SnapHomeDirs will be a slice of length 1 containing "/home".
+// Also generates the data directory globbing expressions for each user.
+// Expected to be run by configstate.Init, returns a slice of home directories.
+func SetSnapHomeDirs(homedirs string) []string {
+	snapHomeDirsMu.Lock()
+	defer snapHomeDirsMu.Unlock()
+
+	//clear old values
+	snapHomeDirs = nil
+	snapDataHomeGlob = nil
+	hiddenSnapDataHomeGlob = nil
+
+	// Do not set the root directory as home unless explicitly specified with "."
+	if homedirs != "" {
+		snapHomeDirs = strings.Split(homedirs, ",")
+		for i := range snapHomeDirs {
+			// clean the path
+			snapHomeDirs[i] = filepath.Clean(snapHomeDirs[i])
+			globalRootDir := GlobalRootDir
+			// Avoid false positives with HasPrefix
+			if globalRootDir != "/" && !strings.HasSuffix(globalRootDir, "/") {
+				globalRootDir += "/"
+			}
+			if !strings.HasPrefix(snapHomeDirs[i], globalRootDir) {
+				snapHomeDirs[i] = filepath.Join(GlobalRootDir, snapHomeDirs[i])
+			}
+			// Generate data directory globbing expressions for each user.
+			snapDataHomeGlob = append(snapDataHomeGlob, filepath.Join(snapHomeDirs[i], "*", UserHomeSnapDir))
+			hiddenSnapDataHomeGlob = append(hiddenSnapDataHomeGlob, filepath.Join(snapHomeDirs[i], "*", HiddenSnapDataHomeDir))
+		}
+	}
+
+	// Make sure /home is part of the list.
+	hasHome := strutil.ListContains(snapHomeDirs, filepath.Join(GlobalRootDir, "/home"))
+
+	// if not add it and create the glob expressions.
+	if !hasHome {
+		snapHomeDirs = append(snapHomeDirs, filepath.Join(GlobalRootDir, "/home"))
+		snapDataHomeGlob = append(snapDataHomeGlob, filepath.Join(GlobalRootDir, "/home", "*", UserHomeSnapDir))
+		hiddenSnapDataHomeGlob = append(hiddenSnapDataHomeGlob, filepath.Join(GlobalRootDir, "/home", "*", HiddenSnapDataHomeDir))
+	}
+
+	return snapHomeDirs
+}
+
 // StripRootDir strips the custom global root directory from the specified argument.
 func StripRootDir(dir string) string {
 	if !filepath.IsAbs(dir) {
@@ -205,6 +275,17 @@ func StripRootDir(dir string) string {
 	return "/" + result
 }
 
+// DataHomeGlobs returns a slice of globbing expressions for the snap directories in use.
+func DataHomeGlobs(opts *SnapDirOptions) []string {
+	snapHomeDirsMu.Lock()
+	defer snapHomeDirsMu.Unlock()
+	if opts != nil && opts.HiddenSnapDataDir {
+		return hiddenSnapDataHomeGlob
+	}
+
+	return snapDataHomeGlob
+}
+
 // SupportsClassicConfinement returns true if the current directory layout supports classic confinement.
 func SupportsClassicConfinement() bool {
 	// Core systems don't support classic confinement as a policy decision.
@@ -366,8 +447,6 @@ func SetRootDir(rootdir string) {
 	}
 
 	SnapDataDir = filepath.Join(rootdir, "/var/snap")
-	SnapDataHomeGlob = filepath.Join(rootdir, "/home/*/", UserHomeSnapDir)
-	HiddenSnapDataHomeGlob = filepath.Join(rootdir, "/home/*/", HiddenSnapDataHomeDir)
 	SnapAppArmorDir = filepath.Join(rootdir, snappyDir, "apparmor", "profiles")
 	SnapDownloadCacheDir = filepath.Join(rootdir, snappyDir, "cache")
 	SnapSeccompBase = filepath.Join(rootdir, snappyDir, "seccomp")
@@ -516,11 +595,15 @@ func SetRootDir(rootdir string) {
 
 	FeaturesDir = FeaturesDirUnder(rootdir)
 
+	// If the root directory changes we also need to reset snapHomeDirs.
+	SetSnapHomeDirs("/home")
+
 	// call the callbacks last so that the callbacks can just reference the
 	// global vars if they want, instead of using the new rootdir directly
 	for _, c := range callbacks {
 		c(rootdir)
 	}
+
 }
 
 // what inside a (non-classic) snap is /usr/lib/snapd, outside can come from different places

dirs/dirs_test.go

@@ -37,6 +37,78 @@ var _ = Suite(&DirsTestSuite{})
 
 type DirsTestSuite struct{}
 
+func (s *DirsTestSuite) TestSnapHomeDirs(c *C) {
+	defer dirs.SetRootDir("")
+	hidden := dirs.SnapDirOptions{HiddenSnapDataDir: true}
+	// Expected ouptut should be ROOTDIR/home
+	c.Check(dirs.SnapHomeDirs(), DeepEquals, []string{filepath.Join(dirs.GlobalRootDir, "/home")})
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, []string{filepath.Join(dirs.GlobalRootDir, "/home", "*", "snap")})
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, []string{filepath.Join(dirs.GlobalRootDir, "/home", "*", ".snap", "data")})
+
+	// Expected output should remove all trailing '/' and add /home if it is not present.
+	getSnapHomeFromSet := dirs.SetSnapHomeDirs("/home/homeDir1,/home/homeDirs/homeDir1///,/home/homeDir2/,/home/homeTest/users/")
+	snapHomeDirs := []string{"/home/homeDir1", "/home/homeDirs/homeDir1", "/home/homeDir2", "/home/homeTest/users", "/home"}
+	dataGlob := []string{"/home/homeDir1/*/snap", "/home/homeDirs/homeDir1/*/snap", "/home/homeDir2/*/snap", "/home/homeTest/users/*/snap",
+		"/home/*/snap"}
+	hiddenDataGlob := []string{"/home/homeDir1/*/.snap/data", "/home/homeDirs/homeDir1/*/.snap/data", "/home/homeDir2/*/.snap/data",
+		"/home/homeTest/users/*/.snap/data", "/home/*/.snap/data"}
+	getSnapHomeDirs := dirs.SnapHomeDirs()
+	c.Check(getSnapHomeDirs, DeepEquals, snapHomeDirs)
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, dataGlob)
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, hiddenDataGlob)
+	// Useful for detecting if any output changes were made to dirs.SnapHomeDirs() and not reflected in dirs.SetSnapHomeDirs()
+	c.Check(getSnapHomeFromSet, DeepEquals, snapHomeDirs)
+
+	// Expected output, should remove all trailing '/' and not add /home since it is already present.
+	getSnapHomeFromSet = dirs.SetSnapHomeDirs("/home/homeDir1,/home/homeDirs/homeDir1///,/home/,/home/homeDir2/,/home/homeTest/users/")
+	snapHomeDirs = []string{"/home/homeDir1", "/home/homeDirs/homeDir1", "/home", "/home/homeDir2", "/home/homeTest/users"}
+	dataGlob = []string{"/home/homeDir1/*/snap", "/home/homeDirs/homeDir1/*/snap", "/home/*/snap", "/home/homeDir2/*/snap",
+		"/home/homeTest/users/*/snap"}
+	hiddenDataGlob = []string{"/home/homeDir1/*/.snap/data", "/home/homeDirs/homeDir1/*/.snap/data", "/home/*/.snap/data",
+		"/home/homeDir2/*/.snap/data", "/home/homeTest/users/*/.snap/data"}
+	getSnapHomeDirs = dirs.SnapHomeDirs()
+	c.Check(getSnapHomeDirs, DeepEquals, snapHomeDirs)
+	c.Check(getSnapHomeFromSet, DeepEquals, snapHomeDirs)
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, dataGlob)
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, hiddenDataGlob)
+
+	// Expected output should only return /ROOTDIR/home as a home directory
+	dirs.SetRootDir("/new/root")
+	getSnapHomeFromSet = dirs.SetSnapHomeDirs("")
+	snapHomeDirs = []string{"/new/root/home"}
+	dataGlob = []string{"/new/root/home/*/snap"}
+	hiddenDataGlob = []string{"/new/root/home/*/.snap/data"}
+	getSnapHomeDirs = dirs.SnapHomeDirs()
+	c.Check(getSnapHomeDirs, DeepEquals, snapHomeDirs)
+	c.Check(getSnapHomeFromSet, DeepEquals, snapHomeDirs)
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, dataGlob)
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, hiddenDataGlob)
+
+	// Expected output should append /ROOTDIR to each entry except if already present
+	getSnapHomeFromSet = dirs.SetSnapHomeDirs("/home/homeDir1,/new/root/home/homeDirs/homeDir1///,/home/homeDir2/,/new/root/home/homeTest/users/")
+	snapHomeDirs = []string{"/new/root/home/homeDir1", "/new/root/home/homeDirs/homeDir1", "/new/root/home/homeDir2", "/new/root/home/homeTest/users", "/new/root/home"}
+	dataGlob = []string{"/new/root/home/homeDir1/*/snap", "/new/root/home/homeDirs/homeDir1/*/snap", "/new/root/home/homeDir2/*/snap",
+		"/new/root/home/homeTest/users/*/snap", "/new/root/home/*/snap"}
+	hiddenDataGlob = []string{"/new/root/home/homeDir1/*/.snap/data", "/new/root/home/homeDirs/homeDir1/*/.snap/data", "/new/root/home/homeDir2/*/.snap/data",
+		"/new/root/home/homeTest/users/*/.snap/data", "/new/root/home/*/.snap/data"}
+	getSnapHomeDirs = dirs.SnapHomeDirs()
+	c.Check(getSnapHomeDirs, DeepEquals, snapHomeDirs)
+	c.Check(getSnapHomeFromSet, DeepEquals, snapHomeDirs)
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, dataGlob)
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, hiddenDataGlob)
+
+	// setting a Root directory should reset the home directory
+	dirs.SetRootDir("/new")
+	c.Check(dirs.SnapHomeDirs(), DeepEquals, []string{"/new/home"})
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, []string{"/new/home/*/snap"})
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, []string{"/new/home/*/.snap/data"})
+	dirs.SetRootDir("/")
+	c.Check(dirs.SnapHomeDirs(), DeepEquals, []string{"/home"})
+	c.Check(dirs.DataHomeGlobs(nil), DeepEquals, []string{"/home/*/snap"})
+	c.Check(dirs.DataHomeGlobs(&hidden), DeepEquals, []string{"/home/*/.snap/data"})
+
+}
+
 func (s *DirsTestSuite) TestStripRootDir(c *C) {
 	dirs.SetRootDir("/")
 	// strip does nothing if the default (empty) root directory is used

overlord/configstate/configcore/export_test.go

@@ -33,6 +33,7 @@ var (
 	UpdateKeyValueStream = updateKeyValueStream
 	AddFSOnlyHandler     = addFSOnlyHandler
 	FilesystemOnlyApply  = filesystemOnlyApply
+	UpdateHomedirsConfig = updateHomedirsConfig
 )
 
 type PlainCoreConfig = plainCoreConfig

overlord/configstate/configcore/homedirs.go

@@ -117,10 +117,8 @@ func updateHomedirsConfig(config string, opts *fsOnlyContext) error {
 		return err
 	}
 
-	var homedirs []string
-	if len(config) > 0 {
-		homedirs = strings.Split(config, ",")
-	}
+	// Update value in dirs
+	homedirs := dirs.SetSnapHomeDirs(config)
 	return configureHomedirsInAppArmorAndReload(homedirs, opts)
 }
 

overlord/configstate/configcore/homedirs_test.go

@@ -142,7 +142,8 @@ func (s *homedirsSuite) TestConfigureApparmorTunableFailure(c *C) {
 		},
 	})
 	c.Check(err, ErrorMatches, "tunable error")
-	c.Check(homedirs, DeepEquals, []string{"/home/existingDir/one", "/home/existingDir/two"})
+	c.Check(homedirs, DeepEquals, []string{filepath.Join(dirs.GlobalRootDir, "/home/existingDir/one"),
+		filepath.Join(dirs.GlobalRootDir, "/home/existingDir/two"), filepath.Join(dirs.GlobalRootDir, "/home")})
 }
 
 func (s *homedirsSuite) TestConfigureApparmorReloadFailure(c *C) {
@@ -243,7 +244,8 @@ func (s *homedirsSuite) TestConfigureHomedirsHappy(c *C) {
 	c.Check(string(contents), Equals, "homedirs=/home/existingDir\n")
 
 	// Check that the AppArmor tunables have been written...
-	c.Check(tunableHomedirs, DeepEquals, []string{"/home/existingDir"})
+	c.Check(tunableHomedirs, DeepEquals, []string{filepath.Join(dirs.GlobalRootDir, "/home/existingDir"),
+		filepath.Join(dirs.GlobalRootDir, "/home")})
 	// ...and that profiles have been reloaded
 	c.Check(reloadProfilesCallCount, Equals, 1)
 	// and finally that snap-confine snippets was called
@@ -309,3 +311,11 @@ func (s *homedirsSuite) TestConfigureHomedirsNotOnCore(c *C) {
 	c.Check(reloadProfilesCallCount, Equals, 0)
 	c.Check(setupSnapConfineSnippetsCalls, Equals, 0)
 }
+
+func (s *homedirsSuite) TestupdateHomedirsConfig(c *C) {
+	config := "/home/homeDir1,/home/homeDirs/homeDir1///,/home/homeDir2/,/home/homeTest/users/"
+	expectedHomeDirs := []string{filepath.Join(dirs.GlobalRootDir, "/home/homeDir1"), filepath.Join(dirs.GlobalRootDir, "/home/homeDirs/homeDir1"),
+		filepath.Join(dirs.GlobalRootDir, "/home/homeDir2"), filepath.Join(dirs.GlobalRootDir, "/home/homeTest/users"), filepath.Join(dirs.GlobalRootDir, "/home")}
+	configcore.UpdateHomedirsConfig(config, nil)
+	c.Check(dirs.SnapHomeDirs(), DeepEquals, expectedHomeDirs)
+}

overlord/configstate/configmgr.go

@@ -23,6 +23,8 @@ package configstate
 import (
 	"regexp"
 
+	"github.com/snapcore/snapd/dirs"
+	"github.com/snapcore/snapd/overlord/configstate/config"
 	"github.com/snapcore/snapd/overlord/configstate/configcore"
 	"github.com/snapcore/snapd/overlord/hookstate"
 	"github.com/snapcore/snapd/overlord/snapstate"
@@ -43,6 +45,16 @@ func MockConfigcoreRun(f func(sysconfig.Device, configcore.RunTransaction) error
 func Init(st *state.State, hookManager *hookstate.HookManager) error {
 	delayedCrossMgrInit()
 
+	// Retrieve home directories
+	st.Lock()
+	defer st.Unlock()
+	tr := config.NewTransaction(st)
+	var homedirs string
+	if err := tr.GetMaybe("core", "homedirs", &homedirs); err != nil {
+		return err
+	}
+	dirs.SetSnapHomeDirs(homedirs)
+
 	// Default configuration is handled via the "default-configure" hook
 	hookManager.Register(regexp.MustCompile("^default-configure$"), newDefaultConfigureHandler)
 

overlord/configstate/configstate_test.go

@@ -308,6 +308,22 @@ func (s *configcoreHijackSuite) SetUpTest(c *C) {
 
 }
 
+func (s *configcoreHijackSuite) TestConfigMngrInitHomeDirs(c *C) {
+	s.o = overlord.Mock()
+	s.state = s.o.State()
+	hookMgr, err := hookstate.Manager(s.state, s.o.TaskRunner())
+	c.Assert(err, IsNil)
+	s.state.Lock()
+	t := config.NewTransaction(s.state)
+	c.Assert(t.Set("core", "homedirs", "/home,/home/department,/users,/users/seniors"), IsNil)
+	t.Commit()
+	s.state.Unlock()
+	err = configstate.Init(s.state, hookMgr)
+	c.Assert(err, IsNil)
+	snapHomeDirs := []string{"/home", "/home/department", "/users", "/users/seniors"}
+	c.Check(dirs.SnapHomeDirs(), DeepEquals, snapHomeDirs)
+}
+
 type witnessManager struct {
 	state     *state.State
 	committed bool