use idiomatic location for security directory and update location of …

…configuration directory

motivation: use idiomatic location for security directory on macOS

changes:
* allow users to customize security directory location with new --security-path CLI option
* on macOS use <user>/Library/org.swiftpm/security for security files and symlink from ~/.swiftpm/security
* move configuration directory from <user>/Library/org.swiftpm to <user>/Library/org.swiftpm/configuration
* add migration code from old configuraiton location to new one
* add and adjust tests
* update docker setup for new locationis

Sources/Basics/FileSystem+Extensions.swift

@@ -21,10 +21,9 @@ extension FileSystem {
     public var dotSwiftPM: AbsolutePath {
         self.homeDirectory.appending(component: ".swiftpm")
     }
-
-    /// SwiftPM security directory
-    public var swiftPMSecurityDirectory: AbsolutePath {
-        self.dotSwiftPM.appending(component: "security")
+
+    fileprivate var idiomaticSwiftPMDirectory: AbsolutePath? {
+        return FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask).first.flatMap { AbsolutePath($0.path) }?.appending(component: "org.swift.swiftpm")
     }
 }
 
@@ -69,52 +68,126 @@ extension FileSystem {
     }
 }
 
-// MARK: - config
+// MARK: - configuration
 
 extension FileSystem {
-    private var idiomaticUserConfigDirectory: AbsolutePath? {
-        return FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask).first.flatMap { AbsolutePath($0.path) }
-    }
-
     /// SwiftPM config directory under user's config directory (if exists)
-    public var swiftPMConfigDirectory: AbsolutePath {
-        if let path = self.idiomaticUserConfigDirectory {
-            return path.appending(component: "org.swift.swiftpm")
+    public var swiftPMConfigurationDirectory: AbsolutePath {
+        if let path = self.idiomaticSwiftPMDirectory {
+            return path.appending(component: "configuration")
         } else {
-            return self.dotSwiftPMConfigDirectory
+            return self.dotSwiftPMConfigurationDirectory
         }
     }
 
-    fileprivate var dotSwiftPMConfigDirectory: AbsolutePath {
-        return self.dotSwiftPM.appending(component: "config")
+    fileprivate var dotSwiftPMConfigurationDirectory: AbsolutePath {
+        return self.dotSwiftPM.appending(component: "configuration")
+    }
+}
+
+extension FileSystem {
+    public func getOrCreateSwiftPMConfigurationDirectory(observabilityScope: ObservabilityScope?) throws -> AbsolutePath {
+        let idiomaticConfigurationDirectory = self.swiftPMConfigurationDirectory
+
+        // temporary 5.6, remove on next version: transition from previous configuration location
+        if !self.exists(idiomaticConfigurationDirectory) {
+            try self.createDirectory(idiomaticConfigurationDirectory, recursive: true)
+        }
+
+        // in the case where ~/.swiftpm/configuration is not the idiomatic location (eg on macOS where its /Users/<user>/Library/org.swift.swiftpm/configuration)
+        if idiomaticConfigurationDirectory != self.dotSwiftPMConfigurationDirectory {
+            // copy the configuration files from old location (eg /Users/<user>/Library/org.swift.swiftpm) to new one (eg /Users/<user>/Library/org.swift.swiftpm/configuration)
+            // but leave them there for backwards compatibility (eg older xcode)
+            let oldConfigDirectory = idiomaticConfigurationDirectory.parentDirectory
+            if self.exists(oldConfigDirectory, followSymlink: false) && self.isDirectory(oldConfigDirectory) {
+                let content = try self.getDirectoryContents(oldConfigDirectory).filter{ !$0.hasSuffix(".lock") }
+                for item in content {
+                    if self.isFile(oldConfigDirectory.appending(component: item)) &&
+                        !self.isSymlink(oldConfigDirectory.appending(component: item)) &&
+                        !self.exists(idiomaticConfigurationDirectory.appending(component: item)) {
+                        observabilityScope?.emit(warning: "Usage of \(oldConfigDirectory.appending(component: item)) has been deprecated. Please delete it and use the new \(idiomaticConfigurationDirectory.appending(component: item)) instead.")
+                        try self.copy(from: oldConfigDirectory.appending(component: item), to: idiomaticConfigurationDirectory.appending(component: item))
+                    }
+                }
+            }
+        // in the case where ~/.swiftpm/configuration is the idiomatic location (eg on Linux)
+        } else {
+            // copy the configuration files from old location (~/.swiftpm/config) to new one (~/.swiftpm/configuration)
+            // but leave them there for backwards compatibility (eg older toolchain)
+            let oldConfigDirectory = self.dotSwiftPM.appending(component: "config")
+            if self.exists(oldConfigDirectory, followSymlink: false) && self.isDirectory(oldConfigDirectory) {
+                let content = try self.getDirectoryContents(oldConfigDirectory).filter{ !$0.hasSuffix(".lock") }
+                for item in content {
+                    if self.isFile(oldConfigDirectory.appending(component: item)) &&
+                        !self.isSymlink(oldConfigDirectory.appending(component: item)) &&
+                        !self.exists(idiomaticConfigurationDirectory.appending(component: item)) {
+                        observabilityScope?.emit(warning: "Usage of \(oldConfigDirectory.appending(component: item)) has been deprecated. Please delete it and use the new \(idiomaticConfigurationDirectory.appending(component: item)) instead.")
+                        try self.copy(from: oldConfigDirectory.appending(component: item), to: idiomaticConfigurationDirectory.appending(component: item))
+                    }
+                }
+            }
+        }
+        // ~temporary 5.6 migration
+
+        // Create idiomatic if necessary
+        if !self.exists(idiomaticConfigurationDirectory) {
+            try self.createDirectory(idiomaticConfigurationDirectory, recursive: true)
+        }
+        // Create ~/.swiftpm if necessary
+        if !self.exists(self.dotSwiftPM) {
+            try self.createDirectory(self.dotSwiftPM, recursive: true)
+        }
+        // Create ~/.swiftpm/configuration symlink if necessary
+        if !self.exists(self.dotSwiftPMConfigurationDirectory, followSymlink: false) {
+            try self.createSymbolicLink(dotSwiftPMConfigurationDirectory, pointingAt: idiomaticConfigurationDirectory, relative: false)
+        }
+
+        return idiomaticConfigurationDirectory
     }
 }
 
+// MARK: - security
+
 extension FileSystem {
-    public func getOrCreateSwiftPMConfigDirectory() throws -> AbsolutePath {
-        let idiomaticConfigDirectory = self.swiftPMConfigDirectory
+    /// SwiftPM security directory under user's security directory (if exists)
+    public var swiftPMSecurityDirectory: AbsolutePath {
+        if let path = self.idiomaticSwiftPMDirectory {
+            return path.appending(component: "security")
+        } else {
+            return self.dotSwiftPMSecurityDirectory
+        }
+    }
 
-        // temporary 5.5, remove on next version: transition from ~/.swiftpm/config to idiomatic location + symbolic link
-        if idiomaticConfigDirectory != self.dotSwiftPMConfigDirectory &&
-            self.exists(self.dotSwiftPMConfigDirectory) && self.isDirectory(self.dotSwiftPMConfigDirectory) &&
-            !self.exists(idiomaticConfigDirectory) {
-            print("transitioning \(self.dotSwiftPMConfigDirectory) to \(idiomaticConfigDirectory)")
-            try self.move(from: self.dotSwiftPMConfigDirectory, to: idiomaticConfigDirectory)
+    fileprivate var dotSwiftPMSecurityDirectory: AbsolutePath {
+        return self.dotSwiftPM.appending(component: "security")
+    }
+}
+
+extension FileSystem {
+    public func getOrCreateSwiftPMSecurityDirectory() throws -> AbsolutePath {
+        let idiomaticSecurityDirectory = self.swiftPMSecurityDirectory
+
+        // temporary 5.6, remove on next version: transition from ~/.swiftpm/security to idiomatic location + symbolic link
+        if idiomaticSecurityDirectory != self.dotSwiftPMSecurityDirectory &&
+            self.exists(self.dotSwiftPMSecurityDirectory) &&
+            self.isDirectory(self.dotSwiftPMSecurityDirectory) {
+            try self.removeFileTree(self.dotSwiftPMSecurityDirectory)
         }
+        // ~temporary 5.6 migration
 
         // Create idiomatic if necessary
-        if !self.exists(idiomaticConfigDirectory) {
-            try self.createDirectory(idiomaticConfigDirectory, recursive: true)
+        if !self.exists(idiomaticSecurityDirectory) {
+            try self.createDirectory(idiomaticSecurityDirectory, recursive: true)
         }
         // Create ~/.swiftpm if necessary
         if !self.exists(self.dotSwiftPM) {
             try self.createDirectory(self.dotSwiftPM, recursive: true)
         }
-        // Create ~/.swiftpm/config symlink if necessary
-        if !self.exists(self.dotSwiftPMConfigDirectory, followSymlink: false) {
-            try self.createSymbolicLink(dotSwiftPMConfigDirectory, pointingAt: idiomaticConfigDirectory, relative: false)
+        // Create ~/.swiftpm/security symlink if necessary
+        if !self.exists(self.dotSwiftPMSecurityDirectory, followSymlink: false) {
+            try self.createSymbolicLink(dotSwiftPMSecurityDirectory, pointingAt: idiomaticSecurityDirectory, relative: false)
         }
-        return idiomaticConfigDirectory
+        return idiomaticSecurityDirectory
     }
 }
 

Sources/Commands/Options.swift

@@ -149,6 +149,9 @@ public struct SwiftToolOptions: ParsableArguments {
     @Option(help: "Specify the shared configuration directory")
     var configPath: AbsolutePath?
 
+    @Option(help: "Specify the shared security directory")
+    var securityPath: AbsolutePath?
+
     /// Disables repository caching.
     @Flag(name: .customLong("repository-cache"), inversion: .prefixedEnableDisable, help: "Use a shared cache when fetching repositories")
     var useRepositoriesCache: Bool = true

Sources/Commands/SwiftTool.swift

@@ -246,6 +246,8 @@ protocol SwiftCommand: ParsableCommand {
 extension SwiftCommand {
     public func run() throws {
         let swiftTool = try SwiftTool(options: swiftOptions)
+        // make sure common directories are created
+        try swiftTool.createSharedDirectories()
         try self.run(swiftTool)
         if swiftTool.observabilityScope.errorsReported || swiftTool.executionStatus == .failure {
             throw ExitCode.failure
@@ -634,29 +636,39 @@ public class SwiftTool {
         }
 
         do {
-            return try localFileSystem.getOrCreateSwiftPMConfigDirectory()
+            return try localFileSystem.getOrCreateSwiftPMConfigurationDirectory(observabilityScope: self.observabilityScope)
         } catch {
             self.observabilityScope.emit(warning: "Failed creating default configuration location, \(error)")
             return .none
         }
     }
 
     private func getSharedSecurityDirectory() throws -> AbsolutePath? {
-        do {
-            let fileSystem = localFileSystem
-            let sharedSecurityDirectory = fileSystem.swiftPMSecurityDirectory
-            if !fileSystem.exists(sharedSecurityDirectory) {
-                try fileSystem.createDirectory(sharedSecurityDirectory, recursive: true)
+        if let explicitSecurityPath = options.securityPath {
+            // Create the explicit security path if necessary
+            if !localFileSystem.exists(explicitSecurityPath) {
+                try localFileSystem.createDirectory(explicitSecurityPath, recursive: true)
             }
+            return explicitSecurityPath
+        }
+
+        do {
+            let sharedSecurityDirectory = try localFileSystem.getOrCreateSwiftPMSecurityDirectory()
             // And make sure we can write files (locking the directory writes a lock file)
-            try fileSystem.withLock(on: sharedSecurityDirectory, type: .exclusive) { }
+            try localFileSystem.withLock(on: sharedSecurityDirectory, type: .exclusive) { }
             return sharedSecurityDirectory
         } catch {
-            self.observabilityScope.emit(warning: "Failed creating shared security directory: \(error)")
+            self.observabilityScope.emit(warning: "Failed creating default security location, \(error)")
             return .none
         }
     }
 
+    fileprivate func createSharedDirectories() throws {
+        _ = try getSharedCacheDirectory()
+        _ = try getSharedConfigurationDirectory()
+        _ = try getSharedSecurityDirectory()
+    }
+
     /// Returns the currently active workspace.
     func getActiveWorkspace() throws -> Workspace {
         if let workspace = _workspace {

Sources/PackageCollections/Providers/JSONPackageCollectionProvider.swift

@@ -55,7 +55,7 @@ struct JSONPackageCollectionProvider: PackageCollectionProvider {
         self.decoder = JSONDecoder.makeWithDefaults()
         self.validator = JSONModel.Validator(configuration: configuration.validator)
         self.signatureValidator = signatureValidator ?? PackageCollectionSigning(
-            trustedRootCertsDir: configuration.trustedRootCertsDir ?? fileSystem.swiftPMConfigDirectory.appending(component: "trust-root-certs").asURL,
+            trustedRootCertsDir: configuration.trustedRootCertsDir ?? fileSystem.swiftPMConfigurationDirectory.appending(component: "trust-root-certs").asURL,
             additionalTrustedRootCerts: sourceCertPolicy.allRootCerts.map { Array($0) },
             observabilityScope: observabilityScope,
             callbackQueue: .sharedConcurrent

Sources/PackageCollections/Storage/FilePackageCollectionsSourcesStorage.swift

@@ -26,7 +26,7 @@ struct FilePackageCollectionsSourcesStorage: PackageCollectionsSourcesStorage {
     init(fileSystem: FileSystem = localFileSystem, path: AbsolutePath? = nil) {
         self.fileSystem = fileSystem
 
-        self.path = path ?? fileSystem.swiftPMConfigDirectory.appending(component: "collections.json")
+        self.path = path ?? fileSystem.swiftPMConfigurationDirectory.appending(component: "collections.json")
         self.encoder = JSONEncoder.makeWithDefaults()
         self.decoder = JSONDecoder.makeWithDefaults()
     }

Sources/SPMTestSupport/MockWorkspace.swift

@@ -241,7 +241,7 @@ public final class MockWorkspace {
                 resolvedVersionsFile: self.sandbox.appending(component: "Package.resolved"),
                 sharedSecurityDirectory: self.fileSystem.swiftPMSecurityDirectory,  
                 sharedCacheDirectory: self.fileSystem.swiftPMCacheDirectory,
-                sharedConfigurationDirectory: self.fileSystem.swiftPMConfigDirectory
+                sharedConfigurationDirectory: self.fileSystem.swiftPMConfigurationDirectory
             ),
             mirrors: self.mirrors,
             customToolsVersion: self.toolsVersion,

Sources/Workspace/WorkspaceConfiguration.swift

@@ -126,7 +126,7 @@ extension Workspace {
                 resolvedVersionsFile: DefaultLocations.resolvedVersionsFile(forRootPackage: rootPath),
                 sharedSecurityDirectory: fileSystem.swiftPMSecurityDirectory,
                 sharedCacheDirectory: fileSystem.swiftPMCacheDirectory,
-                sharedConfigurationDirectory: fileSystem.swiftPMConfigDirectory
+                sharedConfigurationDirectory: fileSystem.swiftPMConfigurationDirectory
             )
         }
     }

Tests/FunctionalTests/MiscellaneousTests.swift

@@ -415,7 +415,7 @@ class MiscellaneousTestCase: XCTestCase {
             XCTAssert(output.contains("does not exist"), "Error from git was not propagated to process output: \(output)")
         }
     }
-    
+
     func testLocalPackageUsedAsURL() throws {
         fixture(name: "Miscellaneous/LocalPackageAsURL", createGitRepo: false) { prefix in
             // This fixture has a setup that is trying to use a local package
@@ -431,7 +431,7 @@ class MiscellaneousTestCase: XCTestCase {
             XCTAssert(output.contains("Cannot clone from local directory"), "Didn't find expected output: \(output)")
         }
     }
-    
+
     func testUnicode() {
         #if !os(Linux) && !os(Android) // TODO: - Linux has trouble with this and needs investigation.
         fixture(name: "Miscellaneous/Unicode") { prefix in
@@ -505,7 +505,7 @@ class MiscellaneousTestCase: XCTestCase {
             XCTAssertMatch(stderr, .contains("warning: '--generate-linuxmain' option is deprecated"))
         }
     }
-    
+
     func testGenerateLinuxMain() {
         #if os(macOS)
         fixture(name: "Miscellaneous/TestDiscovery/Simple") { path in
@@ -552,13 +552,13 @@ class MiscellaneousTestCase: XCTestCase {
         }
         #endif
     }
-    
+
     func testTestsCanLinkAgainstExecutable() throws {
         // Check if the host compiler supports the '-entry-point-function-name' flag.
         #if swift(<5.5)
         try XCTSkipIf(true, "skipping because host compiler doesn't support '-entry-point-function-name'")
         #endif
-        
+
         fixture(name: "Miscellaneous/TestableExe") { prefix in
             do {
                 let (stdout, _) = try executeSwiftTest(prefix)
@@ -648,6 +648,17 @@ class MiscellaneousTestCase: XCTestCase {
             try SwiftPMProduct.SwiftBuild.execute(["--cache-path", customCachePath.pathString], packagePath: path)
             XCTAssertDirectoryExists(customCachePath)
         }
+
+        fixture(name: "Miscellaneous/Simple") { path in
+            try localFileSystem.chmod(.userUnWritable, path: path)
+            let customCachePath = path.appending(components: "custom", "cache")
+            XCTAssertNoSuchPath(customCachePath)
+            let result = try SwiftPMProduct.SwiftBuild.executeProcess(["--cache-path", customCachePath.pathString], packagePath: path)
+            XCTAssert(result.exitStatus != .terminated(code: 0))
+            let output = try result.utf8stderrOutput()
+            XCTAssert(output.contains("error: You don’t have permission"), "expected permissions error")
+            XCTAssertNoSuchPath(customCachePath)
+        }
     }
 
     func testCustomConfigPath() {
@@ -657,5 +668,36 @@ class MiscellaneousTestCase: XCTestCase {
             try SwiftPMProduct.SwiftBuild.execute(["--config-path", customConfigPath.pathString], packagePath: path)
             XCTAssertDirectoryExists(customConfigPath)
         }
+
+        fixture(name: "Miscellaneous/Simple") { path in
+            try localFileSystem.chmod(.userUnWritable, path: path)
+            let customConfigPath = path.appending(components: "custom", "config")
+            XCTAssertNoSuchPath(customConfigPath)
+            let result = try SwiftPMProduct.SwiftBuild.executeProcess(["--config-path", customConfigPath.pathString], packagePath: path)
+            XCTAssert(result.exitStatus != .terminated(code: 0))
+            let output = try result.utf8stderrOutput()
+            XCTAssert(output.contains("error: You don’t have permission"), "expected permissions error")
+            XCTAssertNoSuchPath(customConfigPath)
+        }
+    }
+
+    func testCustomSecurityPath() {
+        fixture(name: "Miscellaneous/Simple") { path in
+            let customSecurityPath = path.appending(components: "custom", "security")
+            XCTAssertNoSuchPath(customSecurityPath)
+            try SwiftPMProduct.SwiftBuild.execute(["--security-path", customSecurityPath.pathString], packagePath: path)
+            XCTAssertDirectoryExists(customSecurityPath)
+        }
+
+        fixture(name: "Miscellaneous/Simple") { path in
+            try localFileSystem.chmod(.userUnWritable, path: path)
+            let customSecurityPath = path.appending(components: "custom", "security")
+            XCTAssertNoSuchPath(customSecurityPath)
+            let result = try SwiftPMProduct.SwiftBuild.executeProcess(["--security-path", customSecurityPath.pathString], packagePath: path)
+            XCTAssert(result.exitStatus != .terminated(code: 0))
+            let output = try result.utf8stderrOutput()
+            XCTAssert(output.contains("error: You don’t have permission"), "expected permissions error")
+            XCTAssertNoSuchPath(customSecurityPath)
+        }
     }
 }

Utilities/Docker/docker-compose.2004.55.yaml

@@ -16,7 +16,6 @@ services:
       args:
         ubuntu_version: "focal"
         swift_version: "5.5"
-        base_image: "swiftlang/swift:nightly-5.5-focal"
 
   build:
     image: swift-package-manager:20.04-5.5

Utilities/Docker/docker-compose.yaml

@@ -27,7 +27,9 @@ services:
       - ~/.ssh:/root/.ssh
       - ~/.cache:/root/.cache
       - ~/.swiftpm/cache:/root/.swiftpm/cache
-      - ~/.swiftpm/config:/root/.swiftpm/config
+      - ~/.swiftpm/configuration:/root/.swiftpm/config # old location, remove after 5.6
+      - ~/.swiftpm/configuration:/root/.swiftpm/configuration
+      - ~/.swiftpm/security:/root/.swiftpm/security
       # swift-package-manager code
       - ../..:/code/swift-package-manager:z
       # bootstrap script requires dependencies to be pre-fetched and in a specific place