Snippets (#7)

* Add snippet-build tool

The `snippet-build` tool crawls the `Snippets` subdirectory of a
Swift Package if present, parses the `.swift` files within, and
creates snippets and snippet groups. If there are any snippets,
they are serialized into a Symbol Graph JSON file into an output directory.

rdar://89650557

* Use 'Snippets' as the default group name

Group names with spaces cause problems with path components / URLs.

* Pass snippets directory to `snippet-build`

Instead of expecting a particular directory name, just in case.

Add some logging to the tool.

* Call `snippet-build` on snippet files

Forward generated Symbol Graph JSON to the docc invocation.

* Pass snippets directory instead of package path

Just in case.

* Temporarily remove ArgumentParser and TSC dependencies

Plugins can't have their own dependencies because they currently
share the same dependency graph as the clients opting into the plugin.

rdar://89789701

* Refactor snippet generation

* Add tests for SnippetBuilder

* Fix typo "ot" -> "to"

Co-authored-by: Ethan Kusters <[email protected]>

* Use @main for snippet-build

* Remove local dependencies

Not necessary for swift-docc-plugin builds at this time.

* Remove debug logging from SnippetBuildCommand

* Add unit tests for snippet parsing

* Add integration tests for building projects with snippets

* [Snippets] Logic: Use total snippet count to early-return

It's enough to check the total number of snippets rather than
checking the groups top-down.

* [Snippets] Use trimmingCharacters for newline trimming

When trimming leading and trailing newlines, use the existing
`trimmingCharacters(in:)` API. NFC.

* Fix snippet builder tests

* Fix pipe output deadlock

* Use shared SwiftPM cache when building in integration tests

Co-authored-by: Ethan Kusters <[email protected]>

IntegrationTests/Package.swift

@@ -29,6 +29,7 @@ let package = Package(
                 .copy("Fixtures/SingleExecutableTarget"),
                 .copy("Fixtures/MixedTargets"),
                 .copy("Fixtures/TargetWithDocCCatalog"),
+                .copy("Fixtures/PackageWithSnippets"),
             ]
         ),
     ]

IntegrationTests/Tests/Fixtures/PackageWithSnippets/Package.swift

@@ -0,0 +1,30 @@
+// swift-tools-version: 5.6
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+import Foundation
+import PackageDescription
+
+let package = Package(
+    name: "PackageWithSnippets",
+    targets: [
+        .target(name: "Library"),
+    ]
+)
+
+// We only expect 'swift-docc-plugin' to be a sibling when this package
+// is running as part of a test.
+//
+// This allows the package to compile outside of tests for easier
+// test development.
+if FileManager.default.fileExists(atPath: "../swift-docc-plugin") {
+    package.dependencies += [
+        .package(path: "../swift-docc-plugin"),
+    ]
+}

IntegrationTests/Tests/Fixtures/PackageWithSnippets/Sources/Library/Library.swift

@@ -0,0 +1,14 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+/// This is BestStruct's documentation.
+///
+/// Is best.
+public struct BestStruct {
+    public func best() {}
+}

IntegrationTests/Tests/Fixtures/PackageWithSnippets/_Snippets/TestTest.swift

@@ -0,0 +1,17 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+//! Create a foo.
+import Library
+
+let best = BestStruct()
+best.best()
+
+// MARK: Hide
+
+print(best)

IntegrationTests/Tests/SnippetDocumentationGenerationTests.swift

@@ -0,0 +1,136 @@
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+
+import XCTest
+
+final class SnippetDocumentationGenerationTests: XCTestCase {
+    func testGenerateDocumentationForPackageWithSnippets() throws {
+        let result = try swiftPackage(
+            "generate-documentation", "--enable-experimental-snippet-support",
+            workingDirectory: try setupTemporaryDirectoryForFixture(named: "PackageWithSnippets")
+        )
+
+        result.assertExitStatusEquals(0)
+        XCTAssertEqual(result.referencedDocCArchives.count, 1)
+
+        let doccArchiveURL = try XCTUnwrap(result.referencedDocCArchives.first)
+
+        let dataDirectoryContents = try filesIn(.dataSubdirectory, of: doccArchiveURL)
+
+        XCTAssertEqual(
+            Set(dataDirectoryContents.map(\.lastTwoPathComponents)),
+            [
+                "documentation/library.json",
+                "library/beststruct.json",
+                "beststruct/best().json",
+            ]
+        )
+
+        let subDirectoriesOfSymbolGraphDirectory = try FileManager.default.contentsOfDirectory(
+            at: result.symbolGraphsDirectory,
+            includingPropertiesForKeys: nil
+        )
+
+        XCTAssertEqual(
+            Set(subDirectoriesOfSymbolGraphDirectory.map(\.lastTwoPathComponents)),
+            [
+                "symbol-graphs/snippet-symbol-graphs",
+                "symbol-graphs/unified-symbol-graphs",
+            ]
+        )
+    }
+
+    func testGenerateDocumentationForPackageWithSnippetsWithoutExperimentalFlag() throws {
+        let result = try swiftPackage(
+            "generate-documentation",
+            workingDirectory: try setupTemporaryDirectoryForFixture(named: "PackageWithSnippets")
+        )
+
+        result.assertExitStatusEquals(0)
+        XCTAssertEqual(result.referencedDocCArchives.count, 1)
+
+        let doccArchiveURL = try XCTUnwrap(result.referencedDocCArchives.first)
+
+        let dataDirectoryContents = try filesIn(.dataSubdirectory, of: doccArchiveURL)
+
+        XCTAssertEqual(
+            Set(dataDirectoryContents.map(\.lastTwoPathComponents)),
+            [
+                "documentation/library.json",
+                "library/beststruct.json",
+                "beststruct/best().json",
+            ]
+        )
+
+        XCTAssertFalse(
+            FileManager.default.fileExists(atPath: result.symbolGraphsDirectory.path),
+            "Unified symbol graph directory created when experimental snippet support flag was not passed."
+        )
+    }
+
+    func testPreviewDocumentationWithSnippets() throws {
+        let outputDirectory = try temporaryDirectory().appendingPathComponent("output")
+
+        let port = try getAvailablePort()
+
+        let process = try swiftPackageProcess(
+            [
+                "--disable-sandbox",
+                "--allow-writing-to-directory", outputDirectory.path,
+                "preview-documentation",
+                "--port", port,
+                "--output-path", outputDirectory.path,
+                "--enable-experimental-snippet-support"
+            ],
+            workingDirectory: try setupTemporaryDirectoryForFixture(named: "PackageWithSnippets")
+        )
+
+        let outputPipe = Pipe()
+        process.standardOutput = outputPipe
+        process.standardError = outputPipe
+
+        try process.run()
+
+        var previewServerHasStarted: Bool {
+            // We expect docc to emit a `data` directory at the root of the
+            // given output path when it's finished compilation.
+            //
+            // At this point we can expect that the preview server will start imminently.
+            return FileManager.default.fileExists(
+                atPath: outputDirectory.appendingPathComponent("data", isDirectory: true).path
+            )
+        }
+
+        let previewServerHasStartedExpectation = expectation(description: "Preview server started.")
+
+        let checkPreviewServerTimer = Timer.scheduledTimer(withTimeInterval: 0.25, repeats: true) { timer in
+            if previewServerHasStarted {
+                previewServerHasStartedExpectation.fulfill()
+                timer.invalidate()
+            }
+        }
+
+        wait(for: [previewServerHasStartedExpectation], timeout: 15)
+        checkPreviewServerTimer.invalidate()
+
+        guard process.isRunning else {
+            XCTFail(
+                """
+                Preview server failed to start.
+
+                Process output:
+                \(try outputPipe.asString() ?? "nil")
+                """
+            )
+            return
+        }
+
+        // Send an interrupt to the SwiftPM parent process
+        process.interrupt()
+    }
+}

IntegrationTests/Tests/Utility/XCTestCase+swiftPackage.swift

@@ -9,44 +9,105 @@
 import Foundation
 import XCTest
 
+private let temporarySwiftPMDirectory = Bundle.module.resourceURL!
+    .appendingPathComponent(
+        "Temporary-SwiftPM-Caching-Directory-\(ProcessInfo.processInfo.globallyUniqueString)",
+        isDirectory: true
+    )
+
+private let swiftPMBuildDirectory = temporarySwiftPMDirectory
+    .appendingPathComponent("BuildDirectory", isDirectory: true)
+
+private let swiftPMCacheDirectory = temporarySwiftPMDirectory
+    .appendingPathComponent("SharedCacheDirectory", isDirectory: true)
+
+private let doccPreviewOutputDirectory = swiftPMBuildDirectory
+    .appendingPathComponent("plugins", isDirectory: true)
+    .appendingPathComponent("Swift-DocC Preview", isDirectory: true)
+    .appendingPathComponent("outputs", isDirectory: true)
+
+private let doccConvertOutputDirectory = swiftPMBuildDirectory
+    .appendingPathComponent("plugins", isDirectory: true)
+    .appendingPathComponent("Swift-DocC", isDirectory: true)
+    .appendingPathComponent("outputs", isDirectory: true)
+
 extension XCTestCase {
     func swiftPackageProcess(
         _ arguments: [CustomStringConvertible],
         workingDirectory directoryURL: URL? = nil
     ) throws -> Process {
+        // Clear any existing plugins state
+        try? FileManager.default.removeItem(at: doccPreviewOutputDirectory)
+        try? FileManager.default.removeItem(at: doccConvertOutputDirectory)
+
         let process = Process()
         process.executableURL = try swiftExecutableURL
         process.environment = ProcessInfo.processInfo.environment
 
-        process.arguments = ["package"] + arguments.map(\.description)
+        process.arguments = [
+            "package",
+            "--cache-path", swiftPMCacheDirectory.path,
+            "--build-path", swiftPMBuildDirectory.path,
+        ] + arguments.map(\.description)
         process.currentDirectoryURL = directoryURL
         return process
     }
 
     /// Invokes the swift package CLI with the given arguments.
     func swiftPackage(
         _ arguments: CustomStringConvertible...,
-        workingDirectory directoryURL: URL? = nil
+        workingDirectory directoryURL: URL
     ) throws -> SwiftInvocationResult {
         let process = try swiftPackageProcess(arguments, workingDirectory: directoryURL)
 
-        let standardOutput = Pipe()
-        let standardError = Pipe()
-
-        process.standardOutput = standardOutput
-        process.standardError = standardError
+        let standardOutputPipe = Pipe()
+        let standardErrorPipe = Pipe()
 
+        process.standardOutput = standardOutputPipe
+        process.standardError = standardErrorPipe
+
+        let processQueue = DispatchQueue(label: "process")
+        var standardOutputData = Data()
+        var standardErrorData = Data()
+
+        standardOutputPipe.fileHandleForReading.readabilityHandler = { handle in
+            let data = handle.availableData
+            processQueue.async {
+                standardOutputData.append(data)
+            }
+        }
+
+        standardErrorPipe.fileHandleForReading.readabilityHandler = { handle in
+            let data = handle.availableData
+            processQueue.async {
+                standardErrorData.append(data)
+            }
+        }
+
         try process.run()
         process.waitUntilExit()
-
-        return SwiftInvocationResult(
-            workingDirectory: directoryURL,
-            swiftExecutable: try swiftExecutableURL,
-            arguments: arguments,
-            standardOutput: try standardOutput.asString() ?? "",
-            standardError: try standardError.asString() ?? "",
-            exitStatus: Int(process.terminationStatus)
-        )
+
+        standardOutputPipe.fileHandleForReading.readabilityHandler = nil
+        standardErrorPipe.fileHandleForReading.readabilityHandler = nil
+
+        processQueue.async {
+            standardOutputPipe.fileHandleForReading.closeFile()
+            standardErrorPipe.fileHandleForReading.closeFile()
+        }
+
+        return try processQueue.sync {
+            let standardOutputString = String(data: standardOutputData, encoding: .utf8)
+            let standardErrorString = String(data: standardErrorData, encoding: .utf8)
+
+            return SwiftInvocationResult(
+                workingDirectory: directoryURL,
+                swiftExecutable: try swiftExecutableURL,
+                arguments: arguments.map(\.description),
+                standardOutput: standardOutputString ?? "",
+                standardError: standardErrorString ?? "",
+                exitStatus: Int(process.terminationStatus)
+            )
+        }
     }
 
     private var swiftExecutableURL: URL {
@@ -74,9 +135,9 @@ extension XCTestCase {
 }
 
 struct SwiftInvocationResult {
-    let workingDirectory: URL?
+    let workingDirectory: URL
     let swiftExecutable: URL
-    let arguments: [CustomStringConvertible]
+    let arguments: [String]
     let standardOutput: String
     let standardError: String
     let exitStatus: Int
@@ -92,7 +153,21 @@ struct SwiftInvocationResult {
             }
             .compactMap(URL.init(fileURLWithPath:))
     }
-
+
+    var pluginOutputsDirectory: URL {
+        if arguments.contains("preview-documentation") {
+            return doccPreviewOutputDirectory
+        } else {
+            return doccConvertOutputDirectory
+        }
+    }
+
+    var symbolGraphsDirectory: URL {
+        return pluginOutputsDirectory
+            .appendingPathComponent(".build", isDirectory: true)
+            .appendingPathComponent("symbol-graphs", isDirectory: true)
+    }
+
     private func gatherShellEnvironmentInfo() throws -> String {
         let gatherEnvironmentProcess = Process.shell(
             """