diff --git a/api/v1/mapping/src/commonMain/kotlin/Mappings.kt b/api/v1/mapping/src/commonMain/kotlin/Mappings.kt index fcfd49ce9..115c099c1 100644 --- a/api/v1/mapping/src/commonMain/kotlin/Mappings.kt +++ b/api/v1/mapping/src/commonMain/kotlin/Mappings.kt @@ -78,6 +78,7 @@ import org.eclipse.apoapsis.ortserver.api.v1.model.Severity as ApiSeverity import org.eclipse.apoapsis.ortserver.api.v1.model.SortDirection as ApiSortDirection import org.eclipse.apoapsis.ortserver.api.v1.model.SortProperty as ApiSortProperty import org.eclipse.apoapsis.ortserver.api.v1.model.SourceCodeOrigin as ApiSourceCodeOrigin +import org.eclipse.apoapsis.ortserver.api.v1.model.SubmoduleFetchStrategy as ApiSubmoduleFetchStrategy import org.eclipse.apoapsis.ortserver.api.v1.model.User as ApiUser import org.eclipse.apoapsis.ortserver.api.v1.model.VcsInfo as ApiVcsInfo import org.eclipse.apoapsis.ortserver.api.v1.model.Vulnerability as ApiVulnerability @@ -122,6 +123,7 @@ import org.eclipse.apoapsis.ortserver.model.ScannerJobConfiguration import org.eclipse.apoapsis.ortserver.model.Secret import org.eclipse.apoapsis.ortserver.model.Severity import org.eclipse.apoapsis.ortserver.model.SourceCodeOrigin +import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy import org.eclipse.apoapsis.ortserver.model.User import org.eclipse.apoapsis.ortserver.model.VulnerabilityRating import org.eclipse.apoapsis.ortserver.model.VulnerabilityWithAccumulatedData @@ -195,6 +197,7 @@ fun AnalyzerJobConfiguration.mapToApi() = enabledPackageManagers, environmentConfig?.mapToApi(), recursiveCheckout, + submoduleFetchStrategy?.mapToApi(), packageCurationProviders.map { it.mapToApi() }, packageManagerOptions?.mapValues { it.value.mapToApi() }, repositoryConfigPath, @@ -208,6 +211,7 @@ fun ApiAnalyzerJobConfiguration.mapToModel() = enabledPackageManagers, environmentConfig?.mapToModel(), recursiveCheckout, + submoduleFetchStrategy?.mapToModel(), packageCurationProviders?.map { it.mapToModel() }.orEmpty(), packageManagerOptions?.mapValues { it.value.mapToModel() }, repositoryConfigPath, @@ -775,3 +779,15 @@ fun VulnerabilityWithAccumulatedData.mapToApi() = ApiProductVulnerability( ortRunIds = ortRunIds, repositoriesCount = repositoriesCount ) + +fun SubmoduleFetchStrategy.mapToApi() = when (this) { + SubmoduleFetchStrategy.DISABLED -> ApiSubmoduleFetchStrategy.DISABLED + SubmoduleFetchStrategy.TOP_LEVEL_ONLY -> ApiSubmoduleFetchStrategy.TOP_LEVEL_ONLY + SubmoduleFetchStrategy.FULLY_RECURSIVE -> ApiSubmoduleFetchStrategy.FULLY_RECURSIVE +} + +fun ApiSubmoduleFetchStrategy.mapToModel() = when (this) { + ApiSubmoduleFetchStrategy.DISABLED -> SubmoduleFetchStrategy.DISABLED + ApiSubmoduleFetchStrategy.TOP_LEVEL_ONLY -> SubmoduleFetchStrategy.TOP_LEVEL_ONLY + ApiSubmoduleFetchStrategy.FULLY_RECURSIVE -> SubmoduleFetchStrategy.FULLY_RECURSIVE +} diff --git a/api/v1/model/src/commonMain/kotlin/JobConfigurations.kt b/api/v1/model/src/commonMain/kotlin/JobConfigurations.kt index 9d9318764..48ef57f68 100644 --- a/api/v1/model/src/commonMain/kotlin/JobConfigurations.kt +++ b/api/v1/model/src/commonMain/kotlin/JobConfigurations.kt @@ -76,9 +76,17 @@ data class AnalyzerJobConfiguration( /** * A flag indicating whether the submodules of the repository should be downloaded during the download process. * If set to `true`, submodules will be downloaded; if `false`, they will be ignored. + * + * Note: This attribute is deprecated and will be removed in a future release. Use [submoduleFetchStrategy] instead. + * */ val recursiveCheckout: Boolean = true, + /** + * The strategy to use for fetching submodules. + */ + val submoduleFetchStrategy: SubmoduleFetchStrategy? = null, + /** * The list of package curation providers to use. */ @@ -372,3 +380,24 @@ data class NotifierJobConfiguration( */ val jira: JiraNotificationConfiguration? = null ) + +@Serializable +/** + * The strategy to use for fetching submodules. + */ +enum class SubmoduleFetchStrategy { + /** + * Don't fetch submodules at all. + */ + DISABLED, + + /** + * Only fetch the top level of submodules. + */ + TOP_LEVEL_ONLY, + + /** + * Fetch all nested submodules recursively. + */ + FULLY_RECURSIVE +} diff --git a/core/src/main/kotlin/apiDocs/RepositoriesDocs.kt b/core/src/main/kotlin/apiDocs/RepositoriesDocs.kt index 0d9b7cd47..8932afbc6 100644 --- a/core/src/main/kotlin/apiDocs/RepositoriesDocs.kt +++ b/core/src/main/kotlin/apiDocs/RepositoriesDocs.kt @@ -64,6 +64,7 @@ import org.eclipse.apoapsis.ortserver.api.v1.model.Secret import org.eclipse.apoapsis.ortserver.api.v1.model.SortDirection import org.eclipse.apoapsis.ortserver.api.v1.model.SortProperty import org.eclipse.apoapsis.ortserver.api.v1.model.SourceCodeOrigin +import org.eclipse.apoapsis.ortserver.api.v1.model.SubmoduleFetchStrategy.FULLY_RECURSIVE import org.eclipse.apoapsis.ortserver.api.v1.model.UpdateRepository import org.eclipse.apoapsis.ortserver.api.v1.model.UpdateSecret import org.eclipse.apoapsis.ortserver.api.v1.model.Username @@ -111,6 +112,7 @@ internal val fullJobConfigurations = JobConfigurations( ) ), recursiveCheckout = true, + submoduleFetchStrategy = FULLY_RECURSIVE, skipExcluded = true ), advisor = AdvisorJobConfiguration( diff --git a/model/src/commonMain/kotlin/JobConfigurations.kt b/model/src/commonMain/kotlin/JobConfigurations.kt index 7bb84dcf6..0026160db 100644 --- a/model/src/commonMain/kotlin/JobConfigurations.kt +++ b/model/src/commonMain/kotlin/JobConfigurations.kt @@ -83,9 +83,18 @@ data class AnalyzerJobConfiguration( /** * A flag indicating whether the submodules of the repository should be downloaded during the download process. * If set to `true`, submodules will be downloaded; if `false`, they will be ignored. + * + * Note: This attribute is deprecated and will be removed in a future release. Use [submoduleFetchStrategy] instead. */ val recursiveCheckout: Boolean = true, + /** + * The strategy to use for fetching submodules. + * + * Note: Submodule fetch strategy [SubmoduleFetchStrategy.TOP_LEVEL_ONLY] is only supported for Git repositories. + */ + val submoduleFetchStrategy: SubmoduleFetchStrategy? = null, + /** * The list of package curation providers to use. */ @@ -387,3 +396,24 @@ data class NotifierJobConfiguration( */ val jira: JiraNotificationConfiguration? = null ) + +@Serializable +/** + * The strategy to use for fetching submodules. + */ +enum class SubmoduleFetchStrategy { + /** + * Don't fetch submodules at all. + */ + DISABLED, + + /** + * Only fetch the top level of submodules. + */ + TOP_LEVEL_ONLY, + + /** + * Fetch all nested submodules recursively. + */ + FULLY_RECURSIVE +} diff --git a/workers/analyzer/src/main/kotlin/analyzer/AnalyzerDownloader.kt b/workers/analyzer/src/main/kotlin/analyzer/AnalyzerDownloader.kt index e86389145..c1be233f9 100644 --- a/workers/analyzer/src/main/kotlin/analyzer/AnalyzerDownloader.kt +++ b/workers/analyzer/src/main/kotlin/analyzer/AnalyzerDownloader.kt @@ -21,8 +21,13 @@ package org.eclipse.apoapsis.ortserver.workers.analyzer import java.io.File +import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy + +import org.ossreviewtoolkit.downloader.VcsHost import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.plugins.api.PluginConfig import org.ossreviewtoolkit.utils.ort.createOrtTempDir import org.slf4j.LoggerFactory @@ -34,22 +39,68 @@ class AnalyzerDownloader { repositoryUrl: String, revision: String, path: String = "", - recursiveCheckout: Boolean = true + recursiveCheckout: Boolean = true, // Deprecated: Will be removed in a future release + submoduleFetchStrategy: SubmoduleFetchStrategy? = null ): File { logger.info("Downloading repository '$repositoryUrl' revision '$revision'.") val outputDir = createOrtTempDir("analyzer-worker") - val vcs = VersionControlSystem.forUrl(repositoryUrl) + val config = buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy) + val vcs = VersionControlSystem.forUrl(repositoryUrl, config) requireNotNull(vcs) { "Could not determine the VCS for URL '$repositoryUrl'." } - val vcsInfo = VcsInfo(vcs.type, repositoryUrl, revision, path) + val vcsInfo = VcsInfo( + type = vcs.type, + url = repositoryUrl, + revision = revision, + path = path + ) + + // The [submoduleFetchStrategy] parameter takes precedence over the deprecated [recursiveCheckout] parameter. + val combinedRecursiveCheckout = evaluateRecursiveCheckoutParameter(recursiveCheckout, submoduleFetchStrategy) val workingTree = vcs.initWorkingTree(outputDir, vcsInfo) - vcs.updateWorkingTree(workingTree, revision, recursive = recursiveCheckout).getOrThrow() + vcs.updateWorkingTree(workingTree, revision, recursive = combinedRecursiveCheckout).getOrThrow() logger.info("Finished downloading '$repositoryUrl' revision '$revision'.") return outputDir } + + /** + * Build custom [PluginConfig] for Git VCS if the [submoduleFetchStrategy] is + * [SubmoduleFetchStrategy.TOP_LEVEL_ONLY]. + */ + internal fun buildCustomVcsPluginConfigMap( + repositoryUrl: String, submoduleFetchStrategy: SubmoduleFetchStrategy? + ) = + if (submoduleFetchStrategy == SubmoduleFetchStrategy.TOP_LEVEL_ONLY) { + val vcsType = VcsHost.parseUrl(repositoryUrl).type + require(vcsType == VcsType.GIT) { + "Submodule fetch strategy TOP_LEVEL_ONLY is only supported for Git repositories, " + + "but got VCS type '$vcsType'." + } + mapOf( + VcsType.GIT.toString() to PluginConfig( + options = mapOf("updateNestedSubmodules" to false.toString()) + ) + ) + } else { + emptyMap() + } + + /** + * Evaluate the [recursiveCheckout] and the [submoduleFetchStrategy] parameter to determine if the working tree + * should be checked out recursively. The [submoduleFetchStrategy] parameter takes precedence over the + * deprecated [recursiveCheckout] parameter. + */ + internal fun evaluateRecursiveCheckoutParameter( + recursiveCheckout: Boolean, submoduleFetchStrategy: SubmoduleFetchStrategy? + ) = + if (submoduleFetchStrategy != null) { + submoduleFetchStrategy != SubmoduleFetchStrategy.DISABLED + } else { + recursiveCheckout + } } diff --git a/workers/analyzer/src/main/kotlin/analyzer/AnalyzerWorker.kt b/workers/analyzer/src/main/kotlin/analyzer/AnalyzerWorker.kt index 2e046d1a8..93630d73b 100644 --- a/workers/analyzer/src/main/kotlin/analyzer/AnalyzerWorker.kt +++ b/workers/analyzer/src/main/kotlin/analyzer/AnalyzerWorker.kt @@ -74,7 +74,8 @@ internal class AnalyzerWorker( repository.url, ortRun.revision, ortRun.path.orEmpty(), - job.configuration.recursiveCheckout + job.configuration.recursiveCheckout, + job.configuration.submoduleFetchStrategy ) val resolvedEnvConfig = environmentService.setUpEnvironment( diff --git a/workers/analyzer/src/test/kotlin/AnalyzerDownloaderTest.kt b/workers/analyzer/src/test/kotlin/AnalyzerDownloaderTest.kt index 8685fcd32..0c0897730 100644 --- a/workers/analyzer/src/test/kotlin/AnalyzerDownloaderTest.kt +++ b/workers/analyzer/src/test/kotlin/AnalyzerDownloaderTest.kt @@ -26,15 +26,20 @@ import io.kotest.matchers.file.shouldContainFile import io.kotest.matchers.file.shouldNotExist import io.kotest.matchers.maps.shouldBeEmpty import io.kotest.matchers.maps.shouldContainExactly +import io.kotest.matchers.maps.shouldNotBeEmpty import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.should +import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNot import java.io.IOException +import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy + import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.plugins.api.PluginConfig class AnalyzerDownloaderTest : WordSpec({ val downloader = AnalyzerDownloader() @@ -57,6 +62,25 @@ class AnalyzerDownloaderTest : WordSpec({ workingTree.getNested().shouldBeEmpty() } + "not recursively clone a Git repository if submoduleFetchStrategy is DISABLED" { + val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val revision = "fcea94bab5835172e826afddb9f6427274c983b9" + + val outputDir = downloader.downloadRepository( + repositoryUrl, revision, submoduleFetchStrategy = SubmoduleFetchStrategy.DISABLED + ) + + outputDir shouldContainFile "LICENSE" + outputDir shouldContainFile "README.md" + + outputDir.resolve("commons-text") should beEmptyDirectory() + outputDir.resolve("test-data-npm") should beEmptyDirectory() + + val workingTree = VersionControlSystem.forDirectory(outputDir) + workingTree.shouldNotBeNull() + workingTree.getNested().shouldBeEmpty() + } + "recursively clone a Git repository if recursiveCheckout is true" { val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" val revision = "fcea94bab5835172e826afddb9f6427274c983b9" @@ -95,6 +119,78 @@ class AnalyzerDownloaderTest : WordSpec({ ) } + "clone only the top level of a Git repository if submoduleFetchStrategy is TOP_LEVEL_ONLY" { + // Intentionally using https://www.github.com instead of https://github.com as a workaround + // for bug https://github.com/oss-review-toolkit/ort/issues/9795 + val repositoryUrl = "https://www.github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val revision = "fcea94bab5835172e826afddb9f6427274c983b9" + + val outputDir = downloader.downloadRepository( + repositoryUrl, revision, submoduleFetchStrategy = SubmoduleFetchStrategy.TOP_LEVEL_ONLY + ) + + outputDir shouldContainFile "LICENSE" + outputDir shouldContainFile "README.md" + + outputDir.resolve("commons-text") shouldNot beEmptyDirectory() + outputDir.resolve("test-data-npm") shouldNot beEmptyDirectory() + + val workingTree = VersionControlSystem.forDirectory(outputDir) + workingTree.shouldNotBeNull() + workingTree.getNested() shouldContainExactly mapOf( + "commons-text" to VcsInfo( + type = VcsType.GIT, + url = "https://github.com/apache/commons-text.git", + revision = "7643b12421100d29fd2b78053e77bcb04a251b2e" + ), + "test-data-npm" to VcsInfo( + type = VcsType.GIT, + url = "https://github.com/oss-review-toolkit/ort-test-data-npm.git", + revision = "ad0367b7b9920144a47b8d30cc0c84cea102b821" + ) + ) + } + + "fully recursively clone a Git repository if submoduleFetchStrategy is FULLY_RECURSIVE" { + val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val revision = "fcea94bab5835172e826afddb9f6427274c983b9" + + val outputDir = downloader.downloadRepository( + repositoryUrl, revision, submoduleFetchStrategy = SubmoduleFetchStrategy.FULLY_RECURSIVE + ) + + outputDir shouldContainFile "LICENSE" + outputDir shouldContainFile "README.md" + + outputDir.resolve("commons-text") shouldNot beEmptyDirectory() + outputDir.resolve("test-data-npm") shouldNot beEmptyDirectory() + + val workingTree = VersionControlSystem.forDirectory(outputDir) + workingTree.shouldNotBeNull() + workingTree.getNested() shouldContainExactly mapOf( + "commons-text" to VcsInfo( + type = VcsType.GIT, + url = "https://github.com/apache/commons-text.git", + revision = "7643b12421100d29fd2b78053e77bcb04a251b2e" + ), + "test-data-npm" to VcsInfo( + type = VcsType.GIT, + url = "https://github.com/oss-review-toolkit/ort-test-data-npm.git", + revision = "ad0367b7b9920144a47b8d30cc0c84cea102b821" + ), + "test-data-npm/isarray" to VcsInfo( + type = VcsType.GIT, + url = "https://github.com/juliangruber/isarray.git", + revision = "63ea4ca0a0d6b0574d6a470ebd26880c3026db4a" + ), + "test-data-npm/long.js" to VcsInfo( + type = VcsType.GIT, + url = "https://github.com/dcodeIO/long.js.git", + revision = "941c5c62471168b5d18153755c2a7b38d2560e58" + ) + ) + } + "clone a sub-directory of a Git repository" { val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-scanner.git" val revision = "63b81fda7961c7426672469caaf4fb350a9d4ee0" @@ -126,4 +222,79 @@ class AnalyzerDownloaderTest : WordSpec({ } } } + + "buildCustomVcsPluginConfigurations" should { + "return an empty map if the submoduleFetchStrategy is null" { + val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val submoduleFetchStrategy = null + + val configurations = downloader.buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy) + + configurations.shouldBeEmpty() + } + + "return an empty map if the submoduleFetchStrategy is DISABLED" { + val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val submoduleFetchStrategy = SubmoduleFetchStrategy.DISABLED + + val configurations = downloader.buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy) + + configurations.shouldBeEmpty() + } + + "return custom configuration for git if the submoduleFetchStrategy is TOP_LEVEL_ONLY" { + val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val submoduleFetchStrategy = SubmoduleFetchStrategy.TOP_LEVEL_ONLY + + val configurations = downloader.buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy) + + configurations.shouldNotBeEmpty() + + configurations shouldContainExactly mapOf( + VcsType.GIT.toString() to PluginConfig( + mapOf("updateNestedSubmodules" to false.toString()) + ) + ) + } + + "return an empty map if the submoduleFetchStrategy is FULLY_RECURSIVE" { + val repositoryUrl = "https://github.com/oss-review-toolkit/ort-test-data-git-submodules.git" + val submoduleFetchStrategy = SubmoduleFetchStrategy.FULLY_RECURSIVE + + val configurations = downloader.buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy) + + configurations.shouldBeEmpty() + } + + "throw an IllegalArgumentException if the submoduleFetchStrategy is TOP_LEVEL_ONLY " + + "and the VCS type is not GIT" { + val repositoryUrl = "https://example.com" + val submoduleFetchStrategy = SubmoduleFetchStrategy.TOP_LEVEL_ONLY + + shouldThrow { + downloader.buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy) + } + } + } + + "evaluateCombinedRecursiveCheckoutParameter" should { + val testData = listOf( + Triple(true, null, true), + Triple(false, null, false), + Triple(true, SubmoduleFetchStrategy.DISABLED, false), + Triple(false, SubmoduleFetchStrategy.DISABLED, false), + Triple(true, SubmoduleFetchStrategy.TOP_LEVEL_ONLY, true), + Triple(false, SubmoduleFetchStrategy.TOP_LEVEL_ONLY, true), + Triple(true, SubmoduleFetchStrategy.FULLY_RECURSIVE, true), + Triple(false, SubmoduleFetchStrategy.FULLY_RECURSIVE, true) + ) + + testData.forEach { (recursiveCheckout, submoduleFetchStrategy, expected) -> + "return $expected for recursiveCheckout $recursiveCheckout " + + "and submoduleFetchStrategy $submoduleFetchStrategy" { + val result = downloader.evaluateRecursiveCheckoutParameter(recursiveCheckout, submoduleFetchStrategy) + result shouldBe expected + } + } + } })