Skip to content

Commit

Permalink
feat(analyzer): Introduce new JobConfiguration parameter for submodules
Browse files Browse the repository at this point in the history
Add a parameter to the Analyzer JobConfiguration that allows specifying
how repository submodules should be fetched. This includes the options
to fetch submodules recursively or to fetch only the top level of
submodules, providing greater flexibility in submodule handling.

Signed-off-by: Wolfgang Klenk <[email protected]>
  • Loading branch information
wkl3nk authored and oheger-bosch committed Jan 21, 2025
1 parent 508e497 commit c901b83
Show file tree
Hide file tree
Showing 7 changed files with 305 additions and 5 deletions.
16 changes: 16 additions & 0 deletions api/v1/mapping/src/commonMain/kotlin/Mappings.kt
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ import org.eclipse.apoapsis.ortserver.api.v1.model.Severity as ApiSeverity
import org.eclipse.apoapsis.ortserver.api.v1.model.SortDirection as ApiSortDirection
import org.eclipse.apoapsis.ortserver.api.v1.model.SortProperty as ApiSortProperty
import org.eclipse.apoapsis.ortserver.api.v1.model.SourceCodeOrigin as ApiSourceCodeOrigin
import org.eclipse.apoapsis.ortserver.api.v1.model.SubmoduleFetchStrategy as ApiSubmoduleFetchStrategy
import org.eclipse.apoapsis.ortserver.api.v1.model.User as ApiUser
import org.eclipse.apoapsis.ortserver.api.v1.model.VcsInfo as ApiVcsInfo
import org.eclipse.apoapsis.ortserver.api.v1.model.Vulnerability as ApiVulnerability
Expand Down Expand Up @@ -122,6 +123,7 @@ import org.eclipse.apoapsis.ortserver.model.ScannerJobConfiguration
import org.eclipse.apoapsis.ortserver.model.Secret
import org.eclipse.apoapsis.ortserver.model.Severity
import org.eclipse.apoapsis.ortserver.model.SourceCodeOrigin
import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy
import org.eclipse.apoapsis.ortserver.model.User
import org.eclipse.apoapsis.ortserver.model.VulnerabilityRating
import org.eclipse.apoapsis.ortserver.model.VulnerabilityWithAccumulatedData
Expand Down Expand Up @@ -195,6 +197,7 @@ fun AnalyzerJobConfiguration.mapToApi() =
enabledPackageManagers,
environmentConfig?.mapToApi(),
recursiveCheckout,
submoduleFetchStrategy?.mapToApi(),
packageCurationProviders.map { it.mapToApi() },
packageManagerOptions?.mapValues { it.value.mapToApi() },
repositoryConfigPath,
Expand All @@ -208,6 +211,7 @@ fun ApiAnalyzerJobConfiguration.mapToModel() =
enabledPackageManagers,
environmentConfig?.mapToModel(),
recursiveCheckout,
submoduleFetchStrategy?.mapToModel(),
packageCurationProviders?.map { it.mapToModel() }.orEmpty(),
packageManagerOptions?.mapValues { it.value.mapToModel() },
repositoryConfigPath,
Expand Down Expand Up @@ -775,3 +779,15 @@ fun VulnerabilityWithAccumulatedData.mapToApi() = ApiProductVulnerability(
ortRunIds = ortRunIds,
repositoriesCount = repositoriesCount
)

fun SubmoduleFetchStrategy.mapToApi() = when (this) {
SubmoduleFetchStrategy.DISABLED -> ApiSubmoduleFetchStrategy.DISABLED
SubmoduleFetchStrategy.TOP_LEVEL_ONLY -> ApiSubmoduleFetchStrategy.TOP_LEVEL_ONLY
SubmoduleFetchStrategy.FULLY_RECURSIVE -> ApiSubmoduleFetchStrategy.FULLY_RECURSIVE
}

fun ApiSubmoduleFetchStrategy.mapToModel() = when (this) {
ApiSubmoduleFetchStrategy.DISABLED -> SubmoduleFetchStrategy.DISABLED
ApiSubmoduleFetchStrategy.TOP_LEVEL_ONLY -> SubmoduleFetchStrategy.TOP_LEVEL_ONLY
ApiSubmoduleFetchStrategy.FULLY_RECURSIVE -> SubmoduleFetchStrategy.FULLY_RECURSIVE
}
29 changes: 29 additions & 0 deletions api/v1/model/src/commonMain/kotlin/JobConfigurations.kt
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,17 @@ data class AnalyzerJobConfiguration(
/**
* A flag indicating whether the submodules of the repository should be downloaded during the download process.
* If set to `true`, submodules will be downloaded; if `false`, they will be ignored.
*
* Note: This attribute is deprecated and will be removed in a future release. Use [submoduleFetchStrategy] instead.
*
*/
val recursiveCheckout: Boolean = true,

/**
* The strategy to use for fetching submodules.
*/
val submoduleFetchStrategy: SubmoduleFetchStrategy? = null,

/**
* The list of package curation providers to use.
*/
Expand Down Expand Up @@ -372,3 +380,24 @@ data class NotifierJobConfiguration(
*/
val jira: JiraNotificationConfiguration? = null
)

@Serializable
/**
* The strategy to use for fetching submodules.
*/
enum class SubmoduleFetchStrategy {
/**
* Don't fetch submodules at all.
*/
DISABLED,

/**
* Only fetch the top level of submodules.
*/
TOP_LEVEL_ONLY,

/**
* Fetch all nested submodules recursively.
*/
FULLY_RECURSIVE
}
2 changes: 2 additions & 0 deletions core/src/main/kotlin/apiDocs/RepositoriesDocs.kt
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import org.eclipse.apoapsis.ortserver.api.v1.model.Secret
import org.eclipse.apoapsis.ortserver.api.v1.model.SortDirection
import org.eclipse.apoapsis.ortserver.api.v1.model.SortProperty
import org.eclipse.apoapsis.ortserver.api.v1.model.SourceCodeOrigin
import org.eclipse.apoapsis.ortserver.api.v1.model.SubmoduleFetchStrategy.FULLY_RECURSIVE
import org.eclipse.apoapsis.ortserver.api.v1.model.UpdateRepository
import org.eclipse.apoapsis.ortserver.api.v1.model.UpdateSecret
import org.eclipse.apoapsis.ortserver.api.v1.model.Username
Expand Down Expand Up @@ -111,6 +112,7 @@ internal val fullJobConfigurations = JobConfigurations(
)
),
recursiveCheckout = true,
submoduleFetchStrategy = FULLY_RECURSIVE,
skipExcluded = true
),
advisor = AdvisorJobConfiguration(
Expand Down
30 changes: 30 additions & 0 deletions model/src/commonMain/kotlin/JobConfigurations.kt
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,18 @@ data class AnalyzerJobConfiguration(
/**
* A flag indicating whether the submodules of the repository should be downloaded during the download process.
* If set to `true`, submodules will be downloaded; if `false`, they will be ignored.
*
* Note: This attribute is deprecated and will be removed in a future release. Use [submoduleFetchStrategy] instead.
*/
val recursiveCheckout: Boolean = true,

/**
* The strategy to use for fetching submodules.
*
* Note: Submodule fetch strategy [SubmoduleFetchStrategy.TOP_LEVEL_ONLY] is only supported for Git repositories.
*/
val submoduleFetchStrategy: SubmoduleFetchStrategy? = null,

/**
* The list of package curation providers to use.
*/
Expand Down Expand Up @@ -387,3 +396,24 @@ data class NotifierJobConfiguration(
*/
val jira: JiraNotificationConfiguration? = null
)

@Serializable
/**
* The strategy to use for fetching submodules.
*/
enum class SubmoduleFetchStrategy {
/**
* Don't fetch submodules at all.
*/
DISABLED,

/**
* Only fetch the top level of submodules.
*/
TOP_LEVEL_ONLY,

/**
* Fetch all nested submodules recursively.
*/
FULLY_RECURSIVE
}
59 changes: 55 additions & 4 deletions workers/analyzer/src/main/kotlin/analyzer/AnalyzerDownloader.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,13 @@ package org.eclipse.apoapsis.ortserver.workers.analyzer

import java.io.File

import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy

import org.ossreviewtoolkit.downloader.VcsHost
import org.ossreviewtoolkit.downloader.VersionControlSystem
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.plugins.api.PluginConfig
import org.ossreviewtoolkit.utils.ort.createOrtTempDir

import org.slf4j.LoggerFactory
Expand All @@ -34,22 +39,68 @@ class AnalyzerDownloader {
repositoryUrl: String,
revision: String,
path: String = "",
recursiveCheckout: Boolean = true
recursiveCheckout: Boolean = true, // Deprecated: Will be removed in a future release
submoduleFetchStrategy: SubmoduleFetchStrategy? = null
): File {
logger.info("Downloading repository '$repositoryUrl' revision '$revision'.")

val outputDir = createOrtTempDir("analyzer-worker")

val vcs = VersionControlSystem.forUrl(repositoryUrl)
val config = buildCustomVcsPluginConfigMap(repositoryUrl, submoduleFetchStrategy)
val vcs = VersionControlSystem.forUrl(repositoryUrl, config)
requireNotNull(vcs) { "Could not determine the VCS for URL '$repositoryUrl'." }

val vcsInfo = VcsInfo(vcs.type, repositoryUrl, revision, path)
val vcsInfo = VcsInfo(
type = vcs.type,
url = repositoryUrl,
revision = revision,
path = path
)

// The [submoduleFetchStrategy] parameter takes precedence over the deprecated [recursiveCheckout] parameter.
val combinedRecursiveCheckout = evaluateRecursiveCheckoutParameter(recursiveCheckout, submoduleFetchStrategy)

val workingTree = vcs.initWorkingTree(outputDir, vcsInfo)
vcs.updateWorkingTree(workingTree, revision, recursive = recursiveCheckout).getOrThrow()
vcs.updateWorkingTree(workingTree, revision, recursive = combinedRecursiveCheckout).getOrThrow()

logger.info("Finished downloading '$repositoryUrl' revision '$revision'.")

return outputDir
}

/**
* Build custom [PluginConfig] for Git VCS if the [submoduleFetchStrategy] is
* [SubmoduleFetchStrategy.TOP_LEVEL_ONLY].
*/
internal fun buildCustomVcsPluginConfigMap(
repositoryUrl: String, submoduleFetchStrategy: SubmoduleFetchStrategy?
) =
if (submoduleFetchStrategy == SubmoduleFetchStrategy.TOP_LEVEL_ONLY) {
val vcsType = VcsHost.parseUrl(repositoryUrl).type
require(vcsType == VcsType.GIT) {
"Submodule fetch strategy TOP_LEVEL_ONLY is only supported for Git repositories, " +
"but got VCS type '$vcsType'."
}
mapOf(
VcsType.GIT.toString() to PluginConfig(
options = mapOf("updateNestedSubmodules" to false.toString())
)
)
} else {
emptyMap()
}

/**
* Evaluate the [recursiveCheckout] and the [submoduleFetchStrategy] parameter to determine if the working tree
* should be checked out recursively. The [submoduleFetchStrategy] parameter takes precedence over the
* deprecated [recursiveCheckout] parameter.
*/
internal fun evaluateRecursiveCheckoutParameter(
recursiveCheckout: Boolean, submoduleFetchStrategy: SubmoduleFetchStrategy?
) =
if (submoduleFetchStrategy != null) {
submoduleFetchStrategy != SubmoduleFetchStrategy.DISABLED
} else {
recursiveCheckout
}
}
3 changes: 2 additions & 1 deletion workers/analyzer/src/main/kotlin/analyzer/AnalyzerWorker.kt
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ internal class AnalyzerWorker(
repository.url,
ortRun.revision,
ortRun.path.orEmpty(),
job.configuration.recursiveCheckout
job.configuration.recursiveCheckout,
job.configuration.submoduleFetchStrategy
)

val resolvedEnvConfig = environmentService.setUpEnvironment(
Expand Down
Loading

0 comments on commit c901b83

Please sign in to comment.