Skip to content

Commit

Permalink
feat(bulk-model-sync-gradle): add configuration options to exclude mo…
Browse files Browse the repository at this point in the history
…dules
  • Loading branch information
mhuster23 committed Aug 26, 2024
1 parent e4bfe2f commit 5db8de9
Show file tree
Hide file tree
Showing 11 changed files with 303 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ class ModelSyncGradlePlugin : Plugin<Project> {
it.outputDir.set(jsonDir)
it.includedModules.set(syncDirection.includedModules)
it.includedModulePrefixes.set(syncDirection.includedModulePrefixes)
it.excludedModules.set(syncDirection.excludedModules)
it.excludedModulePrefixes.set(syncDirection.excludedModulePrefixes)
it.requestTimeoutSeconds.set(serverSource.requestTimeoutSeconds)
}
return exportFromModelServer
Expand Down Expand Up @@ -167,6 +169,8 @@ class ModelSyncGradlePlugin : Plugin<Project> {
it.branchName.set(serverTarget.branchName)
it.includedModules.set(syncDirection.includedModules)
it.includedModulePrefixes.set(syncDirection.includedModulePrefixes)
it.excludedModules.set(syncDirection.excludedModules)
it.excludedModulePrefixes.set(syncDirection.excludedModulePrefixes)
it.continueOnError.set(syncDirection.continueOnError)
it.requestTimeoutSeconds.set(serverTarget.requestTimeoutSeconds)
it.metaProperties.set(serverTarget.metaProperties)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ internal fun buildMpsRunConfigurationForLocalSources(
",",
)
}",
"-Dmodelix.mps.model.sync.bulk.output.modules.excluded=${syncDirection.excludedModules.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.output.modules.prefixes.excluded=${
syncDirection.excludedModulePrefixes.joinToString(
",",
)
}",
"-Dmodelix.mps.model.sync.bulk.repo.path=${localSource.repositoryDir?.absolutePath}",
"-Xmx${localSource.mpsHeapSize}",
localSource.mpsDebugPort?.let { "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=$it" },
Expand Down Expand Up @@ -68,10 +74,19 @@ internal fun buildMpsRunConfigurationForLocalTarget(
plugins = createPluginConfig(localTarget.mpsPlugins),
jvmArgs = buildList {
add("-Dmodelix.mps.model.sync.bulk.input.path=${jsonDir.absolutePath}")
val includeModuleValue = syncDirection.includedModules.joinToString(",")
add("-Dmodelix.mps.model.sync.bulk.input.modules=$includeModuleValue")
val includeModulePrefixesValue = syncDirection.includedModulePrefixes.joinToString(",")
add("-Dmodelix.mps.model.sync.bulk.input.modules.prefixes=$includeModulePrefixesValue")

val includedModulesValue = syncDirection.includedModules.joinToString(",")
add("-Dmodelix.mps.model.sync.bulk.input.modules=$includedModulesValue")

val includedModulePrefixesValue = syncDirection.includedModulePrefixes.joinToString(",")
add("-Dmodelix.mps.model.sync.bulk.input.modules.prefixes=$includedModulePrefixesValue")

val excludedModulesValue = syncDirection.excludedModules.joinToString(",")
add("-Dmodelix.mps.model.sync.bulk.input.modules.excluded=$excludedModulesValue")

val excludedModulePrefixesValue = syncDirection.excludedModulePrefixes.joinToString(",")
add("-Dmodelix.mps.model.sync.bulk.input.modules.prefixes.excluded=$excludedModulePrefixesValue")

add("-Dmodelix.mps.model.sync.bulk.repo.path=${repositoryDir.absolutePath}")
add("-Dmodelix.mps.model.sync.bulk.input.continueOnError=${syncDirection.continueOnError}")
add("-Xmx${localTarget.mpsHeapSize}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ data class SyncDirection(
internal var target: SyncEndpoint? = null,
internal val includedModules: Set<String> = mutableSetOf(),
internal val includedModulePrefixes: Set<String> = mutableSetOf(),
internal val excludedModules: Set<String> = mutableSetOf(),
internal val excludedModulePrefixes: Set<String> = mutableSetOf(),
internal var continueOnError: Boolean = false,
) {
fun fromModelServer(action: Action<ServerSource>) {
Expand Down Expand Up @@ -76,6 +78,14 @@ data class SyncDirection(
(includedModulePrefixes as MutableSet).add(prefix)
}

fun excludeModule(module: String) {
(excludedModules as MutableSet).add(module)
}

fun excludeModulesByPrefix(prefix: String) {
(excludedModulePrefixes as MutableSet).add(prefix)
}

@Deprecated("Registering languages is not necessary. This call can be safely removed.", ReplaceWith(""))
@DeprecationInfo(since = "2024-01-08")
fun registerLanguage(language: ILanguage) {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ abstract class ExportFromModelServer : DefaultTask() {
@get:Input
abstract val includedModulePrefixes: SetProperty<String>

@get:Input
abstract val excludedModules: SetProperty<String>

@get:Input
abstract val excludedModulePrefixes: SetProperty<String>

@get:Input
abstract val requestTimeoutSeconds: Property<Int>

Expand Down Expand Up @@ -101,7 +107,13 @@ abstract class ExportFromModelServer : DefaultTask() {
return root.allChildren.filter {
val isModule = it.concept?.getUID() == BuiltinLanguages.MPSRepositoryConcepts.Module.getUID()
val moduleName = it.getPropertyValue(nameRole) ?: return@filter false
val isIncluded = isModuleIncluded(moduleName, includedModules.get(), includedModulePrefixes.get())
val isIncluded = isModuleIncluded(
moduleName,
includedModules.get(),
includedModulePrefixes.get(),
excludedModules.get(),
excludedModulePrefixes.get(),
)

isModule && isIncluded
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ abstract class ImportIntoModelServer : DefaultTask() {
@get:Input
abstract val includedModulePrefixes: SetProperty<String>

@get:Input
abstract val excludedModules: SetProperty<String>

@get:Input
abstract val excludedModulePrefixes: SetProperty<String>

@get:Input
abstract val continueOnError: Property<Boolean>

Expand All @@ -77,7 +83,13 @@ abstract class ImportIntoModelServer : DefaultTask() {

val branchRef = ModelFacade.createBranchReference(repoId, branchName.get())
val files = inputDir.listFiles()?.filter {
it.extension == "json" && isModuleIncluded(it.nameWithoutExtension, includedModules.get(), includedModulePrefixes.get())
it.extension == "json" && isModuleIncluded(
it.nameWithoutExtension,
includedModules.get(),
includedModulePrefixes.get(),
excludedModules.get(),
excludedModulePrefixes.get(),
)
}
if (files.isNullOrEmpty()) error("no json files found for included modules")

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package org.modelix.model.sync.bulk.gradle

import io.kotest.matchers.collections.shouldContainAll
import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder
import org.modelix.model.sync.bulk.gradle.config.LocalSource
import org.modelix.model.sync.bulk.gradle.config.LocalTarget
import org.modelix.model.sync.bulk.gradle.config.ServerSource
import org.modelix.model.sync.bulk.gradle.config.ServerTarget
import org.modelix.model.sync.bulk.gradle.config.SyncDirection
import java.io.File
import kotlin.test.Test
Expand All @@ -29,6 +32,8 @@ class MpsRunnerConfigurationTest {
"-Dmodelix.mps.model.sync.bulk.input.path=/jsonDir",
"-Dmodelix.mps.model.sync.bulk.input.modules=",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes=",
"-Dmodelix.mps.model.sync.bulk.input.modules.excluded=",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes.excluded=",
"-Dmodelix.mps.model.sync.bulk.repo.path=/repositoryDir",
"-Dmodelix.mps.model.sync.bulk.input.continueOnError=false",
"-Xmx2g",
Expand All @@ -55,6 +60,8 @@ class MpsRunnerConfigurationTest {
"-Dmodelix.mps.model.sync.bulk.input.path=/jsonDir",
"-Dmodelix.mps.model.sync.bulk.input.modules=",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes=",
"-Dmodelix.mps.model.sync.bulk.input.modules.excluded=",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes.excluded=",
"-Dmodelix.mps.model.sync.bulk.repo.path=/repositoryDir",
"-Dmodelix.mps.model.sync.bulk.input.continueOnError=false",
"-Xmx2g",
Expand Down Expand Up @@ -85,6 +92,8 @@ class MpsRunnerConfigurationTest {
"-Dmodelix.mps.model.sync.bulk.input.path=/jsonDir",
"-Dmodelix.mps.model.sync.bulk.input.modules=",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes=",
"-Dmodelix.mps.model.sync.bulk.input.modules.excluded=",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes.excluded=",
"-Dmodelix.mps.model.sync.bulk.repo.path=/repositoryDir",
"-Dmodelix.mps.model.sync.bulk.input.continueOnError=false",
"-Xmx2g",
Expand All @@ -96,4 +105,77 @@ class MpsRunnerConfigurationTest {

jvmArgs shouldContainExactlyInAnyOrder expectedJvmArgs
}

@Test
fun `module inclusion and exclusion (local to server)`() {
val serverTarget = ServerTarget(
url = "someUrl",
repositoryId = "someRepositoryId",
branchName = "someBranch",
)

val localSource = LocalSource(repositoryDir = File("/someDir"))

val includedModules = setOf("includedModuleA", "includedModuleB")
val includedModulePrefixes = setOf("includedPrefixA", "includedPrefixB")
val excludedModules = setOf("excludedModuleA", "excludedModuleB")
val excludedModulePrefixes = setOf("excludedPrefixA", "excludedPrefixB")

val syncDirection = SyncDirection(
name = "syncDirection",
source = localSource,
target = serverTarget,
includedModules = includedModules,
includedModulePrefixes = includedModulePrefixes,
excludedModules = excludedModules,
excludedModulePrefixes = excludedModulePrefixes,
)

val expectedJvmArgs = setOf(
"-Dmodelix.mps.model.sync.bulk.output.modules=${includedModules.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.output.modules.prefixes=${includedModulePrefixes.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.output.modules.excluded=${excludedModules.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.output.modules.prefixes.excluded=${excludedModulePrefixes.joinToString(",")}",
)

val config = buildMpsRunConfigurationForLocalSources(syncDirection, classPathElements, jsonDir)

config.jvmArgs shouldContainAll expectedJvmArgs
}

@Test
fun `module inclusion and exclusion (server to local)`() {
val serverSource = ServerSource(
url = "someUrl",
repositoryId = "someRepositoryId",
branchName = "someBranch",
)
val localTarget = LocalTarget(repositoryDir = File("/someDir"))

val includedModules = setOf("includedModuleA", "includedModuleB")
val includedModulePrefixes = setOf("includedPrefixA", "includedPrefixB")
val excludedModules = setOf("excludedModuleA", "excludedModuleB")
val excludedModulePrefixes = setOf("excludedPrefixA", "excludedPrefixB")

val syncDirection = SyncDirection(
name = "syncDirection",
source = serverSource,
target = localTarget,
includedModules = includedModules,
includedModulePrefixes = includedModulePrefixes,
excludedModules = excludedModules,
excludedModulePrefixes = excludedModulePrefixes,
)

val expectedJvmArgs = setOf(
"-Dmodelix.mps.model.sync.bulk.input.modules=${includedModules.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes=${includedModulePrefixes.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.input.modules.excluded=${excludedModules.joinToString(",")}",
"-Dmodelix.mps.model.sync.bulk.input.modules.prefixes.excluded=${excludedModulePrefixes.joinToString(",")}",
)

val config = buildMpsRunConfigurationForLocalTarget(syncDirection, classPathElements, jsonDir)

config.jvmArgs shouldContainAll expectedJvmArgs
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,28 @@ import kotlin.math.max
/**
* Checks if a module is included in the sync.
*
* @param moduleName name of the module to be checked
* @param includedModules collection of included module names
* @param includedPrefixes collection of included module name prefixes
* Modules can be included via name ([includedModules]) or prefix ([includedPrefixes])
* Exclusion works analogously.
*
* Returns true iff the module is included and not excluded.
*/
fun isModuleIncluded(
moduleName: String,
includedModules: Collection<String>,
includedPrefixes: Collection<String>,
excludedModules: Collection<String> = emptySet(),
excludedPrefixes: Collection<String> = emptySet(),
): Boolean {
val includedDirectly = includedModules.contains(moduleName)
val includedByPrefix = includedPrefixes.any { prefix -> moduleName.startsWith(prefix) }
val included = includedDirectly || includedByPrefix
if (!included) return false

val excludedDirectly = excludedModules.contains(moduleName)
val excludedByPrefix = excludedPrefixes.any { prefix -> moduleName.startsWith(prefix) }
val excluded = excludedDirectly || excludedByPrefix

return includedDirectly || includedByPrefix
return !excluded
}

fun mergeModelData(models: Collection<ModelData>): ModelData {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2024.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.modelix.model.sync.bulk

import kotlin.js.JsName
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue

class ModuleInclusionTest {

private val allModules = setOf(
"a", "b", "c",
"prefix.a", "prefix.b", "prefix.c",
"prefix2.a", "prefix2.b", "prefix2.c",
)

@Test
@JsName("no_modules_are_included_by_default")
fun `no modules are included by default`() {
val result = allModules.filter { isModuleIncluded(it, emptySet(), emptySet()) }
assertTrue { result.isEmpty() }
}

@Test
@JsName("modules_are_included_correctly")
fun `modules are included correctly`() {
val result = allModules.filter {
isModuleIncluded(
moduleName = it,
includedModules = setOf("a", "b"),
includedPrefixes = setOf("prefix2"),
)
}.toSet()

val expected = setOf("a", "b", "prefix2.a", "prefix2.b", "prefix2.c")
assertEquals(expected, result)
}

@Test
@JsName("modules_are_excluded_directly")
fun `modules are excluded correctly`() {
val result = allModules.filter {
isModuleIncluded(
moduleName = it,
includedModules = setOf("a", "b"),
includedPrefixes = setOf("prefix"),
excludedModules = setOf("prefix.b"),
excludedPrefixes = setOf("prefix2"),
)
}.toSet()

val expected = setOf("a", "b", "prefix.a", "prefix.c")
assertEquals(expected, result)
}

@Test
@JsName("exclusion_has_priority_over_inclusion")
fun `exclusion has priority over inclusion`() {
val result = allModules.filter {
isModuleIncluded(
it,
includedModules = setOf("a", "b"),
includedPrefixes = setOf("prefix"),
excludedModules = setOf("a", "b"),
excludedPrefixes = setOf("prefix"),
)
}
assertTrue { result.isEmpty() }
}
}
Loading

0 comments on commit 5db8de9

Please sign in to comment.