From b6c3aaacfdbc947874a1264ab6c96056b4e2db2c Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 5 Feb 2025 09:57:13 +0100 Subject: [PATCH 01/37] feat(mps-sync-plugin): re-implementation of the sync plugin for MPS --- .../RecreateProjectFromModelServerTest.kt | 269 ---------- .../org/modelix/model/sync/bulk/IModelMask.kt | 37 ++ .../modelix/model/sync/bulk/ModelImporter.kt | 26 +- .../model/sync/bulk/ModelSynchronizer.kt | 63 +-- .../sync/bulk/NodeAssociationToModelServer.kt | 2 +- .../model/sync/bulk/ModelSynchronizerTest.kt | 15 +- .../model/sync/bulk/InvalidatingVisitor.kt | 10 +- .../model/sync/bulk/InvalidationTree.kt | 111 +++- .../model/sync/bulk/InvalidationTreeTest.kt | 10 +- .../mps/model/sync/bulk/CompositeFilter.kt | 2 +- .../model/sync/bulk/IncludedModulesFilter.kt | 2 +- .../model/sync/bulk/MPSBulkSynchronizer.kt | 2 +- ...ectSyncFilter.kt => MPSProjectSyncMask.kt} | 36 +- commitlint.config.js | 1 + gradle/libs.versions.toml | 1 + .../org/modelix/model/api/BuiltinLanguages.kt | 14 + .../org/modelix/model/api/INodeReference.kt | 2 + .../org/modelix/model/api/IWritableNode.kt | 12 + .../org/modelix/model/api/PNodeReference.kt | 27 +- .../modelix/model/client2/IModelClientV2.kt | 1 + .../modelix/model/client2/ModelClientV2.kt | 49 +- .../org/modelix/model/client2/LazyLoading.kt | 2 + .../model/persistent/OperationSerializer.kt | 2 +- .../org/modelix/model/mpsadapters/MPSArea.kt | 13 +- .../mpsadapters/MPSDevKitDependencyAsNode.kt | 8 + .../mpsadapters/MPSGenericNodeAdapter.kt | 49 +- .../mpsadapters/MPSJavaModuleFacetAsNode.kt | 4 + .../model/mpsadapters/MPSModelAsNode.kt | 46 +- .../model/mpsadapters/MPSModelImportAsNode.kt | 17 +- .../model/mpsadapters/MPSModuleAsNode.kt | 213 ++++++-- .../mpsadapters/MPSModuleDependencyAsNode.kt | 8 + .../mpsadapters/MPSModuleReferenceAsNode.kt | 20 +- .../model/mpsadapters/MPSProjectAsNode.kt | 20 + .../mpsadapters/MPSProjectModuleAsNode.kt | 8 + .../model/mpsadapters/MPSReferences.kt | 1 + .../model/mpsadapters/MPSRepositoryAsNode.kt | 48 ++ .../MPSSingleLanguageDependencyAsNode.kt | 8 + .../model/mpsadapters/MPSWritableNode.kt | 63 ++- .../ModelPersistenceWithFixedId.kt | 8 +- .../model/mpsadapters/SolutionProducer.kt | 36 +- mps-sync-plugin3/build.gradle.kts | 93 ++++ .../modelix/mps/sync3/IModelSyncService.kt | 60 +++ .../mps/sync3/MPSInvalidatingListener.kt | 218 ++++++++ .../mps/sync3/ModelSyncForMPSProject.kt | 499 ++++++++++++++++++ .../mps/sync3/ModelSyncStartupActivity.kt | 11 + .../org/modelix/mps/sync3/ValidatingJob.kt | 31 ++ .../src/main/resources/META-INF/plugin.xml | 31 ++ .../main/resources/META-INF/pluginIcon.svg | 66 +++ .../org/modelix/mps/sync3/MPSTestBase.kt | 124 +++++ .../org/modelix/mps/sync3/ProjectSnapshot.kt | 99 ++++ .../org/modelix/mps/sync3/ProjectSyncTest.kt | 361 +++++++++++++ .../kotlin/org/modelix/mps/sync3}/XMLUtils.kt | 2 +- .../src/test/resources/logback-test.xml | 21 + mps-sync-plugin3/testdata/.gitignore | 5 + .../testdata/change1}/.mps/.gitignore | 0 .../testdata/change1}/.mps/migration.xml | 0 .../testdata/change1/.mps/modules.xml | 12 + .../devkits/NewDevkit/NewDevkit.devkit | 8 + .../languages/NewLanguage/NewLanguage.mpl | 0 ...Language.generator.templates@generator.mps | 99 ++++ .../models/NewLanguage.behavior.mps | 0 .../models/NewLanguage.constraints.mps | 0 .../NewLanguage/models/NewLanguage.editor.mps | 0 .../models/NewLanguage.structure.mps | 0 .../models/NewLanguage.typesystem.mps | 0 .../languages/NewLanguage2/NewLanguage2.mpl | 109 ++++ ...anguage2.generator.templates@generator.mps | 23 + .../models/NewLanguage2.behavior.mps | 11 + .../models/NewLanguage2.constraints.mps | 18 + .../models/NewLanguage2.editor.mps | 11 + .../models/NewLanguage2.structure.mps | 31 ++ .../models/NewLanguage2.typesystem.mps | 10 + .../NewRuntimeSolution/NewRuntimeSolution.msd | 41 ++ .../models/NewRuntimeSolution.modelB.mps | 81 +++ .../models/NewRuntimeSolution.plugin.mps | 107 ++++ .../solutions/NewSolution/NewSolution.msd | 40 ++ .../models/NewSolution.a_model.mps | 133 +++++ .../models/NewSolution.b_model.mps | 0 .../testdata/initial/.mps/.gitignore | 3 + .../testdata/initial/.mps/migration.xml | 6 + .../testdata/initial}/.mps/modules.xml | 1 + .../devkits/NewDevkit/NewDevkit.devkit | 1 + .../languages/NewLanguage/NewLanguage.mpl | 143 +++++ ...Language.generator.templates@generator.mps | 0 ...nguage.generator01.templates@generator.mps | 23 + .../models/NewLanguage.behavior.mps | 11 + .../models/NewLanguage.constraints.mps | 18 + .../NewLanguage/models/NewLanguage.editor.mps | 11 + .../models/NewLanguage.structure.mps | 68 +++ .../models/NewLanguage.typesystem.mps | 10 + .../NewRuntimeSolution/NewRuntimeSolution.msd | 0 .../models/NewRuntimeSolution.plugin.mps | 0 .../solutions/NewSolution/NewSolution.msd | 5 + .../models/NewSolution.a_model.mps | 0 .../models/NewSolution.b_model.mps | 30 ++ .../models/NewSolution.toBeDeletedModel.mps | 7 + .../solutions/ToBeDeleted/ToBeDeleted.msd | 18 + settings.gradle.kts | 3 +- 98 files changed, 3418 insertions(+), 561 deletions(-) delete mode 100644 bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/RecreateProjectFromModelServerTest.kt create mode 100644 bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IModelMask.kt rename bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/{MPSProjectSyncFilter.kt => MPSProjectSyncMask.kt} (63%) create mode 100644 mps-sync-plugin3/build.gradle.kts create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt create mode 100644 mps-sync-plugin3/src/main/resources/META-INF/plugin.xml create mode 100644 mps-sync-plugin3/src/main/resources/META-INF/pluginIcon.svg create mode 100644 mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt create mode 100644 mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt create mode 100644 mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt rename {bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test => mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3}/XMLUtils.kt (98%) create mode 100644 mps-sync-plugin3/src/test/resources/logback-test.xml create mode 100644 mps-sync-plugin3/testdata/.gitignore rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/.mps/.gitignore (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/.mps/migration.xml (100%) create mode 100644 mps-sync-plugin3/testdata/change1/.mps/modules.xml create mode 100644 mps-sync-plugin3/testdata/change1/devkits/NewDevkit/NewDevkit.devkit rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/languages/NewLanguage/NewLanguage.mpl (100%) create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/languages/NewLanguage/models/NewLanguage.behavior.mps (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/languages/NewLanguage/models/NewLanguage.constraints.mps (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/languages/NewLanguage/models/NewLanguage.editor.mps (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/languages/NewLanguage/models/NewLanguage.structure.mps (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/languages/NewLanguage/models/NewLanguage.typesystem.mps (100%) create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/NewLanguage2.mpl create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/generator/templates/NewLanguage2.generator.templates@generator.mps create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.behavior.mps create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.constraints.mps create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.editor.mps create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.structure.mps create mode 100644 mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.typesystem.mps create mode 100644 mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/NewRuntimeSolution.msd create mode 100644 mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.modelB.mps create mode 100644 mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps create mode 100644 mps-sync-plugin3/testdata/change1/solutions/NewSolution/NewSolution.msd create mode 100644 mps-sync-plugin3/testdata/change1/solutions/NewSolution/models/NewSolution.a_model.mps rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/change1}/solutions/NewSolution/models/NewSolution.b_model.mps (100%) create mode 100644 mps-sync-plugin3/testdata/initial/.mps/.gitignore create mode 100644 mps-sync-plugin3/testdata/initial/.mps/migration.xml rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/.mps/modules.xml (85%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/devkits/NewDevkit/NewDevkit.devkit (85%) create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/NewLanguage.mpl rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps (100%) create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/generator1/templates/NewLanguage.generator01.templates@generator.mps create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.behavior.mps create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.constraints.mps create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.editor.mps create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.structure.mps create mode 100644 mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.typesystem.mps rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/solutions/NewRuntimeSolution/NewRuntimeSolution.msd (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps (100%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/solutions/NewSolution/NewSolution.msd (87%) rename {bulk-model-sync-lib/mps-test/testdata/nonTrivialProject => mps-sync-plugin3/testdata/initial}/solutions/NewSolution/models/NewSolution.a_model.mps (100%) create mode 100644 mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.b_model.mps create mode 100644 mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.toBeDeletedModel.mps create mode 100644 mps-sync-plugin3/testdata/initial/solutions/ToBeDeleted/ToBeDeleted.msd diff --git a/bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/RecreateProjectFromModelServerTest.kt b/bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/RecreateProjectFromModelServerTest.kt deleted file mode 100644 index dcac9d7d95..0000000000 --- a/bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/RecreateProjectFromModelServerTest.kt +++ /dev/null @@ -1,269 +0,0 @@ -package org.modelix.model.sync.bulk.lib.test - -import com.intellij.ide.impl.OpenProjectTask -import com.intellij.openapi.Disposable -import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.project.Project -import com.intellij.openapi.project.ProjectManager -import com.intellij.openapi.project.ex.ProjectManagerEx -import com.intellij.openapi.util.Disposer -import com.intellij.testFramework.TestApplicationManager -import com.intellij.testFramework.UsefulTestCase -import com.intellij.util.io.delete -import jetbrains.mps.ide.ThreadUtils -import jetbrains.mps.ide.project.ProjectHelper -import jetbrains.mps.project.AbstractModule -import jetbrains.mps.project.MPSProject -import jetbrains.mps.smodel.Language -import jetbrains.mps.smodel.MPSModuleRepository -import org.jetbrains.mps.openapi.model.EditableSModel -import org.modelix.model.api.PBranch -import org.modelix.model.api.getRootNode -import org.modelix.model.client.IdGenerator -import org.modelix.model.data.ModelData -import org.modelix.model.data.asData -import org.modelix.model.lazy.CLTree -import org.modelix.model.lazy.CLVersion -import org.modelix.model.lazy.ObjectStoreCache -import org.modelix.model.mpsadapters.asReadableNode -import org.modelix.model.mpsadapters.asWritableNode -import org.modelix.model.persistent.MapBaseStore -import org.modelix.model.sync.bulk.ModelSynchronizer -import org.modelix.model.sync.bulk.NodeAssociationFromModelServer -import org.modelix.model.sync.bulk.NodeAssociationToModelServer -import org.modelix.mps.api.ModelixMpsApi -import org.modelix.mps.model.sync.bulk.MPSProjectSyncFilter -import org.w3c.dom.Element -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import kotlin.io.path.ExperimentalPathApi -import kotlin.io.path.absolute - -class RecreateProjectFromModelServerTest : UsefulTestCase() { - - protected lateinit var project: Project - - override fun runInDispatchThread() = false - - override fun setUp() { - super.setUp() - } - - fun `test same file content`() { - TestApplicationManager.getInstance() - - val originalProject = openTestProject("nonTrivialProject") - project = originalProject - - val store = ObjectStoreCache(MapBaseStore()).getAsyncStore() - val idGenerator = IdGenerator.getInstance(0xabcd) - val emptyVersion = CLVersion.createRegularVersion( - id = idGenerator.generate(), - time = null, - author = this::class.java.name, - tree = CLTree.builder(store).repositoryId("unit-test-repo").build(), - baseVersion = null, - operations = emptyArray(), - ) - val branch = PBranch(emptyVersion.getTree(), idGenerator) - mpsProject.modelAccess.runReadAction { - branch.runWrite { - val mpsRoot = mpsProject.repository.asReadableNode() - val modelServerRoot = branch.getRootNode().asWritableNode() - ModelSynchronizer( - filter = MPSProjectSyncFilter(listOf(mpsProject), toMPS = false), - sourceRoot = mpsRoot, - targetRoot = modelServerRoot, - nodeAssociation = NodeAssociationToModelServer(branch), - ).synchronize() - println(ModelData(root = modelServerRoot.asLegacyNode().asData()).toJson()) - } - } - - fun filterFiles(files: Map) = files.filter { - val name = it.key - if (name.startsWith(".mps/")) { - false // name == ".mps/modules.xml" - } else if (name.contains("/source_gen") || name.contains("/classes_gen")) { - false - } else { - true - } - } - - val originalContents = filterFiles(originalProject.captureFileContents()) - originalProject.close() - - val emptyProject = openTestProject(null) - project = emptyProject - - mpsProject.modelAccess.executeCommandInEDT { - branch.runRead { - val mpsRoot = mpsProject.repository.asWritableNode() - val modelServerRoot = branch.getRootNode().asReadableNode() - val modelSynchronizer = ModelSynchronizer( - filter = MPSProjectSyncFilter(listOf(mpsProject), toMPS = true), - sourceRoot = modelServerRoot, - targetRoot = mpsRoot, - nodeAssociation = NodeAssociationFromModelServer(branch, mpsRoot.getModel()), - ) - ModelixMpsApi.runWithProject(mpsProject) { - modelSynchronizer.synchronize() - } - } - } - - val syncedContents = filterFiles(emptyProject.captureFileContents()) - - fun Map.contentsAsString(): String { - return entries.sortedBy { it.key }.joinToString("\n\n\n") { "------ ${it.key} ------\n${it.value}" } - .replace(""" writeAction(body: () -> R): R { - return mpsProject.modelAccess.computeWriteAction(body) - } - - protected fun writeActionOnEdt(body: () -> R): R { - return onEdt { writeAction { body() } } - } - - protected fun onEdt(body: () -> R): R { - var result: R? = null - ThreadUtils.runInUIThreadAndWait { - result = body() - } - return result as R - } - - protected fun readAction(body: () -> R): R { - var result: R? = null - mpsProject.modelAccess.runReadAction { - result = body() - } - return result as R - } -} - -fun Project.close() { - ApplicationManager.getApplication().invokeLaterOnWriteThread { - runCatching { - ProjectManager.getInstance().closeAndDispose(this) - } - } - ApplicationManager.getApplication().invokeAndWait { } -} - -private fun Project.captureFileContents(): Map { - ApplicationManager.getApplication().invokeAndWait { - MPSModuleRepository.getInstance().modelAccess.runWriteAction { - for (module in ProjectHelper.fromIdeaProject(this)!!.projectModules.flatMap { - listOf(it) + ((it as? Language)?.generators ?: emptyList()) - }) { - module as AbstractModule - module.save() - for (model in module.models.filterIsInstance()) { - ModelixMpsApi.forceSave(model) - } - } - } - ApplicationManager.getApplication().saveAll() - save() - } - return File(this.basePath).walk().filter { it.isFile }.associate { file -> - val name = file.absoluteFile.relativeTo(File(basePath).absoluteFile).path - val content = file.readText().trim() - val xmlEndings = setOf("mps", "devkit", "mpl", "msd") - val normalizedContent = when { - xmlEndings.contains(name.substringAfterLast(".")) -> normalizeXmlFile(content) - else -> content - } - name to normalizedContent - } -} - -private fun normalizeXmlFile(content: String): String { - val xml = readXmlFile(content.byteInputStream()) - xml.visitAll { node -> - if (node !is Element) return@visitAll - when (node.tagName) { - "node" -> { - node.childElements("property").sortByRole() - node.childElements("ref").sortByRole() - node.childElements("node").sortByRole() - } - "sourceRoot" -> { - val location = node.getAttribute("location") - val path = node.getAttribute("path") - if (path.isNullOrEmpty() && !location.isNullOrEmpty()) { - val contentPath = (node.parentNode as Element).getAttribute("contentPath") - node.removeAttribute("location") - node.setAttribute("path", "$contentPath/$location") - } - } - } - } - return xmlToString(xml).lineSequence().map { it.trim() }.filter { it.isEmpty() }.joinToString("\n") -} - -private fun List.sortByRole() { - if (size < 2) return - val sorted = sortedBy { it.getAttribute("role") } - for (i in (0..sorted.lastIndex - 1).reversed()) { - sorted[i].parentNode.insertBefore(sorted[i], sorted[i + 1]) - } -} diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IModelMask.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IModelMask.kt new file mode 100644 index 0000000000..e35a360515 --- /dev/null +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/IModelMask.kt @@ -0,0 +1,37 @@ +package org.modelix.model.sync.bulk + +import org.modelix.model.api.IChildLinkReference +import org.modelix.model.api.IReadableNode + +interface IModelMask { + fun filterChildren(parent: IReadableNode, role: IChildLinkReference, children: List): List + + fun filterChildren(parent: IReadableNode, children: List): List { + val included = children.groupBy { it.getContainmentLink() } + .flatMap { filterChildren(parent, it.key, it.value) } + .toSet() + return children.filter { included.contains(it) } + } +} + +class UnfilteredModelMask : IModelMask { + override fun filterChildren( + parent: IReadableNode, + role: IChildLinkReference, + children: List, + ): List { + return children + } +} + +class CombinedModelMask(val mask1: IModelMask, val mask2: IModelMask) : IModelMask { + override fun filterChildren( + parent: IReadableNode, + role: IChildLinkReference, + children: List, + ): List { + return mask2.filterChildren(parent, role, mask1.filterChildren(parent, role, children)) + } +} + +fun IModelMask.combine(mask2: IModelMask): IModelMask = CombinedModelMask(this, mask2) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt index a6e3aaf5af..d9b0a9449e 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelImporter.kt @@ -4,13 +4,11 @@ import mu.KotlinLogging import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.INode import org.modelix.model.api.IReadableNode -import org.modelix.model.api.IWritableNode import org.modelix.model.api.PNodeAdapter import org.modelix.model.data.ModelData import org.modelix.model.data.NodeData import org.modelix.model.data.NodeDataAsNode import org.modelix.model.data.ensureHasId -import org.modelix.model.sync.bulk.ModelSynchronizer.IFilter import kotlin.jvm.JvmName private val LOG = KotlinLogging.logger { } @@ -65,26 +63,18 @@ class ModelImporter( val targetNodes = sourceAndTargetNodes.map { it.existingNode.asWritableNode() } ModelSynchronizer( - filter = object : IFilter { - override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean { - return true - } - - override fun needsSynchronization(node: IReadableNode): Boolean { - return true - } - - override fun filterTargetChildren( - parent: IWritableNode, + sourceRoot = sourceNodes.first(), + targetRoot = targetNodes.first(), + nodeAssociation = nodeAssociation, + targetMask = object : IModelMask { + override fun filterChildren( + parent: IReadableNode, role: IChildLinkReference, - children: List, - ): List { + children: List, + ): List { return children.filter { childFilter(it.asLegacyNode()) } } }, - sourceRoot = sourceNodes.first(), - targetRoot = targetNodes.first(), - nodeAssociation = nodeAssociation, ).synchronize(sourceNodes, targetNodes) } } diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 56003fbeb5..2d39211d41 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -12,14 +12,14 @@ import org.modelix.model.api.PNodeAdapter import org.modelix.model.api.getOriginalOrCurrentReference import org.modelix.model.api.getOriginalReference import org.modelix.model.api.isChildRoleOrdered +import org.modelix.model.api.isOrdered import org.modelix.model.api.matches import org.modelix.model.api.mergeWith import org.modelix.model.api.remove import org.modelix.model.api.syncNewChild import org.modelix.model.api.syncNewChildren -import org.modelix.model.api.tryResolve import org.modelix.model.data.NodeData -import org.modelix.model.sync.bulk.ModelSynchronizer.IFilter +import org.modelix.model.sync.bulk.ModelSynchronizer.IIncrementalUpdateInformation /** * Similar to [ModelImporter], but the input is two [INode] instances instead of [INode] and [NodeData]. @@ -33,10 +33,12 @@ import org.modelix.model.sync.bulk.ModelSynchronizer.IFilter * @param nodeAssociation mapping between source and target nodes, that is used for internal optimizations */ class ModelSynchronizer( - val filter: IFilter, + val filter: IIncrementalUpdateInformation = FullSyncFilter(), val sourceRoot: IReadableNode, val targetRoot: IWritableNode, val nodeAssociation: INodeAssociation, + val sourceMask: IModelMask = UnfilteredModelMask(), + val targetMask: IModelMask = UnfilteredModelMask(), ) { private val nodesToRemove: MutableSet = HashSet() private val pendingReferences: MutableList = ArrayList() @@ -47,26 +49,25 @@ class ModelSynchronizer( } fun synchronize(sourceNodes: List, targetNodes: List) { - logger.info { "Synchronizing nodes..." } + logger.debug { "Synchronizing nodes..." } for ((sourceNode, targetNode) in sourceNodes.zip(targetNodes)) { synchronizeNode(sourceNode, targetNode) } - synchronizeNode(sourceRoot, targetRoot) - logger.info { "Synchronizing pending references..." } + logger.debug { "Synchronizing pending references..." } pendingReferences.forEach { if (!it.trySyncReference()) { it.copyTargetRef() } } - logger.info { "Removing extra nodes..." } + logger.debug { "Removing extra nodes..." } nodesToRemove.filter { it.isValid() }.forEach { it.remove() } - logger.info { "Synchronization finished." } + logger.debug { "Synchronization finished." } } private fun synchronizeNode(sourceNode: IReadableNode, targetNode: IWritableNode) { nodeAssociation.associate(sourceNode, targetNode) if (filter.needsSynchronization(sourceNode)) { - logger.info { "Synchronizing changed node. sourceNode = $sourceNode" } + logger.trace { "Synchronizing changed node. sourceNode = $sourceNode" } synchronizeProperties(sourceNode, targetNode) synchronizeReferences(sourceNode, targetNode) @@ -81,12 +82,13 @@ class ModelSynchronizer( syncChildren(sourceNode, conceptCorrectedTargetNode) } else if (filter.needsDescentIntoSubtree(sourceNode)) { - for (sourceChild in sourceNode.getAllChildren()) { - val targetChild = nodeAssociation.resolveTarget(sourceChild) ?: error("Expected target node was not found. sourceChild=$sourceChild") + for (sourceChild in sourceMask.filterChildren(sourceNode, sourceNode.getAllChildren())) { + val targetChild = nodeAssociation.resolveTarget(sourceChild) + ?: error("Expected target node was not found. sourceChild=${sourceChild.getNodeReference()}, originalId=${sourceChild.getOriginalReference()}") synchronizeNode(sourceChild, targetChild) } } else { - logger.info { "Skipping subtree due to filter. root = $sourceNode" } + logger.trace { "Skipping subtree due to filter. root = $sourceNode" } } } @@ -119,11 +121,11 @@ class ModelSynchronizer( } private fun getFilteredSourceChildren(parent: IReadableNode, role: IChildLinkReference): List { - return parent.getChildren(role).let { filter.filterSourceChildren(parent, role, it) } + return parent.getChildren(role).let { sourceMask.filterChildren(parent, role, it) } } private fun getFilteredTargetChildren(parent: IWritableNode, role: IChildLinkReference): List { - return parent.getChildren(role).let { filter.filterTargetChildren(parent, role, it) } + return parent.getChildren(role).let { targetMask.filterChildren(parent, role, it) } } private fun syncChildren(sourceParent: IReadableNode, targetParent: IWritableNode) { @@ -191,7 +193,7 @@ class ModelSynchronizer( // it is potentially moved to a new parent and role. if (existingNode.getParent() != targetParent || !existingNode.getContainmentLink().matches(role) || - role.tryResolve(targetParent.getConceptReference())?.isOrdered != false + targetParent.isOrdered(role) ) { targetParent.moveChild(role, newIndex, existingNode) } @@ -294,7 +296,7 @@ class ModelSynchronizer( * * It is valid for [needsDescentIntoSubtree] and [needsSynchronization] to return true for the same node. */ - interface IFilter { + interface IIncrementalUpdateInformation { /** * Checks if a subtree needs synchronization. * @@ -310,16 +312,17 @@ class ModelSynchronizer( * @return true iff the node must not be skipped */ fun needsSynchronization(node: IReadableNode): Boolean - - fun filterSourceChildren(parent: IReadableNode, role: IChildLinkReference, children: List): List = children - - fun filterTargetChildren(parent: IWritableNode, role: IChildLinkReference, children: List): List = children } } -fun IFilter.and(other: IFilter): IFilter = AndFilter(this, other) +class FullSyncFilter : IIncrementalUpdateInformation { + override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean = true + override fun needsSynchronization(node: IReadableNode): Boolean = true +} + +fun IIncrementalUpdateInformation.and(other: IIncrementalUpdateInformation): IIncrementalUpdateInformation = AndFilter(this, other) -class AndFilter(val filter1: IFilter, val filter2: IFilter) : IFilter { +class AndFilter(val filter1: IIncrementalUpdateInformation, val filter2: IIncrementalUpdateInformation) : IIncrementalUpdateInformation { override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean { return filter1.needsDescentIntoSubtree(subtreeRoot) && filter2.needsDescentIntoSubtree(subtreeRoot) } @@ -327,22 +330,6 @@ class AndFilter(val filter1: IFilter, val filter2: IFilter) : IFilter { override fun needsSynchronization(node: IReadableNode): Boolean { return filter1.needsSynchronization(node) && filter2.needsSynchronization(node) } - - override fun filterSourceChildren( - parent: IReadableNode, - role: IChildLinkReference, - children: List, - ): List { - return filter2.filterSourceChildren(parent, role, filter1.filterSourceChildren(parent, role, children)) - } - - override fun filterTargetChildren( - parent: IWritableNode, - role: IChildLinkReference, - children: List, - ): List { - return filter2.filterTargetChildren(parent, role, filter1.filterTargetChildren(parent, role, children)) - } } private fun INode.originalIdOrFallback(): String? { diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/NodeAssociationToModelServer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/NodeAssociationToModelServer.kt index d87c90ea81..513337011f 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/NodeAssociationToModelServer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/NodeAssociationToModelServer.kt @@ -48,9 +48,9 @@ class NodeAssociationFromModelServer(val branch: IBranch, val targetModel: IMuta } override fun associate(sourceNode: IReadableNode, targetNode: IWritableNode) { - val id = nodeId(sourceNode) val expected = sourceNode.getOriginalOrCurrentReference() if (expected != targetNode.getOriginalOrCurrentReference()) { + val id = nodeId(sourceNode) pendingAssociations[id] = targetNode.getNodeReference().serialize() } } diff --git a/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/ModelSynchronizerTest.kt b/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/ModelSynchronizerTest.kt index 246691732e..5b0c8a004b 100644 --- a/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/ModelSynchronizerTest.kt +++ b/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/ModelSynchronizerTest.kt @@ -3,7 +3,6 @@ package org.modelix.model.sync.bulk import org.modelix.model.ModelFacade import org.modelix.model.api.IBranch import org.modelix.model.api.IChildLinkReference -import org.modelix.model.api.IReadableNode import org.modelix.model.api.PBranch import org.modelix.model.api.getDescendants import org.modelix.model.api.getRootNode @@ -94,7 +93,7 @@ open class ModelSynchronizerTest : AbstractModelSyncTest() { targetBranch.runWrite { val targetRoot = targetBranch.getRootNode() val synchronizer = ModelSynchronizer( - filter = BasicFilter, + filter = FullSyncFilter(), sourceRoot = sourceRoot.asReadableNode(), targetRoot = targetRoot.asWritableNode(), nodeAssociation = NodeAssociationToModelServer(targetBranch), @@ -143,7 +142,7 @@ open class ModelSynchronizerTest : AbstractModelSyncTest() { otBranch.runWrite { ModelSynchronizer( - filter = BasicFilter, + filter = FullSyncFilter(), sourceRoot = sourceBranch.getRootNode().asReadableNode(), targetRoot = targetBranch.getRootNode().asWritableNode(), nodeAssociation = NodeAssociationToModelServer(targetBranch), @@ -159,16 +158,6 @@ open class ModelSynchronizerTest : AbstractModelSyncTest() { operations.size <= expectedNumOps } } - - object BasicFilter : ModelSynchronizer.IFilter { - override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean { - return true - } - - override fun needsSynchronization(node: IReadableNode): Boolean { - return true - } - } } private fun IBranch.toOTBranch(): OTBranch { diff --git a/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidatingVisitor.kt b/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidatingVisitor.kt index cc650a9362..2b1b6f875f 100644 --- a/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidatingVisitor.kt +++ b/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidatingVisitor.kt @@ -2,14 +2,17 @@ package org.modelix.model.sync.bulk import org.modelix.model.api.ITree import org.modelix.model.api.ITreeChangeVisitorEx -import org.modelix.model.data.NodeData +import org.modelix.model.api.TreePointer +import org.modelix.model.api.getNode /** * Visitor that visits a [tree] and stores the invalidation information in an [invalidationTree]. */ -class InvalidatingVisitor(val tree: ITree, val invalidationTree: InvalidationTree) : ITreeChangeVisitorEx { +class InvalidatingVisitor(val newTree: ITree, val invalidationTree: InvalidationTree) : ITreeChangeVisitorEx { - private fun invalidateNode(nodeId: Long) = invalidationTree.invalidate(tree, nodeId) + private fun invalidateNode(nodeId: Long) { + invalidationTree.invalidate(TreePointer(newTree).getNode(nodeId).asReadableNode(), false) + } override fun containmentChanged(nodeId: Long) { // Containment can only change if also the children of the parent changed. @@ -25,7 +28,6 @@ class InvalidatingVisitor(val tree: ITree, val invalidationTree: InvalidationTre } override fun propertyChanged(nodeId: Long, role: String) { - if (role == NodeData.ID_PROPERTY_KEY) return invalidateNode(nodeId) } diff --git a/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt b/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt index eb7ff84dc1..95a72c2382 100644 --- a/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt +++ b/bulk-model-sync-lib/src/jvmMain/kotlin/org/modelix/model/sync/bulk/InvalidationTree.kt @@ -1,10 +1,12 @@ package org.modelix.model.sync.bulk -import gnu.trove.map.TLongObjectMap -import gnu.trove.map.hash.TLongObjectHashMap +import org.modelix.model.api.IModel import org.modelix.model.api.IReadableNode import org.modelix.model.api.ITree +import org.modelix.model.api.NodeReference import org.modelix.model.api.PNodeAdapter +import org.modelix.model.api.ancestors +import org.modelix.model.api.toSerialized /** * The purpose of this data structure is to store which nodes changed and need to be synchronized, @@ -13,15 +15,50 @@ import org.modelix.model.api.PNodeAdapter * If there are many changes in one part of the model (changed subtree size > [sizeLimit]), * we do not keep track of the individual changes anymore and just synchronize the entire subtree. */ -class InvalidationTree(val sizeLimit: Int) : ModelSynchronizer.IFilter { - private val rootNode = Node(ITree.ROOT_ID) +class InvalidationTree(sizeLimit: Int = 100_000) : GenericInvalidationTree(ITree.ROOT_ID, sizeLimit) { + override fun ancestorsAndSelf(model: ITree, nodeId: Long): List { + return model.ancestorsAndSelf(nodeId).toList() + } + + override fun getId(node: IReadableNode): Long { + val subtreeRoot = node.asLegacyNode() + require(subtreeRoot is PNodeAdapter) + return subtreeRoot.nodeId + } +} + +class DefaultInvalidationTree(val root: NodeReference, sizeLimit: Int = 100_000) : + GenericInvalidationTree(root, sizeLimit = sizeLimit) { + override fun ancestorsAndSelf( + model: IModel, + nodeId: NodeReference, + ): List { + return model.resolveNode(nodeId) + ?.ancestors(includeSelf = true) + ?.map { it.getNodeReference().toSerialized() } + ?.toList() + ?: emptyList() + } + + override fun getId(node: IReadableNode): NodeReference { + return node.getNodeReference().toSerialized() + } +} + +abstract class GenericInvalidationTree(root: ID, val sizeLimit: Int = 100_000) : ModelSynchronizer.IIncrementalUpdateInformation { + private val rootNode = Node(root) + + abstract fun ancestorsAndSelf(model: M, nodeId: ID): List + abstract fun getId(node: IReadableNode): ID + + private fun getContainmentPath(node: IReadableNode) = node.ancestors(includeSelf = true).map { getId(it) }.toList().asReversed() /** * Marks the node stored in the given containment path as changed. */ - fun invalidate(containmentPath: LongArray) { - require(containmentPath[0] == ITree.ROOT_ID) { "Path must start with the root node" } - rootNode.invalidate(containmentPath, 0) + fun invalidate(containmentPath: List, includingDescendants: Boolean = false) { + require(containmentPath[0] == rootNode.id) { "Path must start with the root node. Expected: ${rootNode.id}, was: $containmentPath" } + rootNode.invalidate(containmentPath, 0, includingDescendants) rootNode.rebalance(sizeLimit) } @@ -31,38 +68,54 @@ class InvalidationTree(val sizeLimit: Int) : ModelSynchronizer.IFilter { * @param tree used internally for the calculation of the containment path * @param nodeId the id of the changed node */ - fun invalidate(tree: ITree, nodeId: Long) { - val containmentPath = tree.ancestorsAndSelf(nodeId).toList().asReversed().toLongArray() - invalidate(containmentPath) + fun invalidate(node: IReadableNode, includingDescendants: Boolean = false) { + invalidate(getContainmentPath(node), includingDescendants) + } + + fun reset() { + rootNode.reset() } + fun hasAnyInvalidations(): Boolean = rootNode.hasAnyInvalidations() + override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean { - val subtreeRoot = subtreeRoot.asLegacyNode() - require(subtreeRoot is PNodeAdapter) - val path = subtreeRoot.branch.transaction.tree.ancestorsAndSelf(subtreeRoot.nodeId).toList().asReversed() - return rootNode.needsDescentIntoSubtree(path, 0) + return rootNode.needsDescentIntoSubtree(getContainmentPath(subtreeRoot), 0) } override fun needsSynchronization(node: IReadableNode): Boolean { - val node = node.asLegacyNode() - require(node is PNodeAdapter) - val path = node.branch.transaction.tree.ancestorsAndSelf(node.nodeId).toList().asReversed() - return rootNode.nodeNeedsUpdate(path, 0) + return rootNode.nodeNeedsUpdate(getContainmentPath(node), 0) } - private class Node(val id: Long) { + private class Node(val id: E) { private var subtreeSize = 1 private var nodeNeedsUpdate: Boolean = false private var allDescendantsNeedUpdate = false - private var invalidChildren: TLongObjectMap = TLongObjectHashMap() + private var invalidChildren: MutableMap> = HashMap() + + fun hasAnyInvalidations() = nodeNeedsUpdate || allDescendantsNeedUpdate || invalidChildren.isNotEmpty() + + fun reset() { + subtreeSize = 1 + nodeNeedsUpdate = false + allDescendantsNeedUpdate = false + invalidChildren = HashMap() + } /** * @return number of added nodes */ - fun invalidate(path: LongArray, currentIndex: Int): Int { + fun invalidate(path: List, currentIndex: Int, includingDescendants: Boolean): Int { var addedNodesCount = 0 if (currentIndex > path.lastIndex) { nodeNeedsUpdate = true + if (includingDescendants) { + addedNodesCount -= subtreeSize - 1 + subtreeSize = 1 + allDescendantsNeedUpdate = true + if (invalidChildren.isNotEmpty()) { + invalidChildren = HashMap(0) + } + } } else { if (allDescendantsNeedUpdate) return addedNodesCount val childId = path[currentIndex] @@ -70,7 +123,7 @@ class InvalidationTree(val sizeLimit: Int) : ModelSynchronizer.IFilter { invalidChildren.put(childId, it) addedNodesCount++ } - addedNodesCount += child.invalidate(path, currentIndex + 1) + addedNodesCount += child.invalidate(path, currentIndex + 1, includingDescendants) } subtreeSize += addedNodesCount return addedNodesCount @@ -82,17 +135,17 @@ class InvalidationTree(val sizeLimit: Int) : ModelSynchronizer.IFilter { if (sizeLimit >= subtreeSize) return // limit already fulfilled - if ((sizeLimit - 1) < invalidChildren.size()) { + if ((sizeLimit - 1) < invalidChildren.size) { // rebalancing not possible without removing nodes allDescendantsNeedUpdate = true - invalidChildren = TLongObjectHashMap(0) + invalidChildren = HashMap(0) subtreeSize = 1 return } var remainingNodesToRemove = subtreeSize - sizeLimit - val sortedChildren: List = invalidChildren.valueCollection().sortedByDescending { it.subtreeSize } + val sortedChildren: List> = invalidChildren.values.sortedByDescending { it.subtreeSize } val rebalancedSizes: IntArray = sortedChildren.map { it.subtreeSize }.toIntArray() while (remainingNodesToRemove > 0) { for (i in rebalancedSizes.indices) { @@ -112,20 +165,20 @@ class InvalidationTree(val sizeLimit: Int) : ModelSynchronizer.IFilter { child.rebalance(rebalancedSizes[i]) } - subtreeSize = invalidChildren.valueCollection().sumOf { it.subtreeSize } + 1 + subtreeSize = invalidChildren.values.sumOf { it.subtreeSize } + 1 } - fun needsDescentIntoSubtree(path: List, currentIndex: Int): Boolean { + fun needsDescentIntoSubtree(path: List, currentIndex: Int): Boolean { if (allDescendantsNeedUpdate) return true if (currentIndex < path.size) { val child = invalidChildren[path[currentIndex]] ?: return false return child.needsDescentIntoSubtree(path, currentIndex + 1) } else { - return invalidChildren.size() > 0 + return invalidChildren.size > 0 } } - fun nodeNeedsUpdate(path: List, currentIndex: Int): Boolean { + fun nodeNeedsUpdate(path: List, currentIndex: Int): Boolean { if (allDescendantsNeedUpdate) return true if (currentIndex < path.size) { val child = invalidChildren[path[currentIndex]] ?: return false diff --git a/bulk-model-sync-lib/src/jvmTest/kotlin/org/modelix/model/sync/bulk/InvalidationTreeTest.kt b/bulk-model-sync-lib/src/jvmTest/kotlin/org/modelix/model/sync/bulk/InvalidationTreeTest.kt index f3dde90572..7b8b05b363 100644 --- a/bulk-model-sync-lib/src/jvmTest/kotlin/org/modelix/model/sync/bulk/InvalidationTreeTest.kt +++ b/bulk-model-sync-lib/src/jvmTest/kotlin/org/modelix/model/sync/bulk/InvalidationTreeTest.kt @@ -19,7 +19,7 @@ class InvalidationTreeTest { val testTree = getTestTreeData() val treePointer = TreePointer(testTree) - invalidationTree.invalidate(testTree, 32L) + invalidationTree.invalidate(treePointer.getNode(32L).asReadableNode()) val invalidatedNode = treePointer.computeRead { treePointer.getNode(32L).asReadableNode() } assertTrue { invalidationTree.needsSynchronization(invalidatedNode) } @@ -31,7 +31,7 @@ class InvalidationTreeTest { val testTree = getTestTreeData() val treePointer = TreePointer(testTree) - invalidationTree.invalidate(testTree, 32L) + invalidationTree.invalidate(treePointer.getNode(32L).asReadableNode()) val invalidatedNode = treePointer.computeRead { treePointer.getNode(32L) } val ancestors = invalidatedNode.getAncestors(includeSelf = false) @@ -44,7 +44,7 @@ class InvalidationTreeTest { val testTree = getTestTreeData() val treePointer = TreePointer(testTree) - invalidationTree.invalidate(testTree, 3L) + invalidationTree.invalidate(treePointer.getNode(3L).asReadableNode()) val descendants = treePointer.getNode(3L).getDescendants(false) assertTrue { descendants.none { invalidationTree.needsSynchronization(it.asReadableNode()) } } @@ -57,8 +57,8 @@ class InvalidationTreeTest { val testTree = getTestTreeData() val treePointer = TreePointer(testTree) - invalidationTree.invalidate(testTree, 3L) - invalidationTree.invalidate(testTree, 311L) + invalidationTree.invalidate(treePointer.getNode(3L).asReadableNode()) + invalidationTree.invalidate(treePointer.getNode(311L).asReadableNode()) val ancestors = treePointer.getNode(3L).getAncestors(false) diff --git a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/CompositeFilter.kt b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/CompositeFilter.kt index 1ad83e7d54..e5be2bc4da 100644 --- a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/CompositeFilter.kt +++ b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/CompositeFilter.kt @@ -9,7 +9,7 @@ import org.modelix.model.sync.bulk.ModelSynchronizer * * @param filters collection of filters. If the collection is ordered, the filters will be evaluated in the specified order. */ -class CompositeFilter(private val filters: Collection) : ModelSynchronizer.IFilter { +class CompositeFilter(private val filters: Collection) : ModelSynchronizer.IIncrementalUpdateInformation { override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean = filters.all { it.needsDescentIntoSubtree(subtreeRoot) } diff --git a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt index 1791c72812..65a228c5db 100644 --- a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt +++ b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt @@ -19,7 +19,7 @@ class IncludedModulesFilter( val includedModulePrefixes: Collection, val excludedModules: Collection = emptySet(), val excludedModulesPrefixes: Collection = emptySet(), -) : ModelSynchronizer.IFilter { +) : ModelSynchronizer.IIncrementalUpdateInformation { override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean { if (subtreeRoot.getConceptReference() != BuiltinLanguages.MPSRepositoryConcepts.Module.getReference()) return true val moduleName = subtreeRoot.getPropertyValue(BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name.toReference()) ?: return true diff --git a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSBulkSynchronizer.kt b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSBulkSynchronizer.kt index a9de46c9a6..70ff59e408 100644 --- a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSBulkSynchronizer.kt +++ b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSBulkSynchronizer.kt @@ -274,8 +274,8 @@ object MPSBulkSynchronizer { println("Loading version ${version.getContentHash()}") executeCommandWithExceptionHandling(repository) { - val invalidationTree = InvalidationTree(1_000_000) val newTree = version.getTree() + val invalidationTree = InvalidationTree() newTree.visitChanges( baseVersion.getTree(), InvalidatingVisitor(newTree, invalidationTree), diff --git a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSProjectSyncFilter.kt b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSProjectSyncMask.kt similarity index 63% rename from bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSProjectSyncFilter.kt rename to bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSProjectSyncMask.kt index b33fedb742..6c13411b13 100644 --- a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSProjectSyncFilter.kt +++ b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/MPSProjectSyncMask.kt @@ -4,31 +4,17 @@ import jetbrains.mps.project.MPSProject import org.modelix.model.api.BuiltinLanguages import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.IReadableNode -import org.modelix.model.api.IWritableNode import org.modelix.model.mpsadapters.MPSModuleAsNode import org.modelix.model.mpsadapters.MPSProjectAsNode -import org.modelix.model.sync.bulk.ModelSynchronizer +import org.modelix.model.sync.bulk.IModelMask -class MPSProjectSyncFilter(val projects: List, val toMPS: Boolean) : ModelSynchronizer.IFilter { +class MPSProjectSyncMask(val projects: List, val isMPSSide: Boolean) : IModelMask { - private val fromMPS: Boolean get() = !toMPS - - override fun needsDescentIntoSubtree(subtreeRoot: IReadableNode): Boolean { - return true - } - - override fun needsSynchronization(node: IReadableNode): Boolean { - return true - } - - private fun filterChildren( + override fun filterChildren( parent: IReadableNode, role: IChildLinkReference, children: List, - isSourceChildren: Boolean, ): List { - val isMPSSide = fromMPS == isSourceChildren - return when (parent.getConceptReference()) { BuiltinLanguages.MPSRepositoryConcepts.Repository.getReference() -> when { role.matches(BuiltinLanguages.MPSRepositoryConcepts.Repository.tempModules.toReference()) -> emptyList() @@ -56,20 +42,4 @@ class MPSProjectSyncFilter(val projects: List, val toMPS: Boolean) : else -> children } } - - override fun filterSourceChildren( - parent: IReadableNode, - role: IChildLinkReference, - children: List, - ): List { - return filterChildren(parent, role, children, true) - } - - override fun filterTargetChildren( - parent: IWritableNode, - role: IChildLinkReference, - children: List, - ): List { - return filterChildren(parent, role, children, false) - } } diff --git a/commitlint.config.js b/commitlint.config.js index c5dbd5206f..9ca8896a03 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -20,6 +20,7 @@ module.exports = { "modelql", "mps-model-adapters", "mps-model-server", + "mps-sync-plugin", "openapi", "ts-model-api", "vue-model-api", diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8bc194f308..3f40ec0a99 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,6 +19,7 @@ npm-publish = { id = "dev.petuska.npm.publish", version = "3.5.2" } test-logger = { id = "com.adarshr.test-logger", version = "4.0.0" } shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } intellij = { id = "org.jetbrains.intellij", version = "1.17.4" } +intellij2 = { id = "org.jetbrains.intellij.platform", version = "2.2.1" } openapi-generator = {id = "org.openapi.generator", version.ref = "openapi"} kotlinx-kover = { id = "org.jetbrains.kotlinx.kover", version = "0.9.1" } docker-compose = { id = "com.avast.gradle.docker-compose" , version = "0.17.12" } diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/BuiltinLanguages.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/BuiltinLanguages.kt index e9e7ce92b8..9cde61b2d2 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/BuiltinLanguages.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/BuiltinLanguages.kt @@ -206,6 +206,15 @@ object BuiltinLanguages { targetConcept = Generator, uid = "0a7577d1-d4e5-431d-98b1-fae38f9aee80/4008363636171860313/7018594982789597990", ).also(this::addChildLink) + + val extendedLanguages = SimpleChildLink( + simpleName = "extendedLanguages", + isMultiple = true, + isOptional = true, + isOrdered = false, + targetConcept = ModuleReference, + uid = "0a7577d1-d4e5-431d-98b1-fae38f9aee80/4008363636171860313/7440567989974771396", + ).also(this::addChildLink) } object DevKit : SimpleConcept( @@ -249,6 +258,11 @@ object BuiltinLanguages { directSuperConcepts = listOf(Module), ) { init { addConcept(this) } + + val alias = SimpleProperty( + "alias", + uid = "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618895/5552089503111831268", + ).also(this::addProperty) } object Repository : SimpleConcept( diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/INodeReference.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/INodeReference.kt index 8a90407041..d0be0511a0 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/INodeReference.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/INodeReference.kt @@ -46,6 +46,8 @@ fun INodeReference.resolveIn(scope: INodeResolutionScope): INode? { } } +fun INodeReference.toSerialized(): NodeReference = if (this is NodeReference) this else NodeReference(this.serialize()) + class NodeReferenceKSerializer : KSerializer { override fun deserialize(decoder: Decoder): INodeReference { val serialized = decoder.decodeString() diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/IWritableNode.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/IWritableNode.kt index 834de4e484..52a86d95c2 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/IWritableNode.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/IWritableNode.kt @@ -67,6 +67,14 @@ interface IWritableNode : IReadableNode { interface ISyncTargetNode : IWritableNode { fun syncNewChildren(role: IChildLinkReference, index: Int, specs: List): List + fun isOrdered(role: IChildLinkReference): Boolean = true +} + +fun IReadableNode.isOrdered(role: IChildLinkReference): Boolean { + return when (this) { + is ISyncTargetNode -> this.isOrdered(role) + else -> role.tryResolve(this.getConceptReference())?.isOrdered != false + } } fun IWritableNode.syncNewChildren(role: IChildLinkReference, index: Int, sourceNodes: List): List { @@ -90,3 +98,7 @@ class NewNodeSpec( ) { constructor(node: IReadableNode) : this(node.getConceptReference(), node, node.getOriginalReference()?.let { NodeReference(it) }) } + +fun T.ancestors(includeSelf: Boolean = false): Sequence { + return generateSequence(if (includeSelf) this else getParent() as T?) { it.getParent() as T? } +} diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt index 3ffd7e0244..5bbb2d6189 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt @@ -3,9 +3,6 @@ package org.modelix.model.api import org.modelix.model.area.IArea data class PNodeReference(val id: Long, val branchId: String) : INodeReference { - init { - PNodeReferenceSerializer.ensureRegistered() - } override fun resolveNode(area: IArea?): INode? { return area?.resolveNode(this) } @@ -17,7 +14,7 @@ data class PNodeReference(val id: Long, val branchId: String) : INodeReference { } override fun toString(): String { - return "PNodeReference_${id.toString(16)}@$branchId" + return serialize() } companion object { @@ -54,25 +51,3 @@ data class PNodeReference(val id: Long, val branchId: String) : INodeReference { } } } - -object PNodeReferenceSerializer : INodeReferenceSerializerEx { - override val prefix = "pnode" - override val supportedReferenceClasses = setOf(PNodeReference::class) - - init { - INodeReferenceSerializer.register(this) - } - - fun ensureRegistered() { - // Is done in the init section. Calling this method just ensures that the object is initialized. - } - - override fun serialize(ref: INodeReference): String { - return (ref as PNodeReference).let { "${ref.id.toString(16)}@${ref.branchId}" } - } - - override fun deserialize(serialized: String): INodeReference { - val parts = serialized.split('@', limit = 2) - return PNodeReference(parts[0].toLong(16), parts[1]) - } -} diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt index 77e2491190..ff095a22ff 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt @@ -22,6 +22,7 @@ import org.modelix.modelql.core.IMonoStep * such as [ModelClientV2]. */ interface IModelClientV2 { + suspend fun getServerId(): String fun getClientId(): Int fun getIdGenerator(): IIdGenerator fun getUserId(): String? diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt index 90f865c029..36795496d0 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt @@ -26,6 +26,7 @@ import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.URLBuilder import io.ktor.http.appendPathSegments +import io.ktor.http.buildUrl import io.ktor.http.contentType import io.ktor.http.takeFrom import io.ktor.serialization.kotlinx.json.json @@ -93,6 +94,15 @@ class ModelClientV2( return storeForRepository.asSequence().first { it.value.keyValueStore == store.keyValueStore }.key } + override suspend fun getServerId(): String { + return httpClient.get { + url { + takeFrom(baseUrl) + appendPathSegments("server-id") + } + }.bodyAsText() + } + private suspend fun updateClientId() { this.clientId = httpClient.post { url { @@ -498,7 +508,11 @@ abstract class ModelClientV2Builder { } fun url(url: String): ModelClientV2Builder { - baseUrl = url + baseUrl = buildUrl { + takeFrom(url) + if (pathSegments.lastOrNull() == "") pathSegments = pathSegments.dropLast(1) + if (pathSegments.lastOrNull() != "v2") appendPathSegments("v2") + }.toString() return this } @@ -655,21 +669,38 @@ suspend fun IModelClientV2.runWriteOnBranch(branchRef: BranchReference, body .takeIf { it != branchRef } ?.let { client.pullIfExists(it) } // master branch ?: client.initRepository(branchRef.repositoryId) - val branch = OTBranch(TreePointer(baseVersion.getTree(), client.getIdGenerator()), client.getIdGenerator(), (baseVersion as CLVersion).store) - val result = branch.computeWrite { + + var result: T? = null + val newVersion = baseVersion.runWrite(this) { + result = body(it) + } + if (newVersion != null) { + client.push(branchRef, newVersion, baseVersion) + } + return result as T +} + +fun IVersion.runWrite(client: IModelClientV2, body: (IBranch) -> Unit): IVersion? { + return runWrite(client.getIdGenerator(), client.getUserId(), body) +} + +fun IVersion.runWrite(idGenerator: IIdGenerator, author: String?, body: (IBranch) -> Unit): IVersion? { + val baseVersion = this + val branch = OTBranch(TreePointer(baseVersion.getTree(), idGenerator), idGenerator, (baseVersion as CLVersion).store) + branch.computeWrite { body(branch) } val (ops, newTree) = branch.getPendingChanges() if (ops.isEmpty()) { - return result + return null } - val newVersion = CLVersion.createRegularVersion( - id = client.getIdGenerator().generate(), - author = client.getUserId(), + return CLVersion.createRegularVersion( + id = idGenerator.generate(), + author = author, tree = newTree as CLTree, baseVersion = baseVersion as CLVersion?, operations = ops.map { it.getOriginalOp() }.toTypedArray(), ) - client.push(branchRef, newVersion, baseVersion) - return result } + +private fun String.ensureSuffix(suffix: String) = if (endsWith(suffix)) this else this + suffix diff --git a/model-client/src/jvmMain/kotlin/org/modelix/model/client2/LazyLoading.kt b/model-client/src/jvmMain/kotlin/org/modelix/model/client2/LazyLoading.kt index aa9e61019a..e0dabb6105 100644 --- a/model-client/src/jvmMain/kotlin/org/modelix/model/client2/LazyLoading.kt +++ b/model-client/src/jvmMain/kotlin/org/modelix/model/client2/LazyLoading.kt @@ -22,6 +22,7 @@ import org.modelix.model.persistent.HashUtil * the number of requests doesn't change by this prefetching, but small requests are filled to up to their limit with * additional prefetch requests. */ +@Deprecated("Use IAsyncTree instead") fun IModelClientV2.lazyLoadVersion(repositoryId: RepositoryId, versionHash: String, config: CacheConfiguration = CacheConfiguration()): IVersion { val store = ObjectStoreCache(ModelClientAsStore(this, repositoryId), config) return CLVersion.loadFromHash(versionHash, store) @@ -31,6 +32,7 @@ fun IModelClientV2.lazyLoadVersion(repositoryId: RepositoryId, versionHash: Stri * An overload of [IModelClientV2.lazyLoadVersion] that reads the current version hash of the branch from the server and * then loads that version with lazy loading support. */ +@Deprecated("Use IAsyncTree instead") suspend fun IModelClientV2.lazyLoadVersion(branchRef: BranchReference, config: CacheConfiguration = CacheConfiguration()): IVersion { return lazyLoadVersion(branchRef.repositoryId, pullHash(branchRef), config) } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt index ecf04f1d67..243d075d52 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt @@ -220,7 +220,7 @@ class OperationSerializer private constructor() { override fun deserialize(serialized: String): SetConceptOp { val parts = serialized.split(SEPARATOR) - return SetConceptOp(nodeId = longFromHex(parts[0]), concept = deserializeConcept(parts[2])) + return SetConceptOp(nodeId = longFromHex(parts[0]), concept = deserializeConcept(parts[1])) } }, ) diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt index c9a1e430f2..30769bf6fa 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt @@ -1,9 +1,7 @@ package org.modelix.model.mpsadapters import jetbrains.mps.ide.ThreadUtils -import jetbrains.mps.project.Project import jetbrains.mps.project.ProjectBase -import jetbrains.mps.project.ProjectManager import jetbrains.mps.project.facets.JavaModuleFacet import jetbrains.mps.project.structure.modules.ModuleReference import jetbrains.mps.smodel.GlobalModelAccess @@ -22,6 +20,7 @@ import org.modelix.model.api.NodeReference import org.modelix.model.area.IArea import org.modelix.model.area.IAreaListener import org.modelix.model.area.IAreaReference +import org.modelix.mps.api.ModelixMpsApi data class MPSArea(val repository: SRepository) : IArea, IAreaReference { @@ -114,7 +113,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference { val inEDT = ThreadUtils.isInEDT() if (inEDT || enforceCommand) { - val projects: Sequence = Sequence { ProjectManager.getInstance().openedProjects.iterator() } + val projects = ModelixMpsApi.getMPSProjects() val modelAccessCandidates = sequenceOf(repository.modelAccess) + projects.map { it.modelAccess } // GlobalModelAccess throws an Exception when trying to execute a command. // Only a ProjectModelAccess can execute a command. @@ -274,8 +273,8 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference { } else { val parts = ref.serialize().substringAfter("${MPSModuleReferenceReference.PREFIX}:").split(MPSModuleReferenceReference.SEPARATOR) MPSModuleReferenceReference( - PersistenceFacade.getInstance().createModuleId(parts[0]), - ChildLinkReferenceByUID(parts[1]), + PersistenceFacade.getInstance().createModuleId(parts[0].urlDecode()), + ChildLinkReferenceByUID(parts[1].urlDecode()), PersistenceFacade.getInstance().createModuleId(parts[2]), ) } @@ -292,7 +291,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference { ref.serialize().substringAfter("${MPSProjectReference.PREFIX}:") } - val project = ProjectManager.getInstance().openedProjects + val project = ModelixMpsApi.getMPSProjects() .filterIsInstance() .find { it.name == projectName } @@ -322,7 +321,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference { NodeReference(projectRef) } - val resolvedNodeForProject = resolveNode(projectRef) ?: return null + val resolvedNodeForProject = resolveNode(projectRef)?.asWritableNode() ?: return null check(resolvedNodeForProject is MPSProjectAsNode) { "Resolved node `$resolvedNodeForProject` does not represent a project." } diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSDevKitDependencyAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSDevKitDependencyAsNode.kt index 21e9941267..e6ca231fbf 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSDevKitDependencyAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSDevKitDependencyAsNode.kt @@ -24,11 +24,19 @@ data class MPSDevKitDependencyAsNode( override fun read(element: MPSDevKitDependencyAsNode): String? { return element.moduleReference.moduleName } + + override fun write(element: MPSDevKitDependencyAsNode, value: String?) { + throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.LanguageDependency.uuid.toReference() to object : IPropertyAccessor { override fun read(element: MPSDevKitDependencyAsNode): String? { return element.moduleReference.moduleId.toString() } + + override fun write(element: MPSDevKitDependencyAsNode, value: String?) { + throw UnsupportedOperationException("read only") + } }, ) } diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSGenericNodeAdapter.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSGenericNodeAdapter.kt index 8d61c54559..04796da6bc 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSGenericNodeAdapter.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSGenericNodeAdapter.kt @@ -2,6 +2,7 @@ package org.modelix.model.mpsadapters import jetbrains.mps.smodel.MPSModuleRepository import org.jetbrains.mps.openapi.language.SConcept +import org.jetbrains.mps.openapi.model.SNodeId import org.jetbrains.mps.openapi.module.SRepository import org.modelix.model.api.ConceptReference import org.modelix.model.api.IChildLinkReference @@ -47,10 +48,18 @@ abstract class MPSGenericNodeAdapter : IWritableNode, ISyncTargetNode { return tryGetReferenceAccessor(role)?.read(getElement()) } + override fun getReferenceTargetRef(role: IReferenceLinkReference): INodeReference? { + return tryGetReferenceAccessor(role)?.readRef(getElement()) + } + override fun getAllReferenceTargets(): List> { return getReferenceAccessors().mapNotNull { it.first to (it.second.read(getElement()) ?: return@mapNotNull null) } } + override fun getAllReferenceTargetRefs(): List> { + return getReferenceAccessors().mapNotNull { it.first to (it.second.readRef(getElement()) ?: return@mapNotNull null) } + } + override fun changeConcept(newConcept: ConceptReference): IWritableNode { throw UnsupportedOperationException("Concept is immutable [node = $this, newConcept = $newConcept]") } @@ -91,6 +100,10 @@ abstract class MPSGenericNodeAdapter : IWritableNode, ISyncTargetNode { } } + override fun isOrdered(role: IChildLinkReference): Boolean { + return tryGetChildAccessor(role)?.isOrdered() != false + } + override fun setReferenceTarget(role: IReferenceLinkReference, target: IWritableNode?) { getReferenceAccessor(role).write(getElement(), target) } @@ -119,40 +132,28 @@ abstract class MPSGenericNodeAdapter : IWritableNode, ISyncTargetNode { return getPropertyAccessors().mapNotNull { it.first to (it.second.read(getElement()) ?: return@mapNotNull null) } } - override fun getReferenceTargetRef(role: IReferenceLinkReference): INodeReference? { - return getReferenceTarget(role)?.getNodeReference() - } - override fun getReferenceLinks(): List { return getReferenceAccessors().map { it.first } } - override fun getAllReferenceTargetRefs(): List> { - return getAllReferenceTargets().map { it.first to it.second.getNodeReference() } - } - interface IPropertyAccessor { fun read(element: E): String? - fun write(element: E, value: String?): Unit = throw UnsupportedOperationException("$this, $value") + fun write(element: E, value: String?) } interface IReferenceAccessor { fun read(element: E): IWritableNode? - fun write(element: E, value: IWritableNode?) { - throw UnsupportedOperationException() - } - fun write(element: E, value: INodeReference?): Unit = throw UnsupportedOperationException() + fun readRef(element: E): INodeReference? = read(element)?.getNodeReference() + fun write(element: E, value: IWritableNode?) + fun write(element: E, value: INodeReference?) } interface IChildAccessor { fun read(element: E): List - fun addNew(element: E, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode { - throw UnsupportedOperationException("$this, $element, $sourceNode") - } - fun move(element: E, index: Int, child: IWritableNode) { - throw UnsupportedOperationException("$this, $element, $child") - } - fun remove(element: E, child: IWritableNode): Unit = throw UnsupportedOperationException() + fun addNew(element: E, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode + fun move(element: E, index: Int, child: IWritableNode): Unit = throw UnsupportedOperationException("unordered") + fun remove(element: E, child: IWritableNode) + fun isOrdered(): Boolean = false } class SpecWithResolvedConcept(val concept: SConcept, val spec: NewNodeSpec?) { @@ -163,3 +164,11 @@ abstract class MPSGenericNodeAdapter : IWritableNode, ISyncTargetNode { } } } + +fun NewNodeSpec.getPreferredSNodeId(): SNodeId? { + // Either use the original SNodeId that it had before it was synchronized to the model server + // or if the node was created outside of MPS, generate an ID based on the ID on the model server. + // The goal is to create a node with the same ID on all clients. + return preferredNodeReference?.let { MPSNodeReference.tryConvert(it) }?.ref?.nodeId + ?: node?.getNodeReference()?.encodeAsForeignId() +} diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt index 2dcbcf9873..b5d3fa64cc 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt @@ -23,6 +23,10 @@ data class MPSJavaModuleFacetAsNode(val facet: JavaModuleFacet) : MPSGenericNode // https://github.com/JetBrains/MPS/blob/2820965ff7b8836ed1d14adaf1bde29744c88147/core/project/source/jetbrains/mps/project/facets/JavaModuleFacetImpl.java return true.toString() } + + override fun write(element: JavaModuleFacet, value: String?) { + if (value != "true") throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.JavaModuleFacet.path.toReference() to object : IPropertyAccessor { override fun read(facet: JavaModuleFacet): String? { diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt index b1a38e2bb7..c62143f901 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt @@ -1,12 +1,16 @@ package org.modelix.model.mpsadapters import jetbrains.mps.extapi.model.SModelDescriptorStub +import jetbrains.mps.extapi.persistence.FileDataSource import jetbrains.mps.project.ModuleId import jetbrains.mps.smodel.ModelImports import jetbrains.mps.smodel.adapter.ids.SLanguageId import jetbrains.mps.smodel.adapter.structure.MetaAdapterFactory import jetbrains.mps.smodel.adapter.structure.language.SLanguageAdapterById +import org.jetbrains.mps.openapi.model.EditableSModel import org.jetbrains.mps.openapi.model.SModel +import org.jetbrains.mps.openapi.model.SModelId +import org.jetbrains.mps.openapi.model.SModelName import org.jetbrains.mps.openapi.module.SModuleId import org.jetbrains.mps.openapi.module.SRepository import org.jetbrains.mps.openapi.persistence.PersistenceFacade @@ -25,12 +29,23 @@ data class MPSModelAsNode(val model: SModel) : MPSGenericNodeAdapter() { private val propertyAccessors = listOf>>( BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name.toReference() to object : IPropertyAccessor { override fun read(element: SModel): String? = element.name.value + override fun write(element: SModel, value: String?) { + require(value != null) { "Model name cannot be null" } + element.rename(value) + } }, BuiltinLanguages.MPSRepositoryConcepts.Model.id.toReference() to object : IPropertyAccessor { override fun read(element: SModel): String? = element.modelId.toString() + override fun write(element: SModel, value: String?) { + throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.Model.stereotype.toReference() to object : IPropertyAccessor { override fun read(element: SModel): String? = element.name.stereotype + override fun write(element: SModel, value: String?) { + val oldName = element.name + element.rename(SModelName(oldName.longName, value).value) + } }, ) private val referenceAccessors = listOf>>() @@ -42,12 +57,17 @@ data class MPSModelAsNode(val model: SModel) : MPSGenericNodeAdapter() { index: Int, sourceNode: SpecWithResolvedConcept, ): IWritableNode { - val nodeId = sourceNode.spec?.preferredNodeReference - ?.let { MPSNodeReference.tryConvert(it) }?.ref?.nodeId - return element.createNode(sourceNode.concept, nodeId) - .also { element.addRootNode(it) } + return element.createNode(sourceNode.concept, sourceNode.spec?.getPreferredSNodeId()) + .also { + it.copyNameFrom(sourceNode.spec) + element.addRootNode(it) + } .let { MPSWritableNode(it) } } + + override fun remove(element: SModel, child: IWritableNode) { + element.removeRootNode((child as MPSWritableNode).node) + } }, BuiltinLanguages.MPSRepositoryConcepts.Model.modelImports.toReference() to object : IChildAccessor { override fun read(element: SModel): List { @@ -75,10 +95,22 @@ data class MPSModelAsNode(val model: SModel) : MPSGenericNodeAdapter() { val moduleName = importedModule.getPropertyValue(BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name.toReference()) ?: "" PersistenceFacade.getInstance().createModuleReference(ModuleId.fromString(moduleId), moduleName) } - val modelRef = PersistenceFacade.getInstance().createModelReference(moduleRef, PersistenceFacade.getInstance().createModelId(modelId), modelName) + val smodelId: SModelId = PersistenceFacade.getInstance().createModelId(modelId) + val modelRef = PersistenceFacade.getInstance().createModelReference( + moduleRef.takeIf { !smodelId.isGloballyUnique }, + smodelId, + modelName, + ) ModelImports(element).addModelImport(modelRef) return MPSModelImportAsNode(modelRef, element) } + + override fun remove( + element: SModel, + child: IWritableNode, + ) { + ModelImports(element).removeModelImport((child as MPSModelImportAsNode).importedModel) + } }, BuiltinLanguages.MPSRepositoryConcepts.Model.usedLanguages.toReference() to object : IChildAccessor { override fun read(element: SModel): List { @@ -192,3 +224,7 @@ data class MPSModelAsNode(val model: SModel) : MPSGenericNodeAdapter() { return null } } + +private fun SModel.rename(newName: String) { + (this as EditableSModel).rename(newName, this.source is FileDataSource) +} diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelImportAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelImportAsNode.kt index 987c40670b..ebde8185f5 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelImportAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelImportAsNode.kt @@ -18,7 +18,22 @@ data class MPSModelImportAsNode(val importedModel: SModelReference, val importin BuiltinLanguages.MPSRepositoryConcepts.ModelReference.model.toReference() to object : IReferenceAccessor { override fun read(element: MPSModelImportAsNode): IWritableNode? { - return MPSModelAsNode(element.importedModel.resolve(element.importingModel.repository)) + // Broken references are a common thing and MPS also just returns null. + // readRef can be used to distinguish a broken reference from one that isn't set. + return element.importedModel.resolve(element.importingModel.repository) + ?.let { MPSModelAsNode(it) } + } + + override fun readRef(element: MPSModelImportAsNode): INodeReference? { + return MPSModelReference(element.importedModel) + } + + override fun write(element: MPSModelImportAsNode, value: IWritableNode?) { + throw UnsupportedOperationException("read only") + } + + override fun write(element: MPSModelImportAsNode, value: INodeReference?) { + throw UnsupportedOperationException("read only") } }, ) diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt index b299efff3f..e7fe3b0300 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt @@ -7,12 +7,14 @@ import jetbrains.mps.project.MPSProject import jetbrains.mps.project.ModuleId import jetbrains.mps.project.Solution import jetbrains.mps.project.facets.JavaModuleFacet +import jetbrains.mps.project.facets.JavaModuleFacetImpl +import jetbrains.mps.project.structure.modules.Dependency import jetbrains.mps.smodel.Generator import jetbrains.mps.smodel.Language -import jetbrains.mps.smodel.MPSModuleRepository import jetbrains.mps.smodel.SModelId import jetbrains.mps.smodel.adapter.ids.SLanguageId import jetbrains.mps.smodel.adapter.structure.MetaAdapterFactory +import jetbrains.mps.workbench.actions.model.DeleteModelHelper import org.jetbrains.mps.openapi.model.SModel import org.jetbrains.mps.openapi.module.FacetsFacade import org.jetbrains.mps.openapi.module.SModule @@ -48,9 +50,24 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { } as MPSModuleAsNode } + internal fun readModuleReference(refNode: IReadableNode): SModuleReference { + val moduleNode = refNode.getReferenceTarget(BuiltinLanguages.MPSRepositoryConcepts.ModuleReference.module.toReference()) ?: run { + val originalRef = requireNotNull(refNode.getReferenceTargetRef(BuiltinLanguages.MPSRepositoryConcepts.ModuleReference.module.toReference())) { + "Reference to module is not set: ${refNode.asLegacyNode().asData().toJson()}" + } + @Suppress("removal") + MPSArea(ModelixMpsApi.getRepository()).resolveNode(originalRef)?.asWritableNode() + } + checkNotNull(moduleNode) + val moduleId = moduleNode.getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.Module.id.toReference())!! + val moduleName = moduleNode.getPropertyValue(BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name.toReference()) ?: "" + return PersistenceFacade.getInstance().createModuleReference(ModuleId.fromString(moduleId), moduleName) + } + private val propertyAccessors = listOf>>( BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name.toReference() to object : IPropertyAccessor { override fun read(element: SModule): String? = element.moduleName + override fun write(element: SModule, value: String?) = TODO() }, BuiltinLanguages.jetbrains_mps_lang_core.BaseConcept.virtualPackage.toReference() to object : IPropertyAccessor { override fun read(element: SModule): String? { @@ -63,17 +80,28 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { }, BuiltinLanguages.MPSRepositoryConcepts.Module.id.toReference() to object : IPropertyAccessor { override fun read(element: SModule): String? = element.moduleId.toString() + override fun write(element: SModule, value: String?) { + throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.Module.moduleVersion.toReference() to object : IPropertyAccessor { override fun read(element: SModule): String? { val version = (element as? AbstractModule)?.moduleDescriptor?.moduleVersion ?: 0 return version.toString() } + override fun write(element: SModule, value: String?) { + (element as? AbstractModule)?.moduleDescriptor?.moduleVersion = value?.toInt() ?: 0 + } }, BuiltinLanguages.MPSRepositoryConcepts.Module.compileInMPS.toReference() to object : IPropertyAccessor { override fun read(element: SModule): String? { return element.getCompileInMPS().toString() } + + override fun write(element: SModule, value: String?) { + if (element.getCompileInMPS().toString() == value) return + (element as Solution).moduleDescriptor.compileInMPS = value.toBoolean() + } }, ) @@ -95,6 +123,17 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { ?.let { PersistenceFacade.getInstance().createModelId(it) } ?: SModelId.generate(), ).let { MPSModelAsNode(it) } } + + override fun move(element: SModule, index: Int, child: IWritableNode) { + throw UnsupportedOperationException() + } + + override fun remove( + element: SModule, + child: IWritableNode, + ) { + DeleteModelHelper.delete(element, (child as MPSModelAsNode).model, true) + } }, BuiltinLanguages.MPSRepositoryConcepts.Module.facets.toReference() to object : IChildAccessor { override fun read(element: SModule): List { @@ -115,10 +154,14 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { BuiltinLanguages.MPSRepositoryConcepts.JavaModuleFacet.getReference() -> { val module = element as AbstractModule val moduleDescriptor = checkNotNull(module.moduleDescriptor) { "Has no moduleDescriptor: $module" } - val newFacet = FacetsFacade.getInstance().getFacetFactory(JavaModuleFacet.FACET_TYPE)!!.create(element) + val newFacet = FacetsFacade.getInstance().getFacetFactory(JavaModuleFacet.FACET_TYPE)!!.create(element) as JavaModuleFacetImpl newFacet.load(MementoImpl()) + val moduleDir = if (element is Generator) element.getGeneratorLocation() else module.moduleSourceDir + if (moduleDir != null) { + newFacet.setGeneratedClassesLocation(moduleDir.findChild(AbstractModule.CLASSES_GEN)) + } moduleDescriptor.addFacetDescriptor(newFacet) - module.setModuleDescriptor(moduleDescriptor) + module.setModuleDescriptor(moduleDescriptor) // notify listeners read(element).filterIsInstance().single() } else -> error("Unsupported facets type: ${sourceNode.getConceptReference()}") @@ -139,9 +182,10 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { val moduleDescriptor = module.moduleDescriptor ?: return emptyList() - return moduleDescriptor.dependencyVersions.map { (ref, version) -> - MPSModuleDependencyAsNode(element, ref) - } + return moduleDescriptor.dependencies.map { it.moduleRef } + .plus(moduleDescriptor.dependencyVersions.map { it.key }) + .distinct() + .map { MPSModuleDependencyAsNode(element, it) } } override fun addNew( @@ -158,8 +202,13 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { "Has no ID: $sourceNode" } val name = sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.ModuleDependency.name.toReference()) ?: "" - val version = sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.ModuleDependency.version.toReference())?.toIntOrNull() ?: 0 val ref = PersistenceFacade.getInstance().createModuleReference(ModuleId.fromString(id), name) + val reexport = sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.ModuleDependency.reexport.toReference()).toBoolean() + val explicit = sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.ModuleDependency.explicit.toReference()).toBoolean() + val version = sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.ModuleDependency.version.toReference())?.toInt() ?: 0 + if (explicit) { + moduleDescriptor.dependencies.add(Dependency(ref, reexport)) + } moduleDescriptor.dependencyVersions[ref] = version MPSModuleDependencyAsNode(element, ref) } @@ -171,6 +220,7 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { val module = element as AbstractModule val moduleDescriptor = checkNotNull(module.moduleDescriptor) { "Has no moduleDescriptor: $module" } val dependency = child as MPSModuleDependencyAsNode + moduleDescriptor.dependencies.removeIf { it.moduleRef == dependency.moduleReference } moduleDescriptor.dependencyVersions.remove(dependency.moduleReference) } }, @@ -181,9 +231,10 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { val moduleDescriptor = module.moduleDescriptor ?: return emptyList() return moduleDescriptor.languageVersions.map { (language, version) -> MPSSingleLanguageDependencyAsNode(language, moduleImporter = module) - } + moduleDescriptor.usedDevkits.map { devKit -> - MPSDevKitDependencyAsNode(devKit, module) } + // moduleDescriptor.usedDevkits is ignored because it is unused in MPS. + // On module level there are only languages, and they are derived from the model dependencies. + // Only models can contain devkit dependencies. } override fun addNew( @@ -203,17 +254,28 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { MPSSingleLanguageDependencyAsNode(lang, moduleImporter = element) } BuiltinLanguages.MPSRepositoryConcepts.DevkitDependency.getReference() -> { - val id = requireNotNull(sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.LanguageDependency.uuid.toReference())) { - "Has no ID: $sourceNode" - } - val name = sourceNode.getNode().getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.LanguageDependency.name.toReference()) ?: "" - val ref = jetbrains.mps.project.structure.modules.ModuleReference(name, ModuleId.fromString(id)) - moduleDescriptor.usedDevkits.add(ref) - MPSDevKitDependencyAsNode(ref, moduleImporter = element) + throw IllegalArgumentException("Modules cannot contain devkit dependencies") } else -> error("Unsupported: ${sourceNode.getConceptReference()}") } } + + override fun remove( + element: SModule, + child: IWritableNode, + ) { + val module = element as AbstractModule + val moduleDescriptor = checkNotNull(module.moduleDescriptor) { "No descriptor: $element" } + when (child) { + is MPSSingleLanguageDependencyAsNode -> { + moduleDescriptor.languageVersions.remove(child.moduleReference) + } + is MPSDevKitDependencyAsNode -> { + moduleDescriptor.usedDevkits.remove(child.moduleReference) + } + else -> throw IllegalArgumentException("Unsupported child type: $child") + } + } }, ) } @@ -268,14 +330,12 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { if (module !is AbstractModule) { return null } - val languageDependencies = module.usedLanguages - languageDependencies?.forEach { entry -> - val sourceModelReference = entry - if (sourceModelReference.sourceModuleReference.moduleId == dependencyId) { - return MPSSingleLanguageDependencyAsNode(sourceModelReference, moduleImporter = module) - } - } - return null + + return childAccessors + .first { it.first == BuiltinLanguages.MPSRepositoryConcepts.Module.languageDependencies.toReference() } + .second.read(module) + .filterIsInstance() + .find { it.moduleReference.sourceModuleReference.moduleId == dependencyId } } internal fun findDevKitDependency(dependencyId: SModuleId): MPSDevKitDependencyAsNode? { @@ -283,12 +343,12 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { if (module !is AbstractModule) { return null } - module.moduleDescriptor?.usedDevkits?.forEach { devKit -> - if (devKit.moduleId == dependencyId) { - return MPSDevKitDependencyAsNode(devKit, module) - } - } - return null + + return childAccessors + .first { it.first == BuiltinLanguages.MPSRepositoryConcepts.Module.languageDependencies.toReference() } + .second.read(module) + .filterIsInstance() + .find { it.moduleReference.moduleId == dependencyId } } } @@ -296,6 +356,21 @@ data class MPSUnknownModuleAsNode(override val module: SModule) : MPSModuleAsNod override fun getConcept(): IConcept = BuiltinLanguages.MPSRepositoryConcepts.Module } data class MPSGeneratorAsNode(override val module: Generator) : MPSModuleAsNode() { + companion object { + private val propertyAccessors = listOf>>( + BuiltinLanguages.MPSRepositoryConcepts.Generator.alias.toReference() to object : IPropertyAccessor { + override fun read(element: Generator): String? = element.moduleDescriptor.alias + override fun write(element: Generator, value: String?) { + element.moduleDescriptor.alias = value + } + }, + ) + } + + override fun getPropertyAccessors(): List>> { + return super.getPropertyAccessors() + propertyAccessors + } + override fun getConcept(): IConcept = BuiltinLanguages.MPSRepositoryConcepts.Generator override fun getContainmentLink(): IChildLinkReference { @@ -320,8 +395,38 @@ data class MPSLanguageAsNode(override val module: Language) : MPSModuleAsNode { + override fun read(element: Language): List { + return element.extendedLanguageRefs.map { + MPSModuleReferenceAsNode( + MPSLanguageAsNode(element), + BuiltinLanguages.MPSRepositoryConcepts.Language.extendedLanguages.toReference(), + it, + ) + } + } + + override fun addNew(element: Language, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode { + val newRef = readModuleReference(sourceNode.getNode()) + element.addExtendedLanguage(newRef) + return MPSModuleReferenceAsNode( + MPSLanguageAsNode(element), + BuiltinLanguages.MPSRepositoryConcepts.Language.extendedLanguages.toReference(), + newRef, + ) + } + + override fun remove(element: Language, child: IWritableNode) { + element.moduleDescriptor.extendedLanguages.remove((child as MPSModuleReferenceAsNode).target) + } }, ) } @@ -335,20 +440,6 @@ data class MPSSolutionAsNode(override val module: Solution) : MPSModuleAsNode() { companion object { - private fun readModuleReference(contextModule: DevKit, refNode: IReadableNode): SModuleReference { - val moduleNode = refNode.getReferenceTarget(BuiltinLanguages.MPSRepositoryConcepts.ModuleReference.module.toReference()) ?: run { - val originalRef = requireNotNull(refNode.getReferenceTargetRef(BuiltinLanguages.MPSRepositoryConcepts.ModuleReference.module.toReference())) { - "Reference to module is not set: ${refNode.asLegacyNode().asData().toJson()}" - } - @Suppress("removal") - MPSArea(contextModule.repository ?: MPSModuleRepository.getInstance()).resolveNode(originalRef)?.asWritableNode() - } - checkNotNull(moduleNode) - val moduleId = moduleNode.getPropertyValue(BuiltinLanguages.MPSRepositoryConcepts.Module.id.toReference())!! - val moduleName = moduleNode.getPropertyValue(BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name.toReference()) ?: "" - return PersistenceFacade.getInstance().createModuleReference(ModuleId.fromString(moduleId), moduleName) - } - private val childAccessors: List>> = MPSModuleAsNode.childAccessors + listOf>>( BuiltinLanguages.MPSRepositoryConcepts.DevKit.exportedLanguages.toReference() to object : IChildAccessor { override fun read(element: DevKit): List { @@ -360,15 +451,15 @@ data class MPSDevkitAsNode(override val module: DevKit) : MPSModuleAsNode { override fun read(element: DevKit): List { @@ -380,15 +471,15 @@ data class MPSDevkitAsNode(override val module: DevKit) : MPSModuleAsNode { override fun read(element: DevKit): List { @@ -405,10 +496,18 @@ data class MPSDevkitAsNode(override val module: DevKit) : MPSModuleAsNode { override fun read(element: MPSModuleDependencyAsNode): String? { @@ -78,6 +82,10 @@ data class MPSModuleDependencyAsNode( override fun read(element: MPSModuleDependencyAsNode): String? { return element.moduleReference.moduleId.toString() } + + override fun write(element: MPSModuleDependencyAsNode, value: String?) { + throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.ModuleDependency.version.toReference() to object : IPropertyAccessor { override fun read(element: MPSModuleDependencyAsNode): String? { diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleReferenceAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleReferenceAsNode.kt index e8f2f2a35c..7d944b4ebe 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleReferenceAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleReferenceAsNode.kt @@ -1,6 +1,5 @@ package org.modelix.model.mpsadapters -import jetbrains.mps.smodel.MPSModuleRepository import org.jetbrains.mps.openapi.module.SModuleId import org.jetbrains.mps.openapi.module.SModuleReference import org.jetbrains.mps.openapi.module.SRepository @@ -12,6 +11,10 @@ import org.modelix.model.api.INodeReference import org.modelix.model.api.IPropertyReference import org.modelix.model.api.IReferenceLinkReference import org.modelix.model.api.IWritableNode +import org.modelix.mps.api.ModelixMpsApi +import java.net.URLDecoder +import java.net.URLEncoder +import java.nio.charset.StandardCharsets data class MPSModuleReferenceAsNode( private val parent: MPSModuleAsNode<*>, @@ -34,9 +37,17 @@ data class MPSModuleReferenceAsNode( return listOf( BuiltinLanguages.MPSRepositoryConcepts.ModuleReference.module.toReference() to object : IReferenceAccessor { override fun read(element: MPSModuleReferenceAsNode): IWritableNode? { - val repo = parent.getRepository() ?: MPSModuleRepository.getInstance() + val repo = ModelixMpsApi.getRepository() return target.resolve(repo)?.let { MPSModuleAsNode(it) } } + + override fun write(element: MPSModuleReferenceAsNode, value: IWritableNode?) { + throw UnsupportedOperationException("read only") + } + + override fun write(element: MPSModuleReferenceAsNode, value: INodeReference?) { + throw UnsupportedOperationException("read only") + } }, ) } @@ -69,6 +80,9 @@ data class MPSModuleReferenceReference(val parent: SModuleId, val link: ChildLin } override fun serialize(): String { - return "$PREFIX:$parent$SEPARATOR${link.getUID()}$SEPARATOR$target" + return "$PREFIX:${parent.toString().urlEncode()}$SEPARATOR${link.getUID().urlEncode()}$SEPARATOR$target" } } + +internal fun String.urlEncode() = URLEncoder.encode(this, StandardCharsets.UTF_8) +internal fun String.urlDecode() = URLDecoder.decode(this, StandardCharsets.UTF_8) diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt index cf67f72b7f..06fcad565f 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt @@ -17,6 +17,10 @@ data class MPSProjectAsNode(val project: ProjectBase) : MPSGenericNodeAdapter>> = listOf( @@ -24,11 +28,27 @@ data class MPSProjectAsNode(val project: ProjectBase) : MPSGenericNodeAdapter { return element.projectModules.map { MPSProjectModuleAsNode(element, it) } } + + override fun addNew(element: ProjectBase, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode { + return TODO() + } + + override fun remove(element: ProjectBase, child: IWritableNode) { + element.removeModule((child as MPSProjectModuleAsNode).module) + } }, BuiltinLanguages.MPSRepositoryConcepts.Project.modules.toReference() to object : IChildAccessor { override fun read(element: ProjectBase): List { return return emptyList() // modules child link is deprecated } + + override fun addNew(element: ProjectBase, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode { + throw UnsupportedOperationException("read only") + } + + override fun remove(element: ProjectBase, child: IWritableNode) { + throw UnsupportedOperationException("read only") + } }, ) } diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectModuleAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectModuleAsNode.kt index f473b8785e..2018f51543 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectModuleAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectModuleAsNode.kt @@ -30,6 +30,14 @@ data class MPSProjectModuleAsNode(val project: ProjectBase, val module: SModule) override fun read(element: MPSProjectModuleAsNode): IWritableNode? { return MPSModuleAsNode(element.module) } + + override fun write(element: MPSProjectModuleAsNode, value: INodeReference?) { + throw UnsupportedOperationException("read only") + } + + override fun write(element: MPSProjectModuleAsNode, value: IWritableNode?) { + throw UnsupportedOperationException("read only") + } }, ) } diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferences.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferences.kt index e17e3532c5..962f1a2320 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferences.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSReferences.kt @@ -112,6 +112,7 @@ data class MPSModuleDependencyReference( } } +// FIXME projectName is not guaranteed to be unique and not suitable to identify a project data class MPSProjectReference(val projectName: String) : INodeReference { companion object { diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSRepositoryAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSRepositoryAsNode.kt index 452c14c903..eb16b45972 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSRepositoryAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSRepositoryAsNode.kt @@ -1,11 +1,16 @@ package org.modelix.model.mpsadapters +import jetbrains.mps.module.ModuleDeleteHelper +import jetbrains.mps.project.AbstractModule import jetbrains.mps.project.MPSProject import jetbrains.mps.project.ModuleId +import jetbrains.mps.project.Project import jetbrains.mps.project.ProjectBase import jetbrains.mps.smodel.Generator +import jetbrains.mps.smodel.Language import jetbrains.mps.smodel.tempmodel.TempModule import jetbrains.mps.smodel.tempmodel.TempModule2 +import org.jetbrains.mps.openapi.model.EditableSModel import org.jetbrains.mps.openapi.module.SModule import org.jetbrains.mps.openapi.module.SRepository import org.modelix.model.api.BuiltinLanguages @@ -62,17 +67,37 @@ data class MPSRepositoryAsNode(@get:JvmName("getRepository_") val repository: SR else -> throw UnsupportedOperationException("Module type not supported yet: ${sourceNode.getConceptReference()}") } } + + override fun remove(element: SRepository, child: IWritableNode) { + (child as MPSModuleAsNode<*>).module.delete() + } }, BuiltinLanguages.MPSRepositoryConcepts.Repository.tempModules.toReference() to object : IChildAccessor { override fun read(element: SRepository): List { return element.modules.filter { it.isTempModule() }.map { MPSModuleAsNode(it) } } + + override fun addNew(element: SRepository, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode { + throw UnsupportedOperationException("read only") + } + + override fun remove(element: SRepository, child: IWritableNode) { + throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.Repository.projects.toReference() to object : IChildAccessor { override fun read(element: SRepository): List { return ModelixMpsApi.getMPSProjects() .map { MPSProjectAsNode(it as ProjectBase) } } + + override fun addNew(element: SRepository, index: Int, sourceNode: SpecWithResolvedConcept): IWritableNode { + throw UnsupportedOperationException("read only") + } + + override fun remove(element: SRepository, child: IWritableNode) { + throw UnsupportedOperationException("read only") + } }, ) } @@ -97,3 +122,26 @@ data class MPSRepositoryAsNode(@get:JvmName("getRepository_") val repository: SR } private fun SModule.isTempModule(): Boolean = this is TempModule || this is TempModule2 + +internal fun SModule.delete() { + // Without saving first, MPS might detect a conflict that can result in data loss and prevents it. + saveModuleAndModels() + if (this is Generator) { + val language = this.sourceLanguage().sourceModule as? Language + if (language != null) { + language.saveModuleAndModels() + language.generators.forEach { it.saveModuleAndModels() } + } + } + + ModuleDeleteHelper(ModelixMpsApi.getMPSProject() as Project).deleteModules( + listOf(this), + false, + true, + ) +} + +private fun SModule.saveModuleAndModels() { + (this as? AbstractModule)?.save() + models.filterIsInstance().forEach { it.save() } +} diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSSingleLanguageDependencyAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSSingleLanguageDependencyAsNode.kt index 841c76ac86..e8cd53f673 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSSingleLanguageDependencyAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSSingleLanguageDependencyAsNode.kt @@ -55,11 +55,19 @@ data class MPSSingleLanguageDependencyAsNode( override fun read(element: MPSSingleLanguageDependencyAsNode): String? { return element.moduleReference.qualifiedName } + + override fun write(element: MPSSingleLanguageDependencyAsNode, value: String?) { + throw UnsupportedOperationException("read only") + } }, BuiltinLanguages.MPSRepositoryConcepts.LanguageDependency.uuid.toReference() to object : IPropertyAccessor { override fun read(element: MPSSingleLanguageDependencyAsNode): String? { return element.moduleReference.sourceModuleReference.moduleId.toString() } + + override fun write(element: MPSSingleLanguageDependencyAsNode, value: String?) { + throw UnsupportedOperationException("read only") + } }, ) diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt index baeafe6efe..4672c9d7d0 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt @@ -1,7 +1,10 @@ package org.modelix.model.mpsadapters import jetbrains.mps.smodel.MPSModuleRepository +import jetbrains.mps.smodel.SNodeId +import jetbrains.mps.smodel.SNodeUtil import jetbrains.mps.smodel.adapter.MetaAdapterByDeclaration +import org.apache.commons.codec.binary.Hex import org.jetbrains.mps.openapi.language.SConcept import org.jetbrains.mps.openapi.language.SContainmentLink import org.jetbrains.mps.openapi.language.SProperty @@ -16,15 +19,20 @@ import org.modelix.model.api.IConcept import org.modelix.model.api.IMutableModel import org.modelix.model.api.INodeReference import org.modelix.model.api.IPropertyReference +import org.modelix.model.api.IReadableNode import org.modelix.model.api.IReferenceLinkReference import org.modelix.model.api.IRoleReferenceByName import org.modelix.model.api.IRoleReferenceByUID import org.modelix.model.api.ISyncTargetNode import org.modelix.model.api.IWritableNode import org.modelix.model.api.NewNodeSpec +import org.modelix.model.api.NodeReference import org.modelix.model.api.meta.NullConcept import org.modelix.mps.api.ModelixMpsApi +fun SNode.asReadableNode(): IReadableNode = MPSWritableNode(this) +fun SNode.asWritableNode(): IWritableNode = MPSWritableNode(this) + data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { override fun getModel(): IMutableModel { return MPSArea(node.model?.repository ?: MPSModuleRepository.getInstance()).asModel() @@ -67,7 +75,7 @@ data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { override fun getParent(): IWritableNode? { DependencyTracking.accessed((MPSContainmentDependency(node))) - return node.parent?.let { MPSWritableNode(it) } + return node.parent?.let { MPSWritableNode(it) } ?: node.model?.let { MPSModelAsNode(it) } } override fun changeConcept(newConcept: ConceptReference): IWritableNode { @@ -108,10 +116,6 @@ data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { return MPSWritableNode(newNode) } - override fun setPropertyValue(property: IPropertyReference, value: String?) { - node.setProperty(resolve(property), value) - } - override fun moveChild(role: IChildLinkReference, index: Int, child: IWritableNode) { require(child is MPSWritableNode) val sChild = child.node @@ -151,7 +155,7 @@ data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { } override fun syncNewChildren(role: IChildLinkReference, index: Int, specs: List): List { - val repo = node.model?.repository ?: MPSModuleRepository.getInstance() + val repo = node.model?.repository ?: ModelixMpsApi.getRepository() val resolvedConcepts = specs.distinct().associate { spec -> spec.conceptRef to repo.resolveConcept(spec.conceptRef) } @@ -165,7 +169,12 @@ data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { return specs.map { spec -> val resolvedConcept = checkNotNull(resolvedConcepts[spec.conceptRef]) - val preferredId = spec.preferredNodeReference?.let { MPSNodeReference.tryConvert(it) }?.ref?.nodeId + + // Either use the original SNodeId that it had before it was synchronized to the model server + // or if the node was created outside of MPS, generate an ID based on the ID on the model server. + // The goal is to create a node with the same ID on all clients. + val preferredId = spec.getPreferredSNodeId() + val newChild = if (model == null) { if (preferredId == null) { jetbrains.mps.smodel.SNode(resolvedConcept) @@ -176,6 +185,8 @@ data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { model.createNode(resolvedConcept, preferredId) } + newChild.copyNameFrom(spec) + if (anchor == null) { node.addChild(link, newChild) } else { @@ -222,11 +233,26 @@ data class MPSWritableNode(val node: SNode) : IWritableNode, ISyncTargetNode { } override fun getPropertyValue(property: IPropertyReference): String? { +// if (property.matches(NodeData.ID_PROPERTY_REF)) { +// // No dependency tracking for read only property necessary +// return node.nodeId.tryDecodeModelixReference()?.serialize() +// } + DependencyTracking.accessed(MPSProperty.tryFromReference(property)?.let { MPSPropertyDependency(node, it.property) } ?: MPSAllReferencesDependency(node)) val mpsProperty = node.properties.firstOrNull { MPSProperty(it).toReference().matches(property) } ?: return null return node.getProperty(mpsProperty) } + override fun setPropertyValue(property: IPropertyReference, value: String?) { +// if (property.matches(NodeData.ID_PROPERTY_REF)) { +// require(node.nodeId.tryDecodeModelixReference()?.serialize() == value) { +// "Property is read only: $property" +// } +// return +// } + node.setProperty(resolve(property), value) + } + override fun getPropertyLinks(): List { DependencyTracking.accessed(MPSAllPropertiesDependency(node)) return node.properties.map { MPSProperty(it).toReference() } @@ -296,3 +322,26 @@ fun SRepository.resolveConcept(concept: ConceptReference): SConcept { "MPS concept not found: $concept" } } + +fun org.jetbrains.mps.openapi.model.SNodeId.tryDecodeModelixReference(): NodeReference? { + if (this !is SNodeId.Foreign) return null + if (id.length < 2 || id.substring(0, 2) != "mx") return null + val hex = id.substring(2) + return NodeReference(String(Hex.decodeHex(hex))) +} + +fun INodeReference.encodeAsForeignId(): SNodeId { + return SNodeId.Foreign.fromIdNoPrefix("mx" + Hex.encodeHexString(serialize().toByteArray())) +} + +/** + * When a reference is set, the name of the target is stored as resolveInfo. + * If the name isn't yet set, the resolveInfo will be empty. + * That's why we set the name as early as possible. + * And because resolveInfo is something MPS specific it is handled here instead of in ModelSynchronizer. + */ +internal fun SNode.copyNameFrom(spec: NewNodeSpec?) { + if (spec == null) return + val name = spec.node?.getPropertyValue(MPSProperty(SNodeUtil.property_INamedConcept_name).toReference()) + if (name != null) setProperty(SNodeUtil.property_INamedConcept_name, name) +} diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/ModelPersistenceWithFixedId.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/ModelPersistenceWithFixedId.kt index 1cb9271b58..259d18e14b 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/ModelPersistenceWithFixedId.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/ModelPersistenceWithFixedId.kt @@ -42,9 +42,11 @@ open class ModelPersistenceWithFixedId(val moduleRef: SModuleReference, val mode throw UnsupportedDataSourceException(dataSource) } val header = SModelHeader.create(ModelPersistence.LAST_VERSION) - // We could provide a module ID to createModelReference, but MPS also doesn't provide one when creating a model. - val modelReference: SModelReference = - PersistenceFacade.getInstance().createModelReference(null, modelId, modelName.value) + val modelReference: SModelReference = PersistenceFacade.getInstance().createModelReference( + moduleRef.takeIf { !modelId.isGloballyUnique }, + modelId, + modelName.value, + ) header.modelReference = modelReference val rv = DefaultSModelDescriptor(ModelPersistenceFacility(this, dataSource as StreamDataSource), header) if (dataSource.getTimestamp() != -1L) { diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt index 2d6ed17150..1a18722f73 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt @@ -1,5 +1,6 @@ package org.modelix.model.mpsadapters +import jetbrains.mps.extapi.persistence.FileBasedModelRoot import jetbrains.mps.ide.project.ProjectHelper import jetbrains.mps.persistence.DefaultModelRoot import jetbrains.mps.project.DevKit @@ -36,6 +37,7 @@ class SolutionProducer(private val myProject: MPSProject) { private fun createSolutionDescriptor(namespace: String, id: ModuleId, descriptorFile: IFile): SolutionDescriptor { val descriptor = SolutionDescriptor() + descriptor.outputRoot = "\${module}/source_gen" descriptor.namespace = namespace descriptor.id = id val moduleLocation = descriptorFile.parent @@ -72,6 +74,7 @@ class LanguageProducer(private val myProject: MPSProject) { private fun createDescriptor(namespace: String, id: ModuleId, descriptorFile: IFile): LanguageDescriptor { val descriptor = LanguageDescriptor() + descriptor.outputRoot = "\${module}/source_gen" descriptor.namespace = namespace descriptor.id = id val moduleLocation = descriptorFile.parent @@ -89,30 +92,45 @@ class LanguageProducer(private val myProject: MPSProject) { class GeneratorProducer(private val myProject: MPSProject) { - fun create(language: Language, name: String, id: ModuleId): Generator { + fun create(language: Language, name: String, id: ModuleId, alias: String?): Generator { val basePath = checkNotNull(ProjectHelper.toIdeaProject(myProject).getBasePath()) { "Project has no base path: $myProject" } val projectBaseDir = myProject.fileSystem.getFile(basePath) val solutionBaseDir = projectBaseDir.findChild("languages").findChild(language.moduleName!!) - return create(language, name, id, solutionBaseDir) + return create(language, name, id, alias, solutionBaseDir) } - fun create(language: Language, namespace: String, id: ModuleId, languageModuleDir: IFile): Generator { - val generatorLocation: IFile = languageModuleDir.findChild("generator") + fun create(language: Language, namespace: String, id: ModuleId, alias: String?, languageModuleDir: IFile): Generator { + val siblingDirs = language.generators.mapNotNull { it.getGeneratorLocation() }.toSet() + val generatorLocation: IFile = findEmptyGeneratorDir(languageModuleDir, siblingDirs) generatorLocation.mkdirs() - val descriptor = createDescriptor(namespace, id, generatorLocation, null) + val descriptor = createDescriptor(namespace, id, alias, generatorLocation, null) descriptor.sourceLanguage = language.moduleReference language.moduleDescriptor.generators.add(descriptor) language.setModuleDescriptor(language.moduleDescriptor) // instantiate generator module + language.save() + return language.generators.first { it.moduleReference.moduleId == id } } - private fun createDescriptor(namespace: String, id: ModuleId, generatorModuleLocation: IFile, templateModelsLocation: IFile?): GeneratorDescriptor { + private fun findEmptyGeneratorDir(languageModuleDir: IFile, siblingDirs: Set): IFile { + var folderName = "generator" + var cnt = 1 + var newChild: IFile? + do { + newChild = languageModuleDir.findChild(folderName) + folderName = "generator" + cnt++ + } while (siblingDirs.contains(newChild) || newChild.exists() && (!newChild.isDirectory || !newChild.children!!.isEmpty())) + return newChild + } + + private fun createDescriptor(namespace: String, id: ModuleId, alias: String?, generatorModuleLocation: IFile, templateModelsLocation: IFile?): GeneratorDescriptor { val descriptor = GeneratorDescriptor() + descriptor.outputRoot = "\${module}/${generatorModuleLocation.name}/source_gen" descriptor.namespace = namespace descriptor.id = id - descriptor.alias = "main" + descriptor.alias = alias ?: "main" val modelRoot = if (templateModelsLocation == null) { DefaultModelRoot.createDescriptor(generatorModuleLocation, generatorModuleLocation.findChild("templates")) } else { @@ -123,6 +141,10 @@ class GeneratorProducer(private val myProject: MPSProject) { } } +fun Generator.getGeneratorLocation(): IFile? { + return modelRoots.filterIsInstance().firstNotNullOfOrNull { it.contentDirectory } +} + class DevkitProducer(private val myProject: MPSProject) { fun create(name: String, id: ModuleId): DevKit { diff --git a/mps-sync-plugin3/build.gradle.kts b/mps-sync-plugin3/build.gradle.kts new file mode 100644 index 0000000000..8443fe7d83 --- /dev/null +++ b/mps-sync-plugin3/build.gradle.kts @@ -0,0 +1,93 @@ +import org.modelix.copyMps +import org.modelix.mpsHomeDir +import org.modelix.mpsMajorVersion + +plugins { + `modelix-kotlin-jvm` + alias(libs.plugins.intellij) + `modelix-project-repositories` +} + +intellij { + localPath = copyMps().absolutePath + instrumentCode = false +} + +dependencies { + implementation(project(":bulk-model-sync-lib")) + implementation(project(":bulk-model-sync-mps")) + implementation(project(":mps-model-adapters")) + implementation(project(":model-client")) + implementation(libs.modelix.mpsApi) + implementation(libs.kotlin.logging) + + compileOnly( + fileTree(mpsHomeDir).matching { + include("lib/**/*.jar") + }, + ) + + // testImplementation("junit:junit:4.13.2") + testImplementation(libs.testcontainers) + testImplementation(libs.kotlin.coroutines.test) + testImplementation(libs.logback.classic) +} + +tasks { + patchPluginXml { + sinceBuild.set("211") + untilBuild.set("241.*") + } + + buildSearchableOptions { + enabled = false + } + + runIde { + autoReloadPlugins.set(true) + } + + test { + dependsOn(":model-server:assemble") + onlyIf { + !setOf( + "2020.3", // incompatible with the intellij plugin + "2021.2", // hangs when executed on CI + "2021.3", // hangs when executed on CI + "2022.2", // hangs when executed on CI + ).contains(mpsMajorVersion) + } + jvmArgs("-Dintellij.platform.load.app.info.from.resources=true") + jvmArgs("-Xmx1000m") + + val arch = System.getProperty("os.arch") + val jnaDir = mpsHomeDir.get().asFile.resolve("lib/jna/$arch") + if (jnaDir.exists()) { + jvmArgs("-Djna.boot.library.path=${jnaDir.absolutePath}") + jvmArgs("-Djna.noclasspath=true") + jvmArgs("-Djna.nosys=true") + } + } + + val mpsPluginDir = project.findProperty("mps.plugins.dir")?.toString()?.let { file(it) } + if (mpsPluginDir != null && mpsPluginDir.isDirectory) { + create("installMpsPlugin") { + dependsOn(prepareSandbox) + from(project.layout.buildDirectory.dir("idea-sandbox/plugins/mps-model-adapters-plugin")) + into(mpsPluginDir.resolve("mps-model-adapters-plugin")) + } + } +} + +group = "org.modelix.mps" + +publishing { + publications { + create("maven") { + artifactId = "mps-sync-plugin3" + artifact(tasks.buildPlugin) { + extension = "zip" + } + } + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt new file mode 100644 index 0000000000..8ad66ba2d3 --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt @@ -0,0 +1,60 @@ +package org.modelix.mps.sync3 + +import com.intellij.openapi.components.service +import jetbrains.mps.ide.project.ProjectHelper +import kotlinx.coroutines.runBlocking +import org.modelix.model.IVersion +import org.modelix.model.lazy.BranchReference +import java.io.Closeable + +interface IModelSyncService { + companion object { + @JvmStatic + fun getInstance(project: com.intellij.openapi.project.Project): IModelSyncService { + return project.service() + } + + @JvmStatic + fun getInstance(project: org.jetbrains.mps.openapi.project.Project): IModelSyncService { + return getInstance(ProjectHelper.toIdeaProject(project as jetbrains.mps.project.Project)) + } + } + + fun addServer(url: String): IServerConnection + fun getServerConnections(): List +} + +interface IServerConnection { + fun activate() + fun deactivate() + fun remove() + fun getStatus(): Status + + suspend fun pullVersion(branchRef: BranchReference): IVersion + + fun bind(branchRef: BranchReference): IBinding = bind(branchRef, null) + fun bind(branchRef: BranchReference, lastSyncedVersionHash: String?): IBinding + fun getBindings(): List + + enum class Status { + CONNECTED, + DISCONNECTED, + } +} + +interface IBinding : Closeable { + val mpsProject: org.jetbrains.mps.openapi.project.Project + val branchRef: BranchReference + fun activate() + fun deactivate() + + override fun close() = deactivate() + + /** + * Blocks until both ends are in sync. + * @exception Throwable if the last synchronization failed + * @return the latest version + */ + suspend fun flush(): IVersion + fun flushBlocking() = runBlocking { flush() } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt new file mode 100644 index 0000000000..aae11a1611 --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt @@ -0,0 +1,218 @@ +package org.modelix.mps.sync3 + +import jetbrains.mps.smodel.SModelInternal +import jetbrains.mps.smodel.event.SModelChildEvent +import jetbrains.mps.smodel.event.SModelDevKitEvent +import jetbrains.mps.smodel.event.SModelImportEvent +import jetbrains.mps.smodel.event.SModelLanguageEvent +import jetbrains.mps.smodel.event.SModelListener +import jetbrains.mps.smodel.event.SModelPropertyEvent +import jetbrains.mps.smodel.event.SModelReferenceEvent +import jetbrains.mps.smodel.event.SModelRenamedEvent +import jetbrains.mps.smodel.event.SModelRootEvent +import jetbrains.mps.smodel.loading.ModelLoadingState +import org.jetbrains.mps.openapi.event.SNodeAddEvent +import org.jetbrains.mps.openapi.event.SNodeRemoveEvent +import org.jetbrains.mps.openapi.event.SPropertyChangeEvent +import org.jetbrains.mps.openapi.event.SReferenceChangeEvent +import org.jetbrains.mps.openapi.language.SLanguage +import org.jetbrains.mps.openapi.model.SModel +import org.jetbrains.mps.openapi.model.SModelReference +import org.jetbrains.mps.openapi.model.SNode +import org.jetbrains.mps.openapi.model.SNodeChangeListener +import org.jetbrains.mps.openapi.module.SDependency +import org.jetbrains.mps.openapi.module.SModule +import org.jetbrains.mps.openapi.module.SModuleListener +import org.jetbrains.mps.openapi.module.SModuleReference +import org.jetbrains.mps.openapi.module.SRepository +import org.jetbrains.mps.openapi.module.SRepositoryListener +import org.modelix.model.api.IReadableNode +import org.modelix.model.api.toSerialized +import org.modelix.model.mpsadapters.GlobalModelListener +import org.modelix.model.mpsadapters.MPSModelAsNode +import org.modelix.model.mpsadapters.MPSModuleAsNode +import org.modelix.model.mpsadapters.MPSRepositoryAsNode +import org.modelix.model.mpsadapters.asReadableNode +import org.modelix.model.sync.bulk.DefaultInvalidationTree +import java.util.concurrent.atomic.AtomicBoolean + +private val LOG = mu.KotlinLogging.logger { } + +abstract class MPSInvalidatingListener(val repository: SRepository) : + GlobalModelListener(), + SNodeChangeListener, + SModuleListener, + SRepositoryListener, + SModelListener, + org.jetbrains.mps.openapi.model.SModelListener { + + private val syncActive = AtomicBoolean(false) + private val invalidationTree: DefaultInvalidationTree = + DefaultInvalidationTree(MPSRepositoryAsNode(repository).getNodeReference().toSerialized()) + + fun hasAnyInvalidations() = synchronized(invalidationTree) { invalidationTree.hasAnyInvalidations() } + + fun runSync(body: (DefaultInvalidationTree) -> R): R { + check(!syncActive.getAndSet(true)) { "Synchronization is already running" } + try { + synchronized(invalidationTree) { + return body(invalidationTree).also { + LOG.trace { "Resetting invalidations" } + invalidationTree.reset() + } + } + } catch (ex: Throwable) { + LOG.error(ex) { "Sync from MPS failed" } + throw ex + } finally { + syncActive.set(false) + } + } + + abstract fun onInvalidation() + + private fun invalidate(node: IReadableNode, includingDescendants: Boolean = false) { + if (syncActive.get()) return + synchronized(invalidationTree) { + LOG.trace { "Invalidating ${node.getNodeReference()}" } + invalidationTree.invalidate(node, includingDescendants) + } + onInvalidation() + } + + private fun invalidate(node: SNode) { + invalidate(node.asReadableNode()) + } + + private fun invalidate(model: SModel) { + invalidate(MPSModelAsNode(model)) + } + + private fun invalidate(module: SModule) { + invalidate(MPSModuleAsNode(module)) + } + + private fun invalidate(repository: SRepository) { + invalidate(MPSRepositoryAsNode(repository)) + } + + override fun addListener(model: SModel) { + model.addChangeListener(this) + model.addModelListener(this) + (model as SModelInternal).addModelListener(this) + } + + override fun removeListener(model: SModel) { + model.removeChangeListener(this) + model.removeModelListener(this) + (model as SModelInternal).removeModelListener(this) + } + + override fun addListener(module: SModule) { + module.addModuleListener(this) + } + + override fun removeListener(module: SModule) { + module.removeModuleListener(this) + } + + override fun addListener(repository: SRepository) { + repository.addRepositoryListener(this) + } + + override fun removeListener(repository: SRepository) { + repository.removeRepositoryListener(this) + } + + override fun propertyChanged(e: SPropertyChangeEvent) { + invalidate(e.node) + } + + override fun referenceChanged(e: SReferenceChangeEvent) { + invalidate(e.node) + } + + override fun nodeAdded(e: SNodeAddEvent) { + val parent = e.parent + if (parent != null) { + invalidate(parent) + } else { + invalidate(e.model) + } + } + + override fun nodeRemoved(e: SNodeRemoveEvent) { + val parent = e.parent + if (parent != null) { + invalidate(parent) + } else { + invalidate(e.model) + } + } + + override fun beforeChildRemoved(event: SModelChildEvent) {} + override fun beforeModelDisposed(model: SModel) {} + override fun beforeModelRenamed(event: SModelRenamedEvent) {} + override fun beforeRootRemoved(event: SModelRootEvent) {} + override fun childAdded(event: SModelChildEvent) { invalidate(event.parent) } + override fun childRemoved(event: SModelChildEvent) { invalidate(event.parent) } + override fun devkitAdded(event: SModelDevKitEvent) { invalidate(event.model) } + override fun devkitRemoved(event: SModelDevKitEvent) { invalidate(event.model) } + override fun getPriority(): SModelListener.SModelListenerPriority { + return SModelListener.SModelListenerPriority.CLIENT + } + + override fun importAdded(event: SModelImportEvent) { invalidate(event.model) } + override fun importRemoved(event: SModelImportEvent) { invalidate(event.model) } + override fun languageAdded(event: SModelLanguageEvent) { invalidate(event.model) } + override fun languageRemoved(event: SModelLanguageEvent) { invalidate(event.model) } + override fun modelLoadingStateChanged(model: SModel, state: ModelLoadingState) {} + override fun modelRenamed(event: SModelRenamedEvent) { invalidate(event.model) } + override fun modelSaved(model: SModel) {} + override fun propertyChanged(event: SModelPropertyEvent) { invalidate(event.node) } + override fun referenceAdded(event: SModelReferenceEvent) { invalidate(event.reference.sourceNode) } + override fun referenceRemoved(event: SModelReferenceEvent) { invalidate(event.reference.sourceNode) } + + @Deprecated("") + override fun rootAdded(event: SModelRootEvent) { + } + + @Deprecated("") + override fun rootRemoved(event: SModelRootEvent) { + } + + override fun modelLoaded(model: SModel, partially: Boolean) {} + override fun modelReplaced(model: SModel) { + invalidate(MPSModelAsNode(model), includingDescendants = true) + } + + override fun modelUnloaded(model: SModel) {} + override fun modelAttached(model: SModel, repository: SRepository) {} + override fun modelDetached(model: SModel, repository: SRepository) {} + override fun conflictDetected(model: SModel) {} + override fun problemsDetected(model: SModel, problems: Iterable) {} + override fun modelAdded(module: SModule, model: SModel) { + invalidate(module) + } + + override fun beforeModelRemoved(module: SModule, model: SModel) {} + + override fun modelRemoved(module: SModule, reference: SModelReference) { + invalidate(module) + } + override fun beforeModelRenamed(module: SModule, model: SModel, reference: SModelReference) {} + override fun modelRenamed(module: SModule, model: SModel, reference: SModelReference) { + invalidate(model) + } + + override fun dependencyAdded(module: SModule, dependency: SDependency) { invalidate(module) } + override fun dependencyRemoved(module: SModule, dependency: SDependency) { invalidate(module) } + override fun languageAdded(module: SModule, language: SLanguage) { invalidate(module) } + override fun languageRemoved(module: SModule, language: SLanguage) { invalidate(module) } + override fun moduleChanged(module: SModule) { invalidate(module) } + override fun moduleAdded(module: SModule) { invalidate(repository) } + override fun beforeModuleRemoved(module: SModule) {} + override fun moduleRemoved(reference: SModuleReference) { invalidate(repository) } + override fun commandStarted(repository: SRepository) {} + override fun commandFinished(repository: SRepository) {} +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt new file mode 100644 index 0000000000..e8d1f9b2c3 --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt @@ -0,0 +1,499 @@ +package org.modelix.mps.sync3 + +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.EDT +import com.intellij.openapi.components.Service +import com.intellij.openapi.components.service +import com.intellij.openapi.project.Project +import jetbrains.mps.ide.project.ProjectHelper +import jetbrains.mps.project.MPSProject +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import org.jetbrains.mps.openapi.module.SRepository +import org.modelix.model.IVersion +import org.modelix.model.api.TreePointer +import org.modelix.model.api.getRootNode +import org.modelix.model.client2.IModelClientV2 +import org.modelix.model.client2.ModelClientV2 +import org.modelix.model.client2.runWrite +import org.modelix.model.lazy.BranchReference +import org.modelix.model.mpsadapters.MPSRepositoryAsNode +import org.modelix.model.sync.bulk.FullSyncFilter +import org.modelix.model.sync.bulk.InvalidatingVisitor +import org.modelix.model.sync.bulk.InvalidationTree +import org.modelix.model.sync.bulk.ModelSynchronizer +import org.modelix.model.sync.bulk.ModelSynchronizer.IIncrementalUpdateInformation +import org.modelix.model.sync.bulk.NodeAssociationFromModelServer +import org.modelix.model.sync.bulk.NodeAssociationToModelServer +import org.modelix.mps.api.ModelixMpsApi +import org.modelix.mps.model.sync.bulk.MPSProjectSyncMask +import org.modelix.mps.sync3.Binding.Companion.LOG +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +@Service(Service.Level.APP) +class AppLevelModelSyncService() : Disposable { + + companion object { + fun getInstance(): AppLevelModelSyncService { + return ApplicationManager.getApplication().service() + } + } + + private val connections = LinkedHashMap() + private val coroutinesScope = CoroutineScope(Dispatchers.Default) + private val connectionCheckingJob = coroutinesScope.launchLoop( + BackoffStrategy( + initialDelay = 3.seconds, + maxDelay = 10.seconds, + factor = 1.2, + ), + ) { + for (connection in synchronized(connections) { connections.values.toList() }) { + connection.checkConnection() + } + } + + @Synchronized + fun addConnection(url: String): ServerConnection { + return synchronized(connections) { connections.getOrPut(url) { ServerConnection(url) } } + } + + override fun dispose() { + coroutinesScope.cancel("disposed") + } + + class ServerConnection(val url: String) { + private var client: ValueWithMutex = ValueWithMutex(null) + private var connected: Boolean = false + + suspend fun getClient(): IModelClientV2 { + return client.getValue() ?: client.updateValue { + it ?: ModelClientV2.builder().url(url).build().also { it.init() } + } + } + + suspend fun checkConnection() { + try { + getClient().getServerId() + connected = true + } catch (ex: Throwable) { + connected = false + } + } + } +} + +@Service(Service.Level.PROJECT) +class ModelSyncService(val project: Project) : IModelSyncService, Disposable { + private val mpsProject: MPSProject get() = ProjectHelper.fromIdeaProjectOrFail(project) + + private val bindings = ArrayList() + private val coroutinesScope = CoroutineScope(Dispatchers.IO) + + @Synchronized + override fun addServer(url: String): IServerConnection { + return AppLevelModelSyncService.getInstance().addConnection(url).let { Connection(it) } + } + + @Synchronized + override fun getServerConnections(): List { + TODO("Not yet implemented") + } + + @Synchronized + override fun dispose() { + bindings.forEach { it.deactivate() } + coroutinesScope.cancel("disposed") + } + + inner class Connection(val connection: AppLevelModelSyncService.ServerConnection) : IServerConnection { + override fun activate() { + TODO("Not yet implemented") + } + + override fun deactivate() { + TODO("Not yet implemented") + } + + override fun remove() { + TODO("Not yet implemented") + } + + override fun getStatus(): IServerConnection.Status { + TODO("Not yet implemented") + } + + override suspend fun pullVersion(branchRef: BranchReference): IVersion { + return connection.getClient().pull(branchRef, null) + } + + override fun bind(branchRef: BranchReference, lastSyncedVersionHash: String?): IBinding { + val binding = Binding( + coroutinesScope = coroutinesScope, + mpsProject = mpsProject, + client = { connection.getClient() }, + branchRef = branchRef, + initialVersionHash = lastSyncedVersionHash, + ) + bindings.add(binding) + binding.activate() + return binding + } + + override fun getBindings(): List { + TODO("Not yet implemented") + } + } +} + +class Binding( + val coroutinesScope: CoroutineScope, + override val mpsProject: org.jetbrains.mps.openapi.project.Project, + val client: suspend () -> IModelClientV2, + override val branchRef: BranchReference, + val initialVersionHash: String?, +) : IBinding { + companion object { + val LOG = mu.KotlinLogging.logger { } + } + + private val activated = AtomicBoolean(false) + private val lastSyncedVersion = ValueWithMutex(null) + private var syncJob: Job? = null + private var syncToServerTask: ValidatingJob? = null + private var invalidatingListener: MyInvalidatingListener? = null + + private val repository: SRepository get() = mpsProject.repository + + override fun activate() { + if (activated.getAndSet(true)) return + syncJob = coroutinesScope.launch { syncJob() } + } + + override fun deactivate() { + if (!activated.getAndSet(false)) return + + syncJob?.cancel() + syncJob = null + syncToServerTask = null + invalidatingListener?.stop() + invalidatingListener = null + } + + private suspend fun checkInSync(): String? { + check(activated.get()) { "Binding is deactivated" } + val version = lastSyncedVersion.flush()?.getOrThrow() + if (version == null) return "Initial sync isn't done yet" + if (invalidatingListener == null) return "No change listener registered in MPS" + if (invalidatingListener?.hasAnyInvalidations() != false) return "There are pending changes in MPS" + val remoteVersion = client().pullHash(branchRef) + if (remoteVersion != version.getContentHash()) return "Local version (${version.getContentHash()} differs from remote version ($remoteVersion)" + return null + } + + override suspend fun flush(): IVersion { + check(syncJob?.isActive == true) { "Synchronization is not active" } + var reason = checkInSync() + var i = 0 + while (reason != null) { + i++ + if (i % 10 == 0) LOG.debug { "Still waiting for the synchronization to finish: $reason" } + delay(100.milliseconds) + reason = checkInSync() + } + return lastSyncedVersion.getValue()!! + } + + private suspend fun CoroutineScope.syncJob() { + // initial sync + initialSync() + + // continuous sync to MPS + launchLoop { + val newHash = client().pollHash(branchRef, lastSyncedVersion.getValue()) + if (newHash != lastSyncedVersion.getValue()?.getContentHash()) { + LOG.debug { "New remote version detected: $newHash" } + syncToMPS() + } + } + + // continuous sync to server + syncToServerTask = launchValidation { + syncToServer() + } + } + + suspend fun ensureInitialized() { + if (lastSyncedVersion.getValue() == null) { + initialSync() + } + } + + private suspend fun initialSync() { + lastSyncedVersion.updateValue { oldVersion -> + LOG.debug { "Running initial synchronization" } + + val baseVersion = oldVersion + ?: initialVersionHash?.let { client().loadVersion(branchRef.repositoryId, it, null) } + if (baseVersion == null) { + // Binding was never activated before. Overwrite local changes or do initial upload. + + val remoteVersion = client().pullIfExists(branchRef) + if (remoteVersion == null) { + LOG.debug { "Repository don't exist. Will copy the local project to the server." } + // repository doesn't exist -> copy the local project to the server + val emptyVersion = client().initRepository(branchRef.repositoryId) + doSyncToServer(emptyVersion) ?: emptyVersion + } else { + LOG.debug { "Repository exists. Will checkout version $remoteVersion" } + doSyncToMPS(null, remoteVersion) + remoteVersion + } + } else { + // Binding was activated before. Preserve local changes. + + // push local changes that happened while the binding was deactivated + val localChanges = doSyncFromMPS(baseVersion) + val remoteVersion = if (localChanges != null) { + val mergedVersion = client().push(branchRef, localChanges, baseVersion) + doSyncToMPS(baseVersion, mergedVersion) + mergedVersion + } else { + client().pull(branchRef, baseVersion) + } + + // load remote changes into MPS + doSyncToMPS(baseVersion, remoteVersion) + + remoteVersion + } + } + } + + suspend fun syncToMPS(): IVersion { + return lastSyncedVersion.updateValue { oldVersion -> + client().pull(branchRef, oldVersion).also { newVersion -> + doSyncToMPS(oldVersion, newVersion) + } + } + } + + suspend fun syncToServer(): IVersion? { + return lastSyncedVersion.updateValue { oldVersion -> + if (oldVersion == null) { + // have to wait for initial sync + oldVersion + } else { + val newVersion = doSyncToServer(oldVersion) + newVersion ?: oldVersion + } + } + } + + private suspend fun doSyncToMPS(oldVersion: IVersion?, newVersion: IVersion) { + if (oldVersion?.getContentHash() == newVersion.getContentHash()) return + + LOG.debug { "Updating MPS project from $oldVersion to $newVersion" } + + val mpsProjects = listOf(mpsProject as MPSProject) + val baseVersion = oldVersion + val filter = if (baseVersion != null) { + val invalidationTree = InvalidationTree(100_000) + val newTree = newVersion.getTree() + newTree.visitChanges( + baseVersion.getTree(), + InvalidatingVisitor(newTree, invalidationTree), + ) + invalidationTree + } else { + FullSyncFilter() + } + + val targetRoot = MPSRepositoryAsNode(repository) + writeToMPS { + if (invalidatingListener?.hasAnyInvalidations() == true) { + // Concurrent modification! + // Write changes from MPS to a new version first and try again after it is merged. + LOG.debug { "Skipping sync to MPS because there are pending changes in MPS" } + return@writeToMPS + } + + getMPSListener().runSync { + val branch = TreePointer(newVersion.getTree()) + val nodeAssociation = NodeAssociationFromModelServer(branch, targetRoot.getModel()) + ModelSynchronizer( + filter = filter, + sourceRoot = branch.getRootNode().asWritableNode(), + targetRoot = targetRoot, + nodeAssociation = nodeAssociation, + sourceMask = MPSProjectSyncMask(mpsProjects, false), + targetMask = MPSProjectSyncMask(mpsProjects, true), + ).synchronize() + } + } + } + + private suspend fun writeToMPS(body: () -> R): R { + val result = ArrayList() + withContext(Dispatchers.EDT) { + repository.modelAccess.executeUndoTransparentCommand { + ModelixMpsApi.runWithProject(mpsProject) { + result += body() + } + } + } + return result.single() + } + + private fun getMPSListener() = invalidatingListener ?: initializeListener() + + private fun initializeListener(): MyInvalidatingListener { + // Being inside a transaction ensure there are not writes, and we don't lose changes. + repository.modelAccess.checkReadAccess() + check(invalidatingListener == null) + return MyInvalidatingListener().also { + invalidatingListener = it + it.start(repository) + } + } + + /** + * @return null if nothing changed + */ + private suspend fun doSyncFromMPS(oldVersion: IVersion): IVersion? { + check(lastSyncedVersion.isLocked()) + + LOG.debug { "Commiting MPS changes" } + + val mpsProjects = listOf(mpsProject as MPSProject) + val client = client() + val newVersion = repository.modelAccess.computeReadAction { + fun sync(invalidationTree: IIncrementalUpdateInformation): IVersion? { + return oldVersion.runWrite(client) { branch -> + ModelixMpsApi.runWithProject(mpsProject) { + ModelSynchronizer( + filter = invalidationTree, + sourceRoot = MPSRepositoryAsNode(ModelixMpsApi.getRepository()), + targetRoot = branch.getRootNode().asWritableNode(), + nodeAssociation = NodeAssociationToModelServer(branch), + sourceMask = MPSProjectSyncMask(mpsProjects, true), + targetMask = MPSProjectSyncMask(mpsProjects, false), + ).synchronize() + } + } + } + + if (invalidatingListener == null) { + sync(FullSyncFilter()).also { + // registering the listener after the sync is sufficient + // because we are in a read action that prevents model changes + initializeListener() + } + } else { + invalidatingListener!!.runSync { sync(it) } + } + } + + LOG.debug { if (newVersion == null) "Nothing changed" else "New version created: $newVersion" } + + return newVersion + } + + /** + * @return null if nothing changed + */ + private suspend fun doSyncToServer(oldVersion: IVersion): IVersion? { + return doSyncFromMPS(oldVersion)?.let { client().push(branchRef, it, oldVersion) } + } + + private inner class MyInvalidatingListener : MPSInvalidatingListener(repository) { + override fun onInvalidation() { + syncToServerTask?.invalidate() + } + } +} + +fun CoroutineScope.launchLoop(body: suspend () -> Unit) = launchLoop(BackoffStrategy(), body) +fun CoroutineScope.launchLoop(backoffStrategy: BackoffStrategy, body: suspend () -> Unit) = launch { jobLoop(backoffStrategy, body) } + +suspend fun jobLoop(body: suspend () -> Unit): Unit = jobLoop(BackoffStrategy(), body) + +suspend fun jobLoop( + backoffStrategy: BackoffStrategy, + body: suspend () -> Unit, +) { + while (true) { + try { + backoffStrategy.wait() + body() + backoffStrategy.success() + } catch (ex: CancellationException) { + break + } catch (ex: Throwable) { + LOG.warn("Exception during synchronization", ex) + backoffStrategy.failed() + } + } +} + +class BackoffStrategy( + val initialDelay: Duration = 500.milliseconds, + val maxDelay: Duration = 10.seconds, + val factor: Double = 1.5, +) { + var currentDelay: Duration = initialDelay + + fun failed() { + currentDelay = (currentDelay * factor).coerceAtMost(maxDelay) + } + + fun success() { + currentDelay = initialDelay + } + + suspend fun wait() { + delay(currentDelay) + } +} + +class ValueWithMutex(private var value: E) { + private val mutex = Mutex() + private var lastUpdateResult: Result? = null + + suspend fun updateValue(body: suspend (E) -> R): R { + return mutex.withLock { + val newValue = runCatching { + body(value) + } + lastUpdateResult = newValue + newValue.onFailure { + LOG.error(it) { "Value update failed. Keeping $value" } + } + newValue.getOrThrow().also { value = it } + } + } + + /** + * Blocks until any active update is done. + * @return The result of the most recent update attempt. + */ + suspend fun flush(): Result? { + return mutex.withLock { lastUpdateResult } + } + + fun isLocked() = mutex.isLocked + + fun getValue(): E = value +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt new file mode 100644 index 0000000000..c72929f4c1 --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt @@ -0,0 +1,11 @@ +package org.modelix.mps.sync3 + +import com.intellij.openapi.components.service +import com.intellij.openapi.project.Project +import com.intellij.openapi.startup.ProjectActivity + +class ModelSyncStartupActivity : ProjectActivity { + override suspend fun execute(project: Project) { + project.service() // just ensure it's initialized + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt new file mode 100644 index 0000000000..26e3ef30cd --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt @@ -0,0 +1,31 @@ +package org.modelix.mps.sync3 + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch + +private val LOG = mu.KotlinLogging.logger { } + +class ValidatingJob(private val validate: suspend () -> Unit) { + private val dirty = Channel(1, onBufferOverflow = BufferOverflow.DROP_LATEST) + + fun invalidate() { + dirty.trySend(Unit) + } + + suspend fun run() { + jobLoop { + dirty.receive() + validate() + } + } +} + +fun CoroutineScope.launchValidation(body: suspend () -> Unit): ValidatingJob { + val job = ValidatingJob(body) + launch { + job.run() + } + return job +} diff --git a/mps-sync-plugin3/src/main/resources/META-INF/plugin.xml b/mps-sync-plugin3/src/main/resources/META-INF/plugin.xml new file mode 100644 index 0000000000..c2d71a6c61 --- /dev/null +++ b/mps-sync-plugin3/src/main/resources/META-INF/plugin.xml @@ -0,0 +1,31 @@ + + + + org.modelix.mps.react + + + Modelix Model Synchronization for MPS + + + itemis AG + + + + + + com.intellij.modules.mps + jetbrains.mps.core + + + + + + + diff --git a/mps-sync-plugin3/src/main/resources/META-INF/pluginIcon.svg b/mps-sync-plugin3/src/main/resources/META-INF/pluginIcon.svg new file mode 100644 index 0000000000..ddaf89cb16 --- /dev/null +++ b/mps-sync-plugin3/src/main/resources/META-INF/pluginIcon.svg @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt new file mode 100644 index 0000000000..4928cf567b --- /dev/null +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt @@ -0,0 +1,124 @@ +package org.modelix.mps.sync3 + +import com.intellij.ide.impl.OpenProjectTask +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.EDT +import com.intellij.openapi.project.Project +import com.intellij.openapi.project.ProjectManager +import com.intellij.openapi.project.ex.ProjectManagerEx +import com.intellij.openapi.util.Disposer +import com.intellij.testFramework.UsefulTestCase +import com.intellij.util.io.delete +import jetbrains.mps.ide.ThreadUtils +import jetbrains.mps.ide.project.ProjectHelper +import jetbrains.mps.project.MPSProject +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.absolute +import kotlin.io.path.writeText + +abstract class MPSTestBase : UsefulTestCase() { + + protected lateinit var project: Project + + override fun runInDispatchThread() = false + + @OptIn(ExperimentalPathApi::class) + fun openTestProject(testDataName: String?): Project { + val projectDirParent = Path.of("build", "test-projects").absolute() + projectDirParent.toFile().mkdirs() + val projectDir = Files.createTempDirectory(projectDirParent, "mps-project") + projectDir.delete(recursively = true) + projectDir.toFile().mkdirs() + projectDir.toFile().deleteOnExit() + val options = OpenProjectTask().withProjectName("test-project") + val project = if (testDataName != null) { + val sourceDir = File("testdata/$testDataName") + sourceDir.copyRecursively(projectDir.toFile(), overwrite = true) + ProjectManagerEx.getInstanceEx().openProject(projectDir, options)!! + } else { + projectDir.resolve(".mps").also { it.toFile().mkdirs() }.resolve("modules.xml").writeText( + """ + + + + + + + + """.trimIndent(), + ) + ProjectManagerEx.getInstanceEx().openProject(projectDir, options)!! + } + + disposeOnTearDownInEdt { project.close() } + + ApplicationManager.getApplication().invokeAndWait { + // empty - openTestProject executed not in EDT, so, invokeAndWait just forces + // processing of all events that were queued during project opening + } + + this.project = project + + return project + } + + private fun disposeOnTearDownInEdt(runnable: Runnable) { + Disposer.register( + testRootDisposable, + Disposable { + ApplicationManager.getApplication().invokeAndWait(runnable) + }, + ) + } + + protected val mpsProject: MPSProject get() { + return checkNotNull(ProjectHelper.fromIdeaProject(project)) { "MPS project not loaded" } + } + + protected fun writeAction(body: () -> R): R { + return mpsProject.modelAccess.computeWriteAction(body) + } + + protected suspend fun command(body: () -> R): R { + var result: R? = null + withContext(Dispatchers.EDT) { + mpsProject.modelAccess.executeCommand { result = body() } + } + return result as R + } + + protected fun writeActionOnEdt(body: () -> R): R { + return onEdt { writeAction { body() } } + } + + protected fun onEdt(body: () -> R): R { + var result: R? = null + ThreadUtils.runInUIThreadAndWait { + result = body() + } + return result as R + } + + protected fun readAction(body: () -> R): R { + var result: R? = null + mpsProject.modelAccess.runReadAction { + result = body() + } + return result as R + } +} + +fun Project.close() { + ApplicationManager.getApplication().invokeLaterOnWriteThread { + runCatching { + ProjectManager.getInstance().closeAndDispose(this) + } + } + ApplicationManager.getApplication().invokeAndWait { } +} diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt new file mode 100644 index 0000000000..41f1b811f4 --- /dev/null +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt @@ -0,0 +1,99 @@ +package org.modelix.mps.sync3 + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.project.Project +import jetbrains.mps.ide.project.ProjectHelper +import jetbrains.mps.project.AbstractModule +import jetbrains.mps.smodel.Language +import org.jetbrains.mps.openapi.model.EditableSModel +import org.modelix.mps.api.ModelixMpsApi +import org.w3c.dom.Element +import java.nio.file.Path +import kotlin.io.path.absolute +import kotlin.io.path.isRegularFile +import kotlin.io.path.pathString +import kotlin.io.path.readText +import kotlin.io.path.relativeTo +import kotlin.io.path.walk + +fun Project.captureSnapshot(): String = captureFileContents().let { filterFiles(it) }.contentsAsString() + +private fun Map.contentsAsString(): String { + return entries.sortedBy { it.key }.joinToString("\n\n\n") { "------ ${it.key} ------\n${it.value}" } +} + +private fun filterFiles(files: Map) = files.filter { + val name = it.key + if (name.startsWith(".mps/")) { + name == ".mps/modules.xml" + } else if (name.contains("/source_gen") || name.contains("/classes_gen")) { + false + } else { + true + } +} + +private fun Project.captureFileContents(): Map { + ApplicationManager.getApplication().invokeAndWait { + ProjectHelper.fromIdeaProject(this)!!.modelAccess.runWriteAction { + for (module in ProjectHelper.fromIdeaProject(this)!!.projectModules.flatMap { + listOf(it) + ((it as? Language)?.generators ?: emptyList()) + }) { + module as AbstractModule + module.save() + for (model in module.models.filterIsInstance()) { + ModelixMpsApi.forceSave(model) + } + } + } + ApplicationManager.getApplication().saveAll() + save() + } + return Path.of(this.basePath).walk().filter { it.isRegularFile() }.associate { file -> + val name = file.absolute().relativeTo(Path.of(basePath).absolute()).pathString + val content = file.readText().trim() + val xmlEndings = setOf("mps", "devkit", "mpl", "msd") + val normalizedContent = when { + xmlEndings.contains(name.substringAfterLast(".")) -> normalizeXmlFile(content) + else -> content + } + name to normalizedContent + } +} + +private fun normalizeXmlFile(content: String): String { + val xml = readXmlFile(content.byteInputStream()) + xml.visitAll { node -> + if (node !is Element) return@visitAll + when (node.tagName) { + "node" -> { + node.childElements("property").sortByRole() + node.childElements("ref").sortByRole() + node.childElements("node").sortByRole() + } + "dev-kit" -> { + node.childElements("exported-language").sortByAttribute("name") + } + "sourceRoot" -> { + val location = node.getAttribute("location") + val path = node.getAttribute("path") + if (path.isNullOrEmpty() && !location.isNullOrEmpty()) { + val contentPath = (node.parentNode as Element).getAttribute("contentPath") + node.removeAttribute("location") + node.setAttribute("path", "$contentPath/$location") + } + } + } + } + return xmlToString(xml).lineSequence().filter { it.isNotBlank() }.joinToString("\n") +} + +private fun List.sortByRole() = sortByAttribute("role") +private fun List.sortByAttribute(name: String) = sortBy { it.getAttribute(name) } +private fun > List.sortBy(selector: (Element) -> T) { + if (size < 2) return + val sorted = sortedBy { selector(it) } + for (i in (0..sorted.lastIndex - 1).reversed()) { + sorted[i].parentNode.insertBefore(sorted[i], sorted[i + 1]) + } +} diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt new file mode 100644 index 0000000000..3dc2fc4e2c --- /dev/null +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -0,0 +1,361 @@ +package org.modelix.mps.sync3 + +import com.badoo.reaktive.observable.toList +import com.intellij.testFramework.TestApplicationManager +import jetbrains.mps.smodel.SNodeUtil +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.junit.Assert +import org.modelix.model.IVersion +import org.modelix.model.api.TreePointer +import org.modelix.model.api.async.PropertyChangedEvent +import org.modelix.model.api.async.TreeChangeEvent +import org.modelix.model.api.getDescendants +import org.modelix.model.api.getRootNode +import org.modelix.model.api.key +import org.modelix.model.client2.ModelClientV2 +import org.modelix.model.client2.runWriteOnBranch +import org.modelix.model.data.NodeData +import org.modelix.model.data.asData +import org.modelix.model.lazy.BranchReference +import org.modelix.model.lazy.CLVersion +import org.modelix.model.lazy.RepositoryId +import org.modelix.model.mpsadapters.MPSModuleAsNode +import org.modelix.model.mpsadapters.MPSProperty +import org.modelix.model.mpsadapters.tryDecodeModelixReference +import org.modelix.streams.getSuspending +import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.images.builder.ImageFromDockerfile +import java.nio.file.Path +import java.util.concurrent.atomic.AtomicLong +import kotlin.io.path.absolute +import kotlin.time.Duration.Companion.minutes +import kotlin.time.toJavaDuration + +private val modelServerDir = Path.of("../model-server").absolute().normalize() +private val modelServerImage = ImageFromDockerfile() + .withDockerfile(modelServerDir.resolve("Dockerfile")) + +class ProjectSyncTest : MPSTestBase() { + + private var lastSnapshotBeforeSync: String? = null + private var lastSnapshotAfterSync: String? = null + + override fun setUp() { + super.setUp() + TestApplicationManager.getInstance() + } + + override fun tearDown() { + super.tearDown() + } + + private suspend fun syncProjectToServer( + testDataName: String, + port: Int, + branchRef: BranchReference, + lastSyncedVersion: String? = null, + ): IVersion { + val project = openTestProject(testDataName) + lastSnapshotBeforeSync = project.captureSnapshot() + val service = IModelSyncService.getInstance(project) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef, lastSyncedVersion) + val version = binding.flush() + binding.close() + lastSnapshotAfterSync = project.captureSnapshot() + project.close() + return version + } + + fun `test initial sync to server`(): Unit = runWithModelServer { port -> + val branchRef = RepositoryId("sync-test").getBranchReference() + syncProjectToServer("initial", port, branchRef) + + val client = ModelClientV2.builder().url("http://localhost:$port").build() + val version = client.pull(branchRef, null) + val rootNode = TreePointer(version.getTree()).getRootNode() + val allNodes = rootNode.getDescendants(true) + assertEquals(221, allNodes.count()) + } + + fun `test checkout into empty project`(): Unit = runWithModelServer { port -> + val branchRef = RepositoryId("sync-test").getBranchReference() + syncProjectToServer("initial", port, branchRef) + val expectedSnapshot = lastSnapshotBeforeSync + + val emptyProject = openTestProject(null) + val service = IModelSyncService.getInstance(emptyProject) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef) + binding.flush() + + assertEquals(expectedSnapshot, project.captureSnapshot()) + } + + fun `test write to new repo after checkout`(): Unit = runWithModelServer { port -> + // An existing version on the server ... + val branchRef1 = RepositoryId("sync-test-A").getBranchReference() + syncProjectToServer("initial", port, branchRef1) + + // ... is checked out into an (empty) local project ... + val emptyProject = openTestProject(null) + val service = IModelSyncService.getInstance(emptyProject) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef1) + binding.flush() + + readAction { + assertEquals(5, mpsProject.projectModules.size) + + val allNodes = mpsProject.projectModules.asSequence() + .map { MPSModuleAsNode(it) } + .flatMap { it.getDescendants(true) } + assertEquals(214, allNodes.count()) + } + + binding.close() + + // ... and then written back into a new repository + val branchRef2 = RepositoryId("sync-test-B").getBranchReference() + val binding2 = connection.bind(branchRef2) + binding2.flush() + + suspend fun pullJson(ref: BranchReference) = connection + .pullVersion(ref) + .asNormalizedJson() + + // both repositories should now contain the same data + assertEquals(pullJson(branchRef1), pullJson(branchRef2)) + } + + fun `test sync after MPS change`(): Unit = runWithModelServer { port -> + // An MPS project is connected to a repository ... + val branchRef = RepositoryId("sync-test").getBranchReference() + openTestProject("initial") + val service = IModelSyncService.getInstance(mpsProject) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef) + val version1 = binding.flush() + + // ... and then an MPS user changes the name of a class ... + val nameProperty = SNodeUtil.property_INamedConcept_name + command { + val node = mpsProject.projectModules + .first { it.moduleName == "NewSolution" } + .models + .flatMap { it.rootNodes } + .first { it.getProperty(nameProperty) == "MyClass" } + println("will change property") + node.setProperty(nameProperty, "Changed") + println("property changed") + } + println("command done") + + val version2 = binding.flush() + + println("Version 1: $version1") + println("Version 2: $version2") + + // ... which should result in a new version on the server with a single property change operation + val changes: List = version2.getTree().asAsyncTree().getChanges(version1.getTree().asAsyncTree(), false).toList().getSuspending() + assertEquals(1, changes.size) + val change = changes.single() as PropertyChangedEvent + assertEquals(MPSProperty(nameProperty).getUID(), change.role.getUID()) + assertEquals("MyClass", version1.getTree().getProperty(change.nodeId, change.role.key(version1.getTree()))) + assertEquals("Changed", version2.getTree().getProperty(change.nodeId, change.role.key(version1.getTree()))) + } + + fun `test sync after model-server change`(): Unit = runWithModelServer { port -> + // An MPS project is connected to a repository ... + val branchRef = RepositoryId("sync-test").getBranchReference() + openTestProject("initial") + val service = IModelSyncService.getInstance(mpsProject) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef) + val version1 = binding.flush() + + val nameProperty = MPSProperty(SNodeUtil.property_INamedConcept_name) + val mpsNode = readAction { + mpsProject.projectModules + .first { it.moduleName == "NewSolution" } + .models + .flatMap { it.rootNodes } + .first { it.getProperty(nameProperty.property) == "MyClass" } + } + + assertEquals("MyClass", readAction { mpsNode.getProperty(nameProperty.property) }) + + // ... and then some non-MPS client changes a property ... + val client = ModelClientV2.builder().url("http://localhost:$port").build().also { it.init() } + client.runWriteOnBranch(branchRef) { branch -> + val node = branch.getRootNode().getDescendants(true) + .first { it.getPropertyValue(nameProperty) == "MyClass" } + node.setPropertyValue(nameProperty, "Changed") + } + val version2 = binding.flush() + + // ... which should then be visible in MPS + assertEquals("Changed", readAction { mpsNode.getProperty(nameProperty.property) }) + } + + fun `test new node on model-server`(): Unit = runWithModelServer { port -> + // An MPS project is connected to a repository ... + val branchRef = RepositoryId("sync-test").getBranchReference() + openTestProject("initial") + val service = IModelSyncService.getInstance(mpsProject) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef) + val version1 = binding.flush() + + val nameProperty = MPSProperty(SNodeUtil.property_INamedConcept_name) + val mpsNode = writeAction { + mpsProject.projectModules + .first { it.moduleName == "NewSolution" } + .models + .flatMap { it.rootNodes } + .first { it.getProperty(nameProperty.property) == "MyClass" } + } + + assertEquals("MyClass", readAction { mpsNode.getProperty(nameProperty.property) }) + + // ... and then a non-MPS client adds a new node ... + val client = ModelClientV2.builder().url("http://localhost:$port").build().also { it.init() } + val newNodeIdOnServer = client.runWriteOnBranch(branchRef) { branch -> + val node = branch.getRootNode().getDescendants(true) + .first { it.getPropertyValue(nameProperty) == "MyClass" } + .asWritableNode() + val node2 = node.getParent()!!.addNewChild(node.getContainmentLink(), -1, node.getConceptReference()) + node2.setPropertyValue(nameProperty.toReference(), "NewClass") + node2.getNodeReference().serialize() + } + val version2 = binding.flush() + + // ... which should then be added in MPS ... + readAction { + val siblings = mpsNode.model!!.rootNodes + val newNode = siblings.first { it.getProperty(nameProperty.property) == "NewClass" } + assertEquals("NewClass", newNode.getProperty(nameProperty.property)) + // ... and have an ID that isn't a random one generated by MPS + assertEquals(newNodeIdOnServer, newNode.nodeId.tryDecodeModelixReference()?.serialize()) + } + } + + fun `test sync after reconnect ignoring local`(): Unit = runWithModelServer { port -> + // A version exists on the server ... + val branchRef = RepositoryId("sync-test").getBranchReference() + val version1 = syncProjectToServer("initial", port, branchRef) + + // ... and then some other existing MPS project is connected to that repository ... + val version2 = syncProjectToServer("change1", port, branchRef) + + // ... and since it's an unrelated project, it should just be overwritten + assertEquals(version1.getContentHash(), version2.getContentHash()) + } + + fun `test sync after reconnect merging local`(): Unit = runWithModelServer { port -> + // A version exists on the server ... + val branchRef = RepositoryId("sync-test-A").getBranchReference() + val version1 = syncProjectToServer("initial", port, branchRef) + + // ... and while being disconnected, some changes are made in MPS ... + + // ... and then the project is connected again ... + val version2 = syncProjectToServer("change1", port, branchRef, version1.getContentHash()) + + // ... causing the changes to be commited on top of the remote version ... + Assert.assertNotEquals(version1.getContentHash(), version2.getContentHash()) + assertEquals(version1.getContentHash(), (version2 as CLVersion).baseVersion?.getContentHash()) + + val branchRef2 = RepositoryId("sync-test-B").getBranchReference() + val expected = syncProjectToServer("change1", port, branchRef2) + + // ... and the new version should contain the state of the project before the reconnect. + assertEquals(expected.asNormalizedJson(), version2.asNormalizedJson()) + } + + fun `test sync to MPS after non-trivial commit`(): Unit = runWithModelServer { port -> + // Two clients are in sync with the same version ... + val branchRef = RepositoryId("sync-test").getBranchReference() + val version1 = syncProjectToServer("initial", port, branchRef) + + // ... and while one client is disconnected, the other client continues making changes. + val version2 = syncProjectToServer("change1", port, branchRef, version1.getContentHash()) + val expectedSnapshot = lastSnapshotBeforeSync + + println("initial two versions pushed") + + // The second client then reconnects ... + openTestProject("initial") + println("initial project opened") + val binding = IModelSyncService.getInstance(mpsProject) + .addServer("http://localhost:$port") + .bind(branchRef, version1.getContentHash()) + println("binding created") + val version3 = binding.flush() + + // ... applies all the pending changes and is again in sync with the other client + assertEquals(expectedSnapshot, project.captureSnapshot()) + } + + private fun runWithModelServer(body: suspend (port: Int) -> Unit) = runBlocking { + withTimeout(3.minutes) { + val modelServer: GenericContainer<*> = GenericContainer(modelServerImage) + .withExposedPorts(28101) + .withCommand("-inmemory") + .waitingFor(Wait.forListeningPort().withStartupTimeout(3.minutes.toJavaDuration())) + .withLogConsumer { + println(it.utf8StringWithoutLineEnding) + } + + modelServer.start() + try { + body(modelServer.firstMappedPort) + } finally { + modelServer.stop() + } + } + } + + private fun NodeData.normalizeIds(): NodeData { + val idMap = HashMap() + fillIdSubstitutions(idMap, AtomicLong()) + return replaceIds(idMap) + } + + private fun NodeData.replaceIds(idMap: MutableMap): NodeData { + fun replaceId(id: String) = idMap[id] ?: id + + return copy( + id = id?.let { replaceId(it) }, + children = children.map { it.replaceIds(idMap) }, + references = references.mapValues { replaceId(it.value) }, + ) + } + + private fun NodeData.sortChildren(): NodeData { + return copy( + children = children.sortedWith(compareBy({ it.role }, { it.id })).map { it.sortChildren() }, + ) + } + + private fun NodeData.fillIdSubstitutions(idMap: MutableMap, idGenerator: AtomicLong) { + id?.let { + idMap.getOrPut(it) { + properties[NodeData.ID_PROPERTY_KEY] ?: ("normalized:" + idGenerator.incrementAndGet()) + } + } + children.forEach { it.fillIdSubstitutions(idMap, idGenerator) } + } + + private fun IVersion.asNormalizedJson(): String { + return getTree() + .let { TreePointer(it) } + .getRootNode() + .asData() + .normalizeIds() + .sortChildren() + .toJson() + } +} diff --git a/bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/XMLUtils.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt similarity index 98% rename from bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/XMLUtils.kt rename to mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt index e99689c93f..5c94ece5dd 100644 --- a/bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/XMLUtils.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt @@ -1,4 +1,4 @@ -package org.modelix.model.sync.bulk.lib.test +package org.modelix.mps.sync3 import org.w3c.dom.Document import org.w3c.dom.Element diff --git a/mps-sync-plugin3/src/test/resources/logback-test.xml b/mps-sync-plugin3/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..2ea72810ee --- /dev/null +++ b/mps-sync-plugin3/src/test/resources/logback-test.xml @@ -0,0 +1,21 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/.gitignore b/mps-sync-plugin3/testdata/.gitignore new file mode 100644 index 0000000000..1f02983e16 --- /dev/null +++ b/mps-sync-plugin3/testdata/.gitignore @@ -0,0 +1,5 @@ +test_gen +test_gen.caches +classes_gen +source_gen +source_gen.caches diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/.gitignore b/mps-sync-plugin3/testdata/change1/.mps/.gitignore similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/.gitignore rename to mps-sync-plugin3/testdata/change1/.mps/.gitignore diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/migration.xml b/mps-sync-plugin3/testdata/change1/.mps/migration.xml similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/migration.xml rename to mps-sync-plugin3/testdata/change1/.mps/migration.xml diff --git a/mps-sync-plugin3/testdata/change1/.mps/modules.xml b/mps-sync-plugin3/testdata/change1/.mps/modules.xml new file mode 100644 index 0000000000..ac7db30fc9 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/.mps/modules.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/devkits/NewDevkit/NewDevkit.devkit b/mps-sync-plugin3/testdata/change1/devkits/NewDevkit/NewDevkit.devkit new file mode 100644 index 0000000000..c450cd9622 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/devkits/NewDevkit/NewDevkit.devkit @@ -0,0 +1,8 @@ + + + + + + 4eb87a8f-881e-4d34-9514-f5002000c363(NewRuntimeSolution) + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/NewLanguage.mpl b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/NewLanguage.mpl similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/NewLanguage.mpl rename to mps-sync-plugin3/testdata/change1/languages/NewLanguage/NewLanguage.mpl diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps new file mode 100644 index 0000000000..ec7f4a3508 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.behavior.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.behavior.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.behavior.mps rename to mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.behavior.mps diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.constraints.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.constraints.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.constraints.mps rename to mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.constraints.mps diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.editor.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.editor.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.editor.mps rename to mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.editor.mps diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.structure.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.structure.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.structure.mps rename to mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.structure.mps diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.typesystem.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.typesystem.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/models/NewLanguage.typesystem.mps rename to mps-sync-plugin3/testdata/change1/languages/NewLanguage/models/NewLanguage.typesystem.mps diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/NewLanguage2.mpl b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/NewLanguage2.mpl new file mode 100644 index 0000000000..b0553ef2b8 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/NewLanguage2.mpl @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 96c7c023-6829-44d0-b358-661f058f1c31(NewLanguage) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 96c7c023-6829-44d0-b358-661f058f1c31(NewLanguage) + + diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/generator/templates/NewLanguage2.generator.templates@generator.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/generator/templates/NewLanguage2.generator.templates@generator.mps new file mode 100644 index 0000000000..fd00255bef --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/generator/templates/NewLanguage2.generator.templates@generator.mps @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.behavior.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.behavior.mps new file mode 100644 index 0000000000..de0a9fd4db --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.behavior.mps @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.constraints.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.constraints.mps new file mode 100644 index 0000000000..36181fc7bc --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.constraints.mps @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.editor.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.editor.mps new file mode 100644 index 0000000000..6e0d65d707 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.editor.mps @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.structure.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.structure.mps new file mode 100644 index 0000000000..eb95f4b172 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.structure.mps @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.typesystem.mps b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.typesystem.mps new file mode 100644 index 0000000000..1acdf7e539 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/languages/NewLanguage2/models/NewLanguage2.typesystem.mps @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/NewRuntimeSolution.msd b/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/NewRuntimeSolution.msd new file mode 100644 index 0000000000..7394989548 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/NewRuntimeSolution.msd @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + 6354ebe7-c22a-4a0f-ac54-50b52ab9b065(JDK) + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.modelB.mps b/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.modelB.mps new file mode 100644 index 0000000000..9825c73705 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.modelB.mps @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps b/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps new file mode 100644 index 0000000000..bb0f464f06 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/solutions/NewSolution/NewSolution.msd b/mps-sync-plugin3/testdata/change1/solutions/NewSolution/NewSolution.msd new file mode 100644 index 0000000000..89ef2012ac --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/solutions/NewSolution/NewSolution.msd @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + 6354ebe7-c22a-4a0f-ac54-50b52ab9b065(JDK) + fbc25dd2-5da4-483a-8b19-70928e1b62d7(jetbrains.mps.devkit.general-purpose) + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/change1/solutions/NewSolution/models/NewSolution.a_model.mps b/mps-sync-plugin3/testdata/change1/solutions/NewSolution/models/NewSolution.a_model.mps new file mode 100644 index 0000000000..bca8d43368 --- /dev/null +++ b/mps-sync-plugin3/testdata/change1/solutions/NewSolution/models/NewSolution.a_model.mps @@ -0,0 +1,133 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/models/NewSolution.b_model.mps b/mps-sync-plugin3/testdata/change1/solutions/NewSolution/models/NewSolution.b_model.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/models/NewSolution.b_model.mps rename to mps-sync-plugin3/testdata/change1/solutions/NewSolution/models/NewSolution.b_model.mps diff --git a/mps-sync-plugin3/testdata/initial/.mps/.gitignore b/mps-sync-plugin3/testdata/initial/.mps/.gitignore new file mode 100644 index 0000000000..26d33521af --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/.mps/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/mps-sync-plugin3/testdata/initial/.mps/migration.xml b/mps-sync-plugin3/testdata/initial/.mps/migration.xml new file mode 100644 index 0000000000..d512ce4547 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/.mps/migration.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/modules.xml b/mps-sync-plugin3/testdata/initial/.mps/modules.xml similarity index 85% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/modules.xml rename to mps-sync-plugin3/testdata/initial/.mps/modules.xml index 72a75e08b5..41b3130335 100644 --- a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/modules.xml +++ b/mps-sync-plugin3/testdata/initial/.mps/modules.xml @@ -6,6 +6,7 @@ + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/devkits/NewDevkit/NewDevkit.devkit b/mps-sync-plugin3/testdata/initial/devkits/NewDevkit/NewDevkit.devkit similarity index 85% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/devkits/NewDevkit/NewDevkit.devkit rename to mps-sync-plugin3/testdata/initial/devkits/NewDevkit/NewDevkit.devkit index c250d0145c..ec1e10b7de 100644 --- a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/devkits/NewDevkit/NewDevkit.devkit +++ b/mps-sync-plugin3/testdata/initial/devkits/NewDevkit/NewDevkit.devkit @@ -6,5 +6,6 @@ 4eb87a8f-881e-4d34-9514-f5002000c363(NewRuntimeSolution) + 6354ebe7-c22a-4a0f-ac54-50b52ab9b065(JDK) diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/NewLanguage.mpl b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/NewLanguage.mpl new file mode 100644 index 0000000000..c0d9c0c3f0 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/NewLanguage.mpl @@ -0,0 +1,143 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps rename to mps-sync-plugin3/testdata/initial/languages/NewLanguage/generator/templates/NewLanguage.generator.templates@generator.mps diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/generator1/templates/NewLanguage.generator01.templates@generator.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/generator1/templates/NewLanguage.generator01.templates@generator.mps new file mode 100644 index 0000000000..64cd299a23 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/generator1/templates/NewLanguage.generator01.templates@generator.mps @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.behavior.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.behavior.mps new file mode 100644 index 0000000000..cccaca58d6 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.behavior.mps @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.constraints.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.constraints.mps new file mode 100644 index 0000000000..9174bc3220 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.constraints.mps @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.editor.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.editor.mps new file mode 100644 index 0000000000..189b19c9f5 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.editor.mps @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.structure.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.structure.mps new file mode 100644 index 0000000000..a2c8e2ad2f --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.structure.mps @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.typesystem.mps b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.typesystem.mps new file mode 100644 index 0000000000..4633af7a03 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/languages/NewLanguage/models/NewLanguage.typesystem.mps @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewRuntimeSolution/NewRuntimeSolution.msd b/mps-sync-plugin3/testdata/initial/solutions/NewRuntimeSolution/NewRuntimeSolution.msd similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewRuntimeSolution/NewRuntimeSolution.msd rename to mps-sync-plugin3/testdata/initial/solutions/NewRuntimeSolution/NewRuntimeSolution.msd diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps b/mps-sync-plugin3/testdata/initial/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps rename to mps-sync-plugin3/testdata/initial/solutions/NewRuntimeSolution/models/NewRuntimeSolution.plugin.mps diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/NewSolution.msd b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/NewSolution.msd similarity index 87% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/NewSolution.msd rename to mps-sync-plugin3/testdata/initial/solutions/NewSolution/NewSolution.msd index 1eb14f8508..ddab424b4e 100644 --- a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/NewSolution.msd +++ b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/NewSolution.msd @@ -11,6 +11,9 @@ + + 3f233e7f-b8a6-46d2-a57f-795d56775243(Annotations) + @@ -27,6 +30,8 @@ + + diff --git a/bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/models/NewSolution.a_model.mps b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.a_model.mps similarity index 100% rename from bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/solutions/NewSolution/models/NewSolution.a_model.mps rename to mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.a_model.mps diff --git a/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.b_model.mps b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.b_model.mps new file mode 100644 index 0000000000..f06dc94545 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.b_model.mps @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.toBeDeletedModel.mps b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.toBeDeletedModel.mps new file mode 100644 index 0000000000..dbb460ad90 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/solutions/NewSolution/models/NewSolution.toBeDeletedModel.mps @@ -0,0 +1,7 @@ + + + + + + + diff --git a/mps-sync-plugin3/testdata/initial/solutions/ToBeDeleted/ToBeDeleted.msd b/mps-sync-plugin3/testdata/initial/solutions/ToBeDeleted/ToBeDeleted.msd new file mode 100644 index 0000000000..cd7ea61ec7 --- /dev/null +++ b/mps-sync-plugin3/testdata/initial/solutions/ToBeDeleted/ToBeDeleted.msd @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/settings.gradle.kts b/settings.gradle.kts index 24c6bb6328..e5d0ee9d84 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,8 +23,8 @@ include("model-client") include("model-client:integration-tests") include("model-datastructure") include("model-server") -include("model-server-openapi") include("model-server-api") +include("model-server-openapi") include("modelql-client") include("modelql-core") include("modelql-html") @@ -33,6 +33,7 @@ include("modelql-typed") include("modelql-untyped") include("mps-model-adapters") include("mps-model-adapters-plugin") +include("mps-sync-plugin3") include("streams") include("ts-model-api") include("vue-model-api") From 6b48f5d649aa59b6798439f297786162348a8a7d Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Tue, 18 Feb 2025 22:24:05 +0100 Subject: [PATCH 02/37] feat(mps-sync-plugin): support for MPS 2020.3 --- .github/workflows/mps-compatibility.yaml | 10 ++++ .../org/modelix/model/mpsadapters/MPSArea.kt | 14 +++-- .../mpsadapters/MPSJavaModuleFacetAsNode.kt | 24 ++++++--- .../model/mpsadapters/MPSModuleAsNode.kt | 8 +-- .../model/mpsadapters/MPSWritableNode.kt | 2 +- .../model/mpsadapters/SolutionProducer.kt | 17 ++++-- mps-sync-plugin3/build.gradle.kts | 18 +++++-- .../mps/sync3/MPSInvalidatingListener.kt | 24 +++++---- .../mps/sync3/ModelSyncForMPSProject.kt | 46 ++++++++-------- .../mps/sync3/ModelSyncStartupActivity.kt | 6 +-- .../org/modelix/mps/sync3/ValidatingJob.kt | 7 +-- .../org/modelix/mps/sync3/MPSTestBase.kt | 8 +-- .../org/modelix/mps/sync3/ProjectSnapshot.kt | 54 ++++++++++++++++--- .../kotlin/org/modelix/mps/sync3/XMLUtils.kt | 3 +- 14 files changed, 168 insertions(+), 73 deletions(-) diff --git a/.github/workflows/mps-compatibility.yaml b/.github/workflows/mps-compatibility.yaml index 27b53b375b..de9384a7ee 100644 --- a/.github/workflows/mps-compatibility.yaml +++ b/.github/workflows/mps-compatibility.yaml @@ -15,6 +15,7 @@ jobs: timeout-minutes: 60 strategy: + fail-fast: false matrix: version: - "2020.3" @@ -47,4 +48,13 @@ jobs: :metamodel-export:build :mps-model-adapters:build :mps-model-adapters-plugin:build + :mps-sync-plugin3:build -Pmps.version.major=${{ matrix.version }} + - name: Archive test report + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-report-${{ matrix.version }} + path: | + */build/test-results + */build/reports diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt index 30769bf6fa..a5d5f81435 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSArea.kt @@ -90,11 +90,7 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference { } override fun executeRead(f: () -> T): T { - var result: T? = null - repository.modelAccess.runReadAction { - result = f() - } - return result!! + return repository.computeRead(f) } override fun executeWrite(f: () -> T): T { @@ -365,3 +361,11 @@ data class MPSArea(val repository: SRepository) : IArea, IAreaReference { return repository.asLegacyNode() } } + +fun SRepository.computeRead(body: () -> R): R { + var result: R? = null + modelAccess.runReadAction { + result = body() + } + return result as R +} diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt index b5d3fa64cc..8495594d41 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSJavaModuleFacetAsNode.kt @@ -1,7 +1,8 @@ package org.modelix.model.mpsadapters -import jetbrains.mps.persistence.MementoImpl import jetbrains.mps.project.facets.JavaModuleFacet +import jetbrains.mps.smodel.Generator +import jetbrains.mps.util.MacroHelper import jetbrains.mps.util.MacrosFactory import org.jetbrains.mps.openapi.module.SRepository import org.jetbrains.mps.openapi.persistence.Memento @@ -30,15 +31,24 @@ data class MPSJavaModuleFacetAsNode(val facet: JavaModuleFacet) : MPSGenericNode }, BuiltinLanguages.MPSRepositoryConcepts.JavaModuleFacet.path.toReference() to object : IPropertyAccessor { override fun read(facet: JavaModuleFacet): String? { - return facet.classesGen?.let { MacrosFactory().module(facet.module).shrinkPath(it.path) } + // return facet.classesGen?.let { facet.macroHelper().shrinkPath(it.path) } + return null } override fun write(element: JavaModuleFacet, value: String?) { - element.classesGen - val memento = MementoImpl() - element.save(memento) - memento.getOrCreateChild("classes").put("path", value?.let { MacrosFactory().module(element.module).expandPath(it) }) - element.load(memento) +// val memento = MementoImpl() +// element.save(memento) +// memento.getOrCreateChild("classes").let { +// it.put("generated", "true") +// it.put("path", value.also { println("${element.module} / path = $it") }) +// } +// element.load(memento) + } + + private fun JavaModuleFacet.macroHelper(): MacroHelper { + return module + .let { if (it is Generator) it.sourceLanguage().sourceModule else it } + .let { MacrosFactory().module(it) } } }, ) diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt index e7fe3b0300..4202a5bbd6 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModuleAsNode.kt @@ -156,10 +156,10 @@ abstract class MPSModuleAsNode : MPSGenericNodeAdapter() { val moduleDescriptor = checkNotNull(module.moduleDescriptor) { "Has no moduleDescriptor: $module" } val newFacet = FacetsFacade.getInstance().getFacetFactory(JavaModuleFacet.FACET_TYPE)!!.create(element) as JavaModuleFacetImpl newFacet.load(MementoImpl()) - val moduleDir = if (element is Generator) element.getGeneratorLocation() else module.moduleSourceDir - if (moduleDir != null) { - newFacet.setGeneratedClassesLocation(moduleDir.findChild(AbstractModule.CLASSES_GEN)) - } +// val moduleDir = if (element is Generator) element.getGeneratorLocation() else module.moduleSourceDir +// if (moduleDir != null) { +// newFacet.setGeneratedClassesLocation(moduleDir.findChild(AbstractModule.CLASSES_GEN)) +// } moduleDescriptor.addFacetDescriptor(newFacet) module.setModuleDescriptor(moduleDescriptor) // notify listeners read(element).filterIsInstance().single() diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt index 4672c9d7d0..72b0e2b563 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSWritableNode.kt @@ -331,7 +331,7 @@ fun org.jetbrains.mps.openapi.model.SNodeId.tryDecodeModelixReference(): NodeRef } fun INodeReference.encodeAsForeignId(): SNodeId { - return SNodeId.Foreign.fromIdNoPrefix("mx" + Hex.encodeHexString(serialize().toByteArray())) + return SNodeId.Foreign("~mx" + Hex.encodeHexString(serialize().toByteArray())) } /** diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt index 1a18722f73..35b7cb6976 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/SolutionProducer.kt @@ -1,8 +1,11 @@ +@file:Suppress("removal") + package org.modelix.model.mpsadapters import jetbrains.mps.extapi.persistence.FileBasedModelRoot import jetbrains.mps.ide.project.ProjectHelper import jetbrains.mps.persistence.DefaultModelRoot +import jetbrains.mps.project.AbstractModule import jetbrains.mps.project.DevKit import jetbrains.mps.project.MPSExtentions import jetbrains.mps.project.MPSProject @@ -37,7 +40,9 @@ class SolutionProducer(private val myProject: MPSProject) { private fun createSolutionDescriptor(namespace: String, id: ModuleId, descriptorFile: IFile): SolutionDescriptor { val descriptor = SolutionDescriptor() - descriptor.outputRoot = "\${module}/source_gen" + // using outputPath instead of outputRoot for backwards compatibility + // descriptor.outputRoot = "\${module}/source_gen" + descriptor.outputPath = descriptorFile.parent!!.findChild("source_gen").path descriptor.namespace = namespace descriptor.id = id val moduleLocation = descriptorFile.parent @@ -74,7 +79,9 @@ class LanguageProducer(private val myProject: MPSProject) { private fun createDescriptor(namespace: String, id: ModuleId, descriptorFile: IFile): LanguageDescriptor { val descriptor = LanguageDescriptor() - descriptor.outputRoot = "\${module}/source_gen" + // using genPath instead of outputRoot for backwards compatibility + // descriptor.outputRoot = "\${module}/source_gen" + descriptor.genPath = descriptorFile.parent!!.findChild("source_gen").path descriptor.namespace = namespace descriptor.id = id val moduleLocation = descriptorFile.parent @@ -127,7 +134,9 @@ class GeneratorProducer(private val myProject: MPSProject) { private fun createDescriptor(namespace: String, id: ModuleId, alias: String?, generatorModuleLocation: IFile, templateModelsLocation: IFile?): GeneratorDescriptor { val descriptor = GeneratorDescriptor() - descriptor.outputRoot = "\${module}/${generatorModuleLocation.name}/source_gen" + // using outputPath instead of outputRoot for backwards compatibility + // descriptor.outputRoot = "\${module}/${generatorModuleLocation.name}/source_gen" + descriptor.outputPath = generatorModuleLocation.findChild(AbstractModule.CLASSES_GEN).path descriptor.namespace = namespace descriptor.id = id descriptor.alias = alias ?: "main" @@ -142,7 +151,7 @@ class GeneratorProducer(private val myProject: MPSProject) { } fun Generator.getGeneratorLocation(): IFile? { - return modelRoots.filterIsInstance().firstNotNullOfOrNull { it.contentDirectory } + return modelRoots.filterIsInstance().mapNotNull { it.contentDirectory }.firstOrNull() } class DevkitProducer(private val myProject: MPSProject) { diff --git a/mps-sync-plugin3/build.gradle.kts b/mps-sync-plugin3/build.gradle.kts index 8443fe7d83..2c3842f6a0 100644 --- a/mps-sync-plugin3/build.gradle.kts +++ b/mps-sync-plugin3/build.gradle.kts @@ -1,6 +1,7 @@ import org.modelix.copyMps import org.modelix.mpsHomeDir import org.modelix.mpsMajorVersion +import org.modelix.mpsPlatformVersion plugins { `modelix-kotlin-jvm` @@ -52,9 +53,6 @@ tasks { onlyIf { !setOf( "2020.3", // incompatible with the intellij plugin - "2021.2", // hangs when executed on CI - "2021.3", // hangs when executed on CI - "2022.2", // hangs when executed on CI ).contains(mpsMajorVersion) } jvmArgs("-Dintellij.platform.load.app.info.from.resources=true") @@ -91,3 +89,17 @@ publishing { } } } + +// disable coroutines agent +if (mpsPlatformVersion < 241) { + afterEvaluate { + val testTask = tasks.test.get() + val originalProviders = testTask.jvmArgumentProviders.toList() + testTask.jvmArgumentProviders.clear() + testTask.jvmArgumentProviders.add(object : CommandLineArgumentProvider { + override fun asArguments(): Iterable { + return originalProviders.flatMap { it.asArguments() }.filterNot { it.contains("coroutines-javaagent.jar") } + } + }) + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt index aae11a1611..1de4addcae 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt @@ -25,7 +25,7 @@ import org.jetbrains.mps.openapi.module.SModule import org.jetbrains.mps.openapi.module.SModuleListener import org.jetbrains.mps.openapi.module.SModuleReference import org.jetbrains.mps.openapi.module.SRepository -import org.jetbrains.mps.openapi.module.SRepositoryListener +import org.jetbrains.mps.openapi.module.SRepositoryListenerBase import org.modelix.model.api.IReadableNode import org.modelix.model.api.toSerialized import org.modelix.model.mpsadapters.GlobalModelListener @@ -42,7 +42,6 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : GlobalModelListener(), SNodeChangeListener, SModuleListener, - SRepositoryListener, SModelListener, org.jetbrains.mps.openapi.model.SModelListener { @@ -117,11 +116,11 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : } override fun addListener(repository: SRepository) { - repository.addRepositoryListener(this) + repository.addRepositoryListener(srepositoryListener) } override fun removeListener(repository: SRepository) { - repository.removeRepositoryListener(this) + repository.removeRepositoryListener(srepositoryListener) } override fun propertyChanged(e: SPropertyChangeEvent) { @@ -210,9 +209,16 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : override fun languageAdded(module: SModule, language: SLanguage) { invalidate(module) } override fun languageRemoved(module: SModule, language: SLanguage) { invalidate(module) } override fun moduleChanged(module: SModule) { invalidate(module) } - override fun moduleAdded(module: SModule) { invalidate(repository) } - override fun beforeModuleRemoved(module: SModule) {} - override fun moduleRemoved(reference: SModuleReference) { invalidate(repository) } - override fun commandStarted(repository: SRepository) {} - override fun commandFinished(repository: SRepository) {} + + /** + * For compatibility with MPS 2020.3, SRepositoryListenerBase is used because SRepositoryListener had the additional + * methods updateStarted and updateFinished in that version. + */ + private val srepositoryListener = object : SRepositoryListenerBase() { + override fun moduleAdded(module: SModule) { invalidate(repository) } + override fun beforeModuleRemoved(module: SModule) {} + override fun moduleRemoved(reference: SModuleReference) { invalidate(repository) } + override fun commandStarted(repository: SRepository) {} + override fun commandFinished(repository: SRepository) {} + } } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt index e8d1f9b2c3..2534c062b9 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt @@ -1,10 +1,11 @@ +@file:OptIn(ExperimentalTime::class) + package org.modelix.mps.sync3 import com.intellij.openapi.Disposable import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.application.EDT +import com.intellij.openapi.application.ModalityState import com.intellij.openapi.components.Service -import com.intellij.openapi.components.service import com.intellij.openapi.project.Project import jetbrains.mps.ide.project.ProjectHelper import jetbrains.mps.project.MPSProject @@ -17,7 +18,6 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext import org.jetbrains.mps.openapi.module.SRepository import org.modelix.model.IVersion import org.modelix.model.api.TreePointer @@ -27,6 +27,7 @@ import org.modelix.model.client2.ModelClientV2 import org.modelix.model.client2.runWrite import org.modelix.model.lazy.BranchReference import org.modelix.model.mpsadapters.MPSRepositoryAsNode +import org.modelix.model.mpsadapters.computeRead import org.modelix.model.sync.bulk.FullSyncFilter import org.modelix.model.sync.bulk.InvalidatingVisitor import org.modelix.model.sync.bulk.InvalidationTree @@ -38,16 +39,15 @@ import org.modelix.mps.api.ModelixMpsApi import org.modelix.mps.model.sync.bulk.MPSProjectSyncMask import org.modelix.mps.sync3.Binding.Companion.LOG import java.util.concurrent.atomic.AtomicBoolean -import kotlin.time.Duration -import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.Duration.Companion.seconds +import kotlin.math.roundToLong +import kotlin.time.ExperimentalTime @Service(Service.Level.APP) class AppLevelModelSyncService() : Disposable { companion object { fun getInstance(): AppLevelModelSyncService { - return ApplicationManager.getApplication().service() + return ApplicationManager.getApplication().getService(AppLevelModelSyncService::class.java) } } @@ -55,8 +55,8 @@ class AppLevelModelSyncService() : Disposable { private val coroutinesScope = CoroutineScope(Dispatchers.Default) private val connectionCheckingJob = coroutinesScope.launchLoop( BackoffStrategy( - initialDelay = 3.seconds, - maxDelay = 10.seconds, + initialDelay = 3_000, + maxDelay = 10_000, factor = 1.2, ), ) { @@ -97,7 +97,7 @@ class AppLevelModelSyncService() : Disposable { @Service(Service.Level.PROJECT) class ModelSyncService(val project: Project) : IModelSyncService, Disposable { - private val mpsProject: MPSProject get() = ProjectHelper.fromIdeaProjectOrFail(project) + private val mpsProject: MPSProject get() = ProjectHelper.fromIdeaProject(project)!! private val bindings = ArrayList() private val coroutinesScope = CoroutineScope(Dispatchers.IO) @@ -210,7 +210,7 @@ class Binding( while (reason != null) { i++ if (i % 10 == 0) LOG.debug { "Still waiting for the synchronization to finish: $reason" } - delay(100.milliseconds) + delay(100) reason = checkInSync() } return lastSyncedVersion.getValue()!! @@ -347,13 +347,17 @@ class Binding( private suspend fun writeToMPS(body: () -> R): R { val result = ArrayList() - withContext(Dispatchers.EDT) { - repository.modelAccess.executeUndoTransparentCommand { - ModelixMpsApi.runWithProject(mpsProject) { - result += body() + ApplicationManager.getApplication().invokeAndWait({ + ApplicationManager.getApplication().runWriteAction { + repository.modelAccess.executeUndoTransparentCommand { + ModelixMpsApi.runWithProject(mpsProject) { + result += body() + } } } - } + }, ModalityState.NON_MODAL) +// withContext(Dispatchers.Main) { +// } return result.single() } @@ -379,7 +383,7 @@ class Binding( val mpsProjects = listOf(mpsProject as MPSProject) val client = client() - val newVersion = repository.modelAccess.computeReadAction { + val newVersion = repository.computeRead { fun sync(invalidationTree: IIncrementalUpdateInformation): IVersion? { return oldVersion.runWrite(client) { branch -> ModelixMpsApi.runWithProject(mpsProject) { @@ -449,14 +453,14 @@ suspend fun jobLoop( } class BackoffStrategy( - val initialDelay: Duration = 500.milliseconds, - val maxDelay: Duration = 10.seconds, + val initialDelay: Long = 500, + val maxDelay: Long = 10_000, val factor: Double = 1.5, ) { - var currentDelay: Duration = initialDelay + var currentDelay: Long = initialDelay fun failed() { - currentDelay = (currentDelay * factor).coerceAtMost(maxDelay) + currentDelay = (currentDelay * factor).roundToLong().coerceAtMost(maxDelay) } fun success() { diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt index c72929f4c1..892fb71d61 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncStartupActivity.kt @@ -2,10 +2,10 @@ package org.modelix.mps.sync3 import com.intellij.openapi.components.service import com.intellij.openapi.project.Project -import com.intellij.openapi.startup.ProjectActivity +import com.intellij.openapi.startup.StartupActivity -class ModelSyncStartupActivity : ProjectActivity { - override suspend fun execute(project: Project) { +class ModelSyncStartupActivity : StartupActivity { + override fun runActivity(project: Project) { project.service() // just ensure it's initialized } } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt index 26e3ef30cd..c8b030f401 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValidatingJob.kt @@ -1,17 +1,18 @@ package org.modelix.mps.sync3 import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.launch private val LOG = mu.KotlinLogging.logger { } class ValidatingJob(private val validate: suspend () -> Unit) { - private val dirty = Channel(1, onBufferOverflow = BufferOverflow.DROP_LATEST) + private val dirty = Channel(1) fun invalidate() { - dirty.trySend(Unit) + // can't use trySend because it doesn't exist in MPS 2020.3 + @Suppress("DEPRECATION_ERROR") + dirty.offer(Unit) } suspend fun run() { diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt index 4928cf567b..5539bc70c8 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt @@ -3,7 +3,7 @@ package org.modelix.mps.sync3 import com.intellij.ide.impl.OpenProjectTask import com.intellij.openapi.Disposable import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.application.EDT +import com.intellij.openapi.application.ModalityState import com.intellij.openapi.project.Project import com.intellij.openapi.project.ProjectManager import com.intellij.openapi.project.ex.ProjectManagerEx @@ -87,8 +87,10 @@ abstract class MPSTestBase : UsefulTestCase() { protected suspend fun command(body: () -> R): R { var result: R? = null - withContext(Dispatchers.EDT) { - mpsProject.modelAccess.executeCommand { result = body() } + withContext(Dispatchers.Main) { + ApplicationManager.getApplication().invokeAndWait({ + mpsProject.modelAccess.executeCommand { result = body() } + }, ModalityState.NON_MODAL) } return result as R } diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt index 41f1b811f4..6614a6ac85 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt @@ -4,12 +4,15 @@ import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.project.Project import jetbrains.mps.ide.project.ProjectHelper import jetbrains.mps.project.AbstractModule +import jetbrains.mps.project.MPSExtentions import jetbrains.mps.smodel.Language import org.jetbrains.mps.openapi.model.EditableSModel import org.modelix.mps.api.ModelixMpsApi import org.w3c.dom.Element +import java.io.File import java.nio.file.Path import kotlin.io.path.absolute +import kotlin.io.path.extension import kotlin.io.path.isRegularFile import kotlin.io.path.pathString import kotlin.io.path.readText @@ -49,16 +52,36 @@ private fun Project.captureFileContents(): Map { ApplicationManager.getApplication().saveAll() save() } - return Path.of(this.basePath).walk().filter { it.isRegularFile() }.associate { file -> - val name = file.absolute().relativeTo(Path.of(basePath).absolute()).pathString - val content = file.readText().trim() - val xmlEndings = setOf("mps", "devkit", "mpl", "msd") - val normalizedContent = when { - xmlEndings.contains(name.substringAfterLast(".")) -> normalizeXmlFile(content) - else -> content + + // Files sometimes don't get deleted. Ignore them if they are not listed in the modules.xml + val visibleModules = HashSet() + File(basePath).resolve(".mps/modules.xml").takeIf { it.isFile }?.let { readXmlFile(it) }?.visitAll { + if (it is Element && it.tagName == "modulePath") { + visibleModules.add(Path.of(it.getAttribute("path").replace("\$PROJECT_DIR\$", basePath!!))) } - name to normalizedContent } + + val moduleEndings = setOf(MPSExtentions.DEVKIT, MPSExtentions.LANGUAGE, MPSExtentions.SOLUTION) + val xmlEndings = moduleEndings + setOf(MPSExtentions.MODEL) + + return Path.of(this.basePath).walk() + .filter { it.isRegularFile() } + .filter { + val isModuleFile = moduleEndings.contains(it.extension) + !isModuleFile || visibleModules.contains(it) + } + .associate { file -> + val name = file.absolute().relativeTo(Path.of(basePath).absolute()).pathString + val content = file.readText().trim() + + val normalizedContent = when { + xmlEndings.contains(name.substringAfterLast(".")) -> { + normalizeXmlFile(content) + } + else -> content + } + name to normalizedContent + } } private fun normalizeXmlFile(content: String): String { @@ -83,6 +106,21 @@ private fun normalizeXmlFile(content: String): String { node.setAttribute("path", "$contentPath/$location") } } + "classes" -> { + node.removeAttribute("path") + } + "language", "solution", "generator" -> { + node.removeAttribute("generatorOutputPath") + } + "registry" -> { + // metamodel may not be built yet and the names not available. + // Ignore them as they don't have any semantic meaning. + node.visitAll { (it as? Element)?.removeAttribute("name") } + } + "facets" -> { + // facets are not synchronized yet + node.parentNode.removeChild(node) + } } } return xmlToString(xml).lineSequence().filter { it.isNotBlank() }.joinToString("\n") diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt index 5c94ece5dd..8dad081687 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/XMLUtils.kt @@ -55,8 +55,7 @@ private fun disableDTD(dbf: DocumentBuilderFactory) { fun Node.visitAll(visitor: (Node) -> Unit) { visitor(this) - val childNodes = this.childNodes - for (i in 0 until childNodes.length) childNodes.item(i).visitAll(visitor) + children().forEach { it.visitAll(visitor) } } fun Node.childElements(): List = children().filterIsInstance() From c18ff2264346a1234cc301bcb9fdfe6580f35acf Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 19 Feb 2025 13:10:30 +0100 Subject: [PATCH 03/37] feat(model-api): changed serialization format of references to modelix node Nodes stored on the model-server used to have the format `pnode:@`. Since version 3.17.0, Modelix is already prepared for switching to the preferred format `modelix:@`. BREAKING CHANGE: model-client versions before 3.17.0 are incompatible to this new release --- .../org/modelix/model/api/PNodeReference.kt | 17 +- .../model/metameta/MetaModelMigrationsTest.kt | 190 +++++++++--------- 2 files changed, 103 insertions(+), 104 deletions(-) diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt index 5bbb2d6189..5a1b566143 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/PNodeReference.kt @@ -10,7 +10,7 @@ data class PNodeReference(val id: Long, val branchId: String) : INodeReference { fun toLocal() = LocalPNodeReference(id) override fun serialize(): String { - return "pnode:${id.toString(16)}@$branchId" + return "modelix:$branchId/${id.toString(16)}" } override fun toString(): String { @@ -24,21 +24,20 @@ data class PNodeReference(val id: Long, val branchId: String) : INodeReference { } } fun tryDeserialize(serialized: String): PNodeReference? { + // New format : modelix:25038f9e-e8ad-470a-9ae8-6978ed172184/1a5003b818f + // Legacy format: pnode:1a5003b818f@25038f9e-e8ad-470a-9ae8-6978ed172184 + // + // The 'modelix' prefix is more intuitive for a node stored inside a Modelix repository. + // Having the repository ID first also feels more natural. + if (serialized.startsWith("pnode:") && serialized.contains('@')) { + // legacy format val withoutPrefix = serialized.substringAfter("pnode:") val parts = withoutPrefix.split('@', limit = 2) if (parts.size != 2) return null val nodeId = parts[0].toLongOrNull(16) ?: return null return PNodeReference(nodeId, parts[1]) } else if (serialized.startsWith("modelix:") && serialized.contains('/')) { - // This would be a nicer serialization format that isn't used yet, but supporting it already will make - // future changes easier without breaking old versions of this library. - // - // Example: modelix:25038f9e-e8ad-470a-9ae8-6978ed172184/1a5003b818f - // Old format: pnode:1a5003b818f@25038f9e-e8ad-470a-9ae8-6978ed172184 - // - // The 'modelix' prefix is more intuitive for a node stored inside a Modelix repository. - // Having the repository ID first also feels more natural. val withoutPrefix = serialized.substringAfter("modelix:") val nodeIdStr = withoutPrefix.substringAfterLast('/') val branchId = withoutPrefix.substringBeforeLast('/') diff --git a/model-client/src/commonTest/kotlin/org/modelix/model/metameta/MetaModelMigrationsTest.kt b/model-client/src/commonTest/kotlin/org/modelix/model/metameta/MetaModelMigrationsTest.kt index 0af0e83895..d5fdd02712 100644 --- a/model-client/src/commonTest/kotlin/org/modelix/model/metameta/MetaModelMigrationsTest.kt +++ b/model-client/src/commonTest/kotlin/org/modelix/model/metameta/MetaModelMigrationsTest.kt @@ -49,17 +49,17 @@ class MetaModelMigrationsTest { //language=JSON """ { - "id": "pnode:100000001@aRepositoryId", + "id": "modelix:aRepositoryId/100000001", "concept": "modelix.metameta.Language", "role": "languages", "children": [ { - "id": "pnode:100000002@aRepositoryId", + "id": "modelix:aRepositoryId/100000002", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000003@aRepositoryId", + "id": "modelix:aRepositoryId/100000003", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -68,7 +68,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000004@aRepositoryId", + "id": "modelix:aRepositoryId/100000004", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -77,7 +77,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000005@aRepositoryId", + "id": "modelix:aRepositoryId/100000005", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -86,7 +86,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000006@aRepositoryId", + "id": "modelix:aRepositoryId/100000006", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -96,11 +96,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618895/474657388638618898" }, "references": { - "childConcept": "pnode:100000029@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000029" } }, { - "id": "pnode:100000007@aRepositoryId", + "id": "modelix:aRepositoryId/100000007", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -110,11 +110,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618895/2206727074858242412" }, "references": { - "childConcept": "pnode:10000000a@aRepositoryId" + "childConcept": "modelix:aRepositoryId/10000000a" } }, { - "id": "pnode:100000008@aRepositoryId", + "id": "modelix:aRepositoryId/100000008", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -124,11 +124,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618895/2206727074858242425" }, "references": { - "childConcept": "pnode:10000000b@aRepositoryId" + "childConcept": "modelix:aRepositoryId/10000000b" } }, { - "id": "pnode:100000009@aRepositoryId", + "id": "modelix:aRepositoryId/100000009", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -138,16 +138,16 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618895/2206727074858242439" }, "references": { - "childConcept": "pnode:100000012@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000012" } }, { - "id": "pnode:10000002f@aRepositoryId", + "id": "modelix:aRepositoryId/10000002f", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" }, { - "id": "pnode:100000030@aRepositoryId", + "id": "modelix:aRepositoryId/100000030", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -159,12 +159,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000000a@aRepositoryId", + "id": "modelix:aRepositoryId/10000000a", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000031@aRepositoryId", + "id": "modelix:aRepositoryId/100000031", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -176,12 +176,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000000b@aRepositoryId", + "id": "modelix:aRepositoryId/10000000b", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:10000000c@aRepositoryId", + "id": "modelix:aRepositoryId/10000000c", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -190,7 +190,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000000d@aRepositoryId", + "id": "modelix:aRepositoryId/10000000d", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -199,7 +199,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000000e@aRepositoryId", + "id": "modelix:aRepositoryId/10000000e", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -208,7 +208,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000000f@aRepositoryId", + "id": "modelix:aRepositoryId/10000000f", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -217,7 +217,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000010@aRepositoryId", + "id": "modelix:aRepositoryId/100000010", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -226,7 +226,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000011@aRepositoryId", + "id": "modelix:aRepositoryId/100000011", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -235,7 +235,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000032@aRepositoryId", + "id": "modelix:aRepositoryId/100000032", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -247,12 +247,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000012@aRepositoryId", + "id": "modelix:aRepositoryId/100000012", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000013@aRepositoryId", + "id": "modelix:aRepositoryId/100000013", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -261,7 +261,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000014@aRepositoryId", + "id": "modelix:aRepositoryId/100000014", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -270,7 +270,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000033@aRepositoryId", + "id": "modelix:aRepositoryId/100000033", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -282,16 +282,16 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000015@aRepositoryId", + "id": "modelix:aRepositoryId/100000015", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000034@aRepositoryId", + "id": "modelix:aRepositoryId/100000034", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:100000002@aRepositoryId" + "concept": "modelix:aRepositoryId/100000002" } } ], @@ -302,16 +302,16 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000016@aRepositoryId", + "id": "modelix:aRepositoryId/100000016", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000035@aRepositoryId", + "id": "modelix:aRepositoryId/100000035", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:100000002@aRepositoryId" + "concept": "modelix:aRepositoryId/100000002" } } ], @@ -322,16 +322,16 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000017@aRepositoryId", + "id": "modelix:aRepositoryId/100000017", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000036@aRepositoryId", + "id": "modelix:aRepositoryId/100000036", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:100000002@aRepositoryId" + "concept": "modelix:aRepositoryId/100000002" } } ], @@ -342,12 +342,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000018@aRepositoryId", + "id": "modelix:aRepositoryId/100000018", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000019@aRepositoryId", + "id": "modelix:aRepositoryId/100000019", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -357,11 +357,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618902/474657388638618903" }, "references": { - "childConcept": "pnode:100000002@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000002" } }, { - "id": "pnode:10000001a@aRepositoryId", + "id": "modelix:aRepositoryId/10000001a", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -371,11 +371,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618902/7064605579395546636" }, "references": { - "childConcept": "pnode:10000001c@aRepositoryId" + "childConcept": "modelix:aRepositoryId/10000001c" } }, { - "id": "pnode:10000001b@aRepositoryId", + "id": "modelix:aRepositoryId/10000001b", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -385,11 +385,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618902/8226136427470548682" }, "references": { - "childConcept": "pnode:100000002@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000002" } }, { - "id": "pnode:100000037@aRepositoryId", + "id": "modelix:aRepositoryId/100000037", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -401,12 +401,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000001c@aRepositoryId", + "id": "modelix:aRepositoryId/10000001c", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:10000001d@aRepositoryId", + "id": "modelix:aRepositoryId/10000001d", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -416,11 +416,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/4008363636171860313/4008363636171860450" }, "references": { - "childConcept": "pnode:100000002@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000002" } }, { - "id": "pnode:10000001e@aRepositoryId", + "id": "modelix:aRepositoryId/10000001e", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -430,16 +430,16 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/4008363636171860313/4201834143491306088" }, "references": { - "childConcept": "pnode:100000020@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000020" } }, { - "id": "pnode:100000038@aRepositoryId", + "id": "modelix:aRepositoryId/100000038", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" }, { - "id": "pnode:100000039@aRepositoryId", + "id": "modelix:aRepositoryId/100000039", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -451,12 +451,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000001f@aRepositoryId", + "id": "modelix:aRepositoryId/10000001f", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:10000003a@aRepositoryId", + "id": "modelix:aRepositoryId/10000003a", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -468,12 +468,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000020@aRepositoryId", + "id": "modelix:aRepositoryId/100000020", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000021@aRepositoryId", + "id": "modelix:aRepositoryId/100000021", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -482,11 +482,11 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000003b@aRepositoryId", + "id": "modelix:aRepositoryId/10000003b", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:10000001f@aRepositoryId" + "concept": "modelix:aRepositoryId/10000001f" } } ], @@ -497,12 +497,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000022@aRepositoryId", + "id": "modelix:aRepositoryId/100000022", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:10000003c@aRepositoryId", + "id": "modelix:aRepositoryId/10000003c", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -514,12 +514,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000023@aRepositoryId", + "id": "modelix:aRepositoryId/100000023", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000024@aRepositoryId", + "id": "modelix:aRepositoryId/100000024", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -528,11 +528,11 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000003d@aRepositoryId", + "id": "modelix:aRepositoryId/10000003d", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:100000012@aRepositoryId" + "concept": "modelix:aRepositoryId/100000012" } } ], @@ -543,16 +543,16 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000025@aRepositoryId", + "id": "modelix:aRepositoryId/100000025", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:10000003e@aRepositoryId", + "id": "modelix:aRepositoryId/10000003e", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:100000012@aRepositoryId" + "concept": "modelix:aRepositoryId/100000012" } } ], @@ -563,12 +563,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000026@aRepositoryId", + "id": "modelix:aRepositoryId/100000026", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:100000027@aRepositoryId", + "id": "modelix:aRepositoryId/100000027", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -577,7 +577,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000028@aRepositoryId", + "id": "modelix:aRepositoryId/100000028", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -586,11 +586,11 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000003f@aRepositoryId", + "id": "modelix:aRepositoryId/10000003f", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts", "references": { - "concept": "pnode:10000000a@aRepositoryId" + "concept": "modelix:aRepositoryId/10000000a" } } ], @@ -601,12 +601,12 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:100000029@aRepositoryId", + "id": "modelix:aRepositoryId/100000029", "concept": "modelix.metameta.Concept", "role": "concepts", "children": [ { - "id": "pnode:10000002a@aRepositoryId", + "id": "modelix:aRepositoryId/10000002a", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -615,7 +615,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000002b@aRepositoryId", + "id": "modelix:aRepositoryId/10000002b", "concept": "modelix.metameta.Property", "role": "properties", "properties": { @@ -624,7 +624,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000002c@aRepositoryId", + "id": "modelix:aRepositoryId/10000002c", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -635,7 +635,7 @@ class MetaModelMigrationsTest { } }, { - "id": "pnode:10000002d@aRepositoryId", + "id": "modelix:aRepositoryId/10000002d", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -645,11 +645,11 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618892/6402965165736931000" }, "references": { - "childConcept": "pnode:100000022@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000022" } }, { - "id": "pnode:10000002e@aRepositoryId", + "id": "modelix:aRepositoryId/10000002e", "concept": "modelix.metameta.ChildLink", "role": "childLinks", "properties": { @@ -659,16 +659,16 @@ class MetaModelMigrationsTest { "uid": "0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618892/5381564949800872334" }, "references": { - "childConcept": "pnode:100000023@aRepositoryId" + "childConcept": "modelix:aRepositoryId/100000023" } }, { - "id": "pnode:100000040@aRepositoryId", + "id": "modelix:aRepositoryId/100000040", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" }, { - "id": "pnode:100000041@aRepositoryId", + "id": "modelix:aRepositoryId/100000041", "concept": "modelix.metameta.ConceptReference", "role": "superConcepts" } @@ -692,7 +692,7 @@ class MetaModelMigrationsTest { """ { "root": { - "id": "pnode:1@aRepositoryId", + "id": "modelix:aRepositoryId/1", "children": [ $metamodelDataSerialized, $modelDataSerialized @@ -704,7 +704,7 @@ class MetaModelMigrationsTest { val branchOnlyWithMetaData = """ { "root": { - "id": "pnode:1@aRepositoryId", + "id": "modelix:aRepositoryId/1", "children": [ $metamodelDataSerialized ] @@ -720,18 +720,18 @@ class MetaModelMigrationsTest { //language=JSON """ { - "id": "pnode:100000042@aRepositoryId", + "id": "modelix:aRepositoryId/100000042", "concept": "100000029", "role": "aChild", "properties": { "id": "myId" }, "references": { - "someRef": "pnode:100000043@aRepositoryId" + "someRef": "modelix:aRepositoryId/100000043" }, "children": [ { - "id": "pnode:100000043@aRepositoryId", + "id": "modelix:aRepositoryId/100000043", "concept": "100000029", "role": "aChild" } @@ -750,22 +750,22 @@ class MetaModelMigrationsTest { """ { "root": { - "id": "pnode:1@aRepositoryId", + "id": "modelix:aRepositoryId/1", "children": [ $metamodelDataSerialized, { - "id": "pnode:100000042@aRepositoryId", + "id": "modelix:aRepositoryId/100000042", "concept": "mps:0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618892", "role": "aChild", "properties": { "id": "myId" }, "references": { - "someRef": "pnode:100000043@aRepositoryId" + "someRef": "modelix:aRepositoryId/100000043" }, "children": [ { - "id": "pnode:100000043@aRepositoryId", + "id": "modelix:aRepositoryId/100000043", "concept": "mps:0a7577d1-d4e5-431d-98b1-fae38f9aee80/474657388638618892", "role": "aChild" } @@ -787,18 +787,18 @@ class MetaModelMigrationsTest { //language=JSON """ { - "id": "pnode:100000042@aRepositoryId", + "id": "modelix:aRepositoryId/100000042", "concept": "100000029", "role": "aChild", "properties": { "id": "myId" }, "references": { - "someRef": "pnode:100000043@aRepositoryId" + "someRef": "modelix:aRepositoryId/100000043" }, "children": [ { - "id": "pnode:100000043@aRepositoryId", + "id": "modelix:aRepositoryId/100000043", "concept": "100000029", "role": "aChild" } From 8f4832bec9eada4389801a96fede28e0a8dd6805 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 19 Feb 2025 13:11:57 +0100 Subject: [PATCH 04/37] build: use absolute path to docker executable At least on Mac with temurin 21 the executable isn't found if no absolute path is used. --- model-client/integration-tests/build.gradle.kts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/model-client/integration-tests/build.gradle.kts b/model-client/integration-tests/build.gradle.kts index 3e93a3e350..57128f63c9 100644 --- a/model-client/integration-tests/build.gradle.kts +++ b/model-client/integration-tests/build.gradle.kts @@ -47,6 +47,10 @@ kotlin { } } +dockerCompose { + dockerExecutable = findExecutableAbsolutePath("docker").also { println("docker: $it") } +} + // The tasks "jsNodeTest" and "jsBrowserTest" are of this type. tasks.withType(KotlinTest::class).all { dockerCompose.isRequiredBy(this) @@ -55,3 +59,12 @@ tasks.withType(KotlinTest::class).all { tasks.withType(Test::class).all { dockerCompose.isRequiredBy(this) } + +fun findExecutableAbsolutePath(name: String): String { + return System.getenv("PATH") + ?.split(File.pathSeparatorChar) + ?.map { File(it).resolve(name) } + ?.firstOrNull { it.isFile && it.exists() } + ?.absolutePath + ?: name +} From f53d79964f4fbc8b3baeedfc6ffce359677e37fa Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 19 Feb 2025 13:23:21 +0100 Subject: [PATCH 05/37] test(authorization): fixed flaky test RSATest.`key file changes are detected` --- .../src/test/kotlin/org/modelix/authorization/RSATest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/authorization/src/test/kotlin/org/modelix/authorization/RSATest.kt b/authorization/src/test/kotlin/org/modelix/authorization/RSATest.kt index 32732af9e7..534e51126b 100644 --- a/authorization/src/test/kotlin/org/modelix/authorization/RSATest.kt +++ b/authorization/src/test/kotlin/org/modelix/authorization/RSATest.kt @@ -287,7 +287,7 @@ class RSATest { privateKeyFile.writeText(privateKeyPem1) val verifyingUtil = ModelixJWTUtil() - verifyingUtil.fileRefreshTime = 50.milliseconds + verifyingUtil.fileRefreshTime = 500.milliseconds verifyingUtil.loadKeysFromFiles(publicKeyFile) run { val signingUtil = ModelixJWTUtil() @@ -314,7 +314,7 @@ class RSATest { assertFailsWith { verifyingUtil.verifyToken(tokenString) } - Thread.sleep(50) + Thread.sleep(500) verifyingUtil.verifyToken(tokenString) } } From 5143b517febae28cb248e67c12068a5bad337beb Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 20 Feb 2025 09:59:36 +0100 Subject: [PATCH 06/37] feat(mps-sync-plugin): persist bindings to .mps/modelix.xml and restore during startup --- .../kotlin/org/modelix/mps/sync3/Binding.kt | 302 +++++++++++++ .../modelix/mps/sync3/IModelSyncService.kt | 4 + .../mps/sync3/ModelSyncForMPSProject.kt | 417 ++++++------------ .../org/modelix/mps/sync3/MPSTestBase.kt | 4 +- .../org/modelix/mps/sync3/ProjectSnapshot.kt | 5 +- .../org/modelix/mps/sync3/ProjectSyncTest.kt | 64 +++ 6 files changed, 507 insertions(+), 289 deletions(-) create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt new file mode 100644 index 0000000000..cb67a27caf --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt @@ -0,0 +1,302 @@ +package org.modelix.mps.sync3 + +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.ModalityState +import jetbrains.mps.project.MPSProject +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import mu.KotlinLogging +import org.jetbrains.mps.openapi.module.SRepository +import org.jetbrains.mps.openapi.project.Project +import org.modelix.model.IVersion +import org.modelix.model.api.TreePointer +import org.modelix.model.api.getRootNode +import org.modelix.model.client2.runWrite +import org.modelix.model.lazy.BranchReference +import org.modelix.model.mpsadapters.MPSRepositoryAsNode +import org.modelix.model.mpsadapters.computeRead +import org.modelix.model.sync.bulk.FullSyncFilter +import org.modelix.model.sync.bulk.InvalidatingVisitor +import org.modelix.model.sync.bulk.InvalidationTree +import org.modelix.model.sync.bulk.ModelSynchronizer +import org.modelix.model.sync.bulk.NodeAssociationFromModelServer +import org.modelix.model.sync.bulk.NodeAssociationToModelServer +import org.modelix.mps.api.ModelixMpsApi +import org.modelix.mps.model.sync.bulk.MPSProjectSyncMask +import java.util.concurrent.atomic.AtomicBoolean + +class Binding( + val coroutinesScope: CoroutineScope, + override val mpsProject: Project, + val serverConnection: ModelSyncService.Connection, + override val branchRef: BranchReference, + val initialVersionHash: String?, +) : IBinding { + companion object { + val LOG = KotlinLogging.logger { } + } + + private val activated = AtomicBoolean(false) + private val lastSyncedVersion = ValueWithMutex(null) + private var syncJob: Job? = null + private var syncToServerTask: ValidatingJob? = null + private var invalidatingListener: MyInvalidatingListener? = null + + private val repository: SRepository get() = mpsProject.repository + private suspend fun client() = serverConnection.getClient() + + fun getCurrentVersionHash(): String? = lastSyncedVersion.getValue()?.getContentHash() + + override fun activate() { + if (activated.getAndSet(true)) return + syncJob = coroutinesScope.launch { syncJob() } + } + + override fun deactivate() { + if (!activated.getAndSet(false)) return + + syncJob?.cancel() + syncJob = null + syncToServerTask = null + invalidatingListener?.stop() + invalidatingListener = null + } + + private suspend fun checkInSync(): String? { + check(activated.get()) { "Binding is deactivated" } + val version = lastSyncedVersion.flush()?.getOrThrow() + if (version == null) return "Initial sync isn't done yet" + if (invalidatingListener == null) return "No change listener registered in MPS" + if (invalidatingListener?.hasAnyInvalidations() != false) return "There are pending changes in MPS" + val remoteVersion = client().pullHash(branchRef) + if (remoteVersion != version.getContentHash()) return "Local version (${version.getContentHash()} differs from remote version ($remoteVersion)" + return null + } + + override suspend fun flush(): IVersion { + check(syncJob?.isActive == true) { "Synchronization is not active" } + var reason = checkInSync() + var i = 0 + while (reason != null) { + i++ + if (i % 10 == 0) LOG.debug { "Still waiting for the synchronization to finish: $reason" } + delay(100) + reason = checkInSync() + } + return lastSyncedVersion.getValue()!! + } + + private suspend fun CoroutineScope.syncJob() { + // initial sync + initialSync() + + // continuous sync to MPS + launchLoop { + val newHash = client().pollHash(branchRef, lastSyncedVersion.getValue()) + if (newHash != lastSyncedVersion.getValue()?.getContentHash()) { + LOG.debug { "New remote version detected: $newHash" } + syncToMPS() + } + } + + // continuous sync to server + syncToServerTask = launchValidation { + syncToServer() + } + } + + suspend fun ensureInitialized() { + if (lastSyncedVersion.getValue() == null) { + initialSync() + } + } + + private suspend fun initialSync() { + lastSyncedVersion.updateValue { oldVersion -> + LOG.debug { "Running initial synchronization" } + + val baseVersion = oldVersion + ?: initialVersionHash?.let { client().loadVersion(branchRef.repositoryId, it, null) } + if (baseVersion == null) { + // Binding was never activated before. Overwrite local changes or do initial upload. + + val remoteVersion = client().pullIfExists(branchRef) + if (remoteVersion == null) { + LOG.debug { "Repository don't exist. Will copy the local project to the server." } + // repository doesn't exist -> copy the local project to the server + val emptyVersion = client().initRepository(branchRef.repositoryId) + doSyncToServer(emptyVersion) ?: emptyVersion + } else { + LOG.debug { "Repository exists. Will checkout version $remoteVersion" } + doSyncToMPS(null, remoteVersion) + remoteVersion + } + } else { + // Binding was activated before. Preserve local changes. + + // push local changes that happened while the binding was deactivated + val localChanges = doSyncFromMPS(baseVersion) + val remoteVersion = if (localChanges != null) { + val mergedVersion = client().push(branchRef, localChanges, baseVersion) + doSyncToMPS(baseVersion, mergedVersion) + mergedVersion + } else { + client().pull(branchRef, baseVersion) + } + + // load remote changes into MPS + doSyncToMPS(baseVersion, remoteVersion) + + remoteVersion + } + } + } + + suspend fun syncToMPS(): IVersion { + return lastSyncedVersion.updateValue { oldVersion -> + client().pull(branchRef, oldVersion).also { newVersion -> + doSyncToMPS(oldVersion, newVersion) + } + } + } + + suspend fun syncToServer(): IVersion? { + return lastSyncedVersion.updateValue { oldVersion -> + if (oldVersion == null) { + // have to wait for initial sync + oldVersion + } else { + val newVersion = doSyncToServer(oldVersion) + newVersion ?: oldVersion + } + } + } + + private suspend fun doSyncToMPS(oldVersion: IVersion?, newVersion: IVersion) { + if (oldVersion?.getContentHash() == newVersion.getContentHash()) return + + LOG.debug { "Updating MPS project from $oldVersion to $newVersion" } + + val mpsProjects = listOf(mpsProject as MPSProject) + val baseVersion = oldVersion + val filter = if (baseVersion != null) { + val invalidationTree = InvalidationTree(100_000) + val newTree = newVersion.getTree() + newTree.visitChanges( + baseVersion.getTree(), + InvalidatingVisitor(newTree, invalidationTree), + ) + invalidationTree + } else { + FullSyncFilter() + } + + val targetRoot = MPSRepositoryAsNode(repository) + writeToMPS { + if (invalidatingListener?.hasAnyInvalidations() == true) { + // Concurrent modification! + // Write changes from MPS to a new version first and try again after it is merged. + LOG.debug { "Skipping sync to MPS because there are pending changes in MPS" } + return@writeToMPS + } + + getMPSListener().runSync { + val branch = TreePointer(newVersion.getTree()) + val nodeAssociation = NodeAssociationFromModelServer(branch, targetRoot.getModel()) + ModelSynchronizer( + filter = filter, + sourceRoot = branch.getRootNode().asWritableNode(), + targetRoot = targetRoot, + nodeAssociation = nodeAssociation, + sourceMask = MPSProjectSyncMask(mpsProjects, false), + targetMask = MPSProjectSyncMask(mpsProjects, true), + ).synchronize() + } + } + } + + private suspend fun writeToMPS(body: () -> R): R { + val result = ArrayList() + ApplicationManager.getApplication().invokeAndWait({ + ApplicationManager.getApplication().runWriteAction { + repository.modelAccess.executeUndoTransparentCommand { + ModelixMpsApi.runWithProject(mpsProject) { + result += body() + } + } + } + }, ModalityState.NON_MODAL) + return result.single() + } + + private fun getMPSListener() = invalidatingListener ?: initializeListener() + + private fun initializeListener(): MyInvalidatingListener { + // Being inside a transaction ensure there are not writes, and we don't lose changes. + repository.modelAccess.checkReadAccess() + check(invalidatingListener == null) + return MyInvalidatingListener().also { + invalidatingListener = it + it.start(repository) + } + } + + /** + * @return null if nothing changed + */ + private suspend fun doSyncFromMPS(oldVersion: IVersion): IVersion? { + check(lastSyncedVersion.isLocked()) + + LOG.debug { "Commiting MPS changes" } + + val mpsProjects = listOf(mpsProject as MPSProject) + val client = client() + val newVersion = repository.computeRead { + fun sync(invalidationTree: ModelSynchronizer.IIncrementalUpdateInformation): IVersion? { + return oldVersion.runWrite(client) { branch -> + ModelixMpsApi.runWithProject(mpsProject) { + ModelSynchronizer( + filter = invalidationTree, + sourceRoot = MPSRepositoryAsNode(ModelixMpsApi.getRepository()), + targetRoot = branch.getRootNode().asWritableNode(), + nodeAssociation = NodeAssociationToModelServer(branch), + sourceMask = MPSProjectSyncMask(mpsProjects, true), + targetMask = MPSProjectSyncMask(mpsProjects, false), + ).synchronize() + } + } + } + + if (invalidatingListener == null) { + sync(FullSyncFilter()).also { + // registering the listener after the sync is sufficient + // because we are in a read action that prevents model changes + initializeListener() + } + } else { + invalidatingListener!!.runSync { sync(it) } + } + } + + LOG.debug { if (newVersion == null) "Nothing changed" else "New version created: $newVersion" } + + return newVersion + } + + /** + * @return null if nothing changed + */ + private suspend fun doSyncToServer(oldVersion: IVersion): IVersion? { + return doSyncFromMPS(oldVersion)?.let { + client().push(branchRef, it, oldVersion) + } + } + + private inner class MyInvalidatingListener : MPSInvalidatingListener(repository) { + override fun onInvalidation() { + syncToServerTask?.invalidate() + } + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt index 8ad66ba2d3..f414595e2a 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt @@ -4,6 +4,7 @@ import com.intellij.openapi.components.service import jetbrains.mps.ide.project.ProjectHelper import kotlinx.coroutines.runBlocking import org.modelix.model.IVersion +import org.modelix.model.client2.IModelClientV2 import org.modelix.model.lazy.BranchReference import java.io.Closeable @@ -36,6 +37,9 @@ interface IServerConnection { fun bind(branchRef: BranchReference, lastSyncedVersionHash: String?): IBinding fun getBindings(): List + fun getUrl(): String + suspend fun getClient(): IModelClientV2 + enum class Status { CONNECTED, DISCONNECTED, diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt index 2534c062b9..df074ec7c0 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt @@ -2,43 +2,31 @@ package org.modelix.mps.sync3 +import com.intellij.configurationStore.Property import com.intellij.openapi.Disposable import com.intellij.openapi.application.ApplicationManager -import com.intellij.openapi.application.ModalityState +import com.intellij.openapi.components.PersistentStateComponent import com.intellij.openapi.components.Service +import com.intellij.openapi.components.State +import com.intellij.openapi.components.Storage import com.intellij.openapi.project.Project import jetbrains.mps.ide.project.ProjectHelper import jetbrains.mps.project.MPSProject import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import org.jetbrains.mps.openapi.module.SRepository +import org.jdom.Element import org.modelix.model.IVersion -import org.modelix.model.api.TreePointer -import org.modelix.model.api.getRootNode import org.modelix.model.client2.IModelClientV2 import org.modelix.model.client2.ModelClientV2 -import org.modelix.model.client2.runWrite import org.modelix.model.lazy.BranchReference -import org.modelix.model.mpsadapters.MPSRepositoryAsNode -import org.modelix.model.mpsadapters.computeRead -import org.modelix.model.sync.bulk.FullSyncFilter -import org.modelix.model.sync.bulk.InvalidatingVisitor -import org.modelix.model.sync.bulk.InvalidationTree -import org.modelix.model.sync.bulk.ModelSynchronizer -import org.modelix.model.sync.bulk.ModelSynchronizer.IIncrementalUpdateInformation -import org.modelix.model.sync.bulk.NodeAssociationFromModelServer -import org.modelix.model.sync.bulk.NodeAssociationToModelServer -import org.modelix.mps.api.ModelixMpsApi -import org.modelix.mps.model.sync.bulk.MPSProjectSyncMask +import org.modelix.model.lazy.RepositoryId import org.modelix.mps.sync3.Binding.Companion.LOG -import java.util.concurrent.atomic.AtomicBoolean import kotlin.math.roundToLong import kotlin.time.ExperimentalTime @@ -65,6 +53,9 @@ class AppLevelModelSyncService() : Disposable { } } + @Synchronized + fun getConnections() = synchronized(connections) { connections.values.toList() } + @Synchronized fun addConnection(url: String): ServerConnection { return synchronized(connections) { connections.getOrPut(url) { ServerConnection(url) } } @@ -96,7 +87,11 @@ class AppLevelModelSyncService() : Disposable { } @Service(Service.Level.PROJECT) -class ModelSyncService(val project: Project) : IModelSyncService, Disposable { +@State(name = "modelix-sync", storages = [Storage(value = "modelix.xml")]) +class ModelSyncService(val project: Project) : + IModelSyncService, + Disposable, + PersistentStateComponent { private val mpsProject: MPSProject get() = ProjectHelper.fromIdeaProject(project)!! private val bindings = ArrayList() @@ -109,7 +104,7 @@ class ModelSyncService(val project: Project) : IModelSyncService, Disposable { @Synchronized override fun getServerConnections(): List { - TODO("Not yet implemented") + return AppLevelModelSyncService.getInstance().getConnections().map { Connection(it) } } @Synchronized @@ -118,7 +113,113 @@ class ModelSyncService(val project: Project) : IModelSyncService, Disposable { coroutinesScope.cancel("disposed") } + @Synchronized + override fun getState(): Element? { + return State( + bindings = bindings.map { + BindingState( + url = it.serverConnection.getUrl(), + repository = it.branchRef.repositoryId.id, + branch = it.branchRef.branchName, + versionHash = it.getCurrentVersionHash(), + ) + }, + ).toXml() + // Returning XML seems to be the most reliable way to get the state actually persisted. + // Letting IntelliJ serialize the state sometimes fails silently. + // Using kotlin.serialization is difficult because of version conflicts. + } + + @Synchronized + override fun loadState(state: Element) { + val state = State.fromXml(state) + val statesById = state.bindings.associateBy { it.getId() } + val bindingsById = bindings.associateBy { it.getState().getId() } + val allBindingIds = statesById.keys + bindingsById.keys + + for (id in allBindingIds) { + val state = statesById[id] + val binding = bindingsById[id] + if (state == null) { + if (binding == null) { + // unreachable + } else { + binding.deactivate() + bindings.remove(binding) + } + } else { + if (binding == null) { + loadBinding(state) + } else { + if (binding.initialVersionHash != state.versionHash && binding.getCurrentVersionHash() != state.versionHash) { + binding.deactivate() + bindings.remove(binding) + loadBinding(state) + } + } + } + } + } + + private fun loadBinding(state: BindingState) { + addServer(state.url ?: return).bind( + branchRef = RepositoryId(state.repository ?: return).getBranchReference(state.branch), + lastSyncedVersionHash = state.versionHash, + ) + } + + private fun Binding.getState() = BindingState( + url = serverConnection.getUrl(), + repository = branchRef.repositoryId.id, + branch = branchRef.branchName, + versionHash = getCurrentVersionHash(), + ) + + data class State( + @Property + val bindings: List = emptyList(), + ) { + fun toXml() = Element("model-sync").also { + it.children.addAll(bindings.map { it.toXml() }) + } + companion object { + fun fromXml(element: Element) = State(element.getChildren("binding").map { BindingState.fromXml(it) }) + } + } + + data class BindingState( + val url: String? = null, + val repository: String? = null, + val branch: String? = null, + val versionHash: String? = null, + ) { + fun getId(): Any = copy(versionHash = null) + fun toXml() = Element("binding").also { + it.children.add(Element("url").also { it.text = url }) + it.children.add(Element("repository").also { it.text = repository }) + it.children.add(Element("branch").also { it.text = branch }) + it.children.add(Element("versionHash").also { it.text = versionHash }) + } + companion object { + fun fromXml(element: Element) = BindingState( + // separate elements instead of attributes so that each value has its own line in the .xml file + element.getChild("url")?.text, + element.getChild("repository")?.text, + element.getChild("branch")?.text, + element.getChild("versionHash")?.text, + ) + } + } + inner class Connection(val connection: AppLevelModelSyncService.ServerConnection) : IServerConnection { + override fun getUrl(): String { + return connection.url + } + + override suspend fun getClient(): IModelClientV2 { + return connection.getClient() + } + override fun activate() { TODO("Not yet implemented") } @@ -143,7 +244,7 @@ class ModelSyncService(val project: Project) : IModelSyncService, Disposable { val binding = Binding( coroutinesScope = coroutinesScope, mpsProject = mpsProject, - client = { connection.getClient() }, + serverConnection = this, branchRef = branchRef, initialVersionHash = lastSyncedVersionHash, ) @@ -153,278 +254,20 @@ class ModelSyncService(val project: Project) : IModelSyncService, Disposable { } override fun getBindings(): List { - TODO("Not yet implemented") - } - } -} - -class Binding( - val coroutinesScope: CoroutineScope, - override val mpsProject: org.jetbrains.mps.openapi.project.Project, - val client: suspend () -> IModelClientV2, - override val branchRef: BranchReference, - val initialVersionHash: String?, -) : IBinding { - companion object { - val LOG = mu.KotlinLogging.logger { } - } - - private val activated = AtomicBoolean(false) - private val lastSyncedVersion = ValueWithMutex(null) - private var syncJob: Job? = null - private var syncToServerTask: ValidatingJob? = null - private var invalidatingListener: MyInvalidatingListener? = null - - private val repository: SRepository get() = mpsProject.repository - - override fun activate() { - if (activated.getAndSet(true)) return - syncJob = coroutinesScope.launch { syncJob() } - } - - override fun deactivate() { - if (!activated.getAndSet(false)) return - - syncJob?.cancel() - syncJob = null - syncToServerTask = null - invalidatingListener?.stop() - invalidatingListener = null - } - - private suspend fun checkInSync(): String? { - check(activated.get()) { "Binding is deactivated" } - val version = lastSyncedVersion.flush()?.getOrThrow() - if (version == null) return "Initial sync isn't done yet" - if (invalidatingListener == null) return "No change listener registered in MPS" - if (invalidatingListener?.hasAnyInvalidations() != false) return "There are pending changes in MPS" - val remoteVersion = client().pullHash(branchRef) - if (remoteVersion != version.getContentHash()) return "Local version (${version.getContentHash()} differs from remote version ($remoteVersion)" - return null - } - - override suspend fun flush(): IVersion { - check(syncJob?.isActive == true) { "Synchronization is not active" } - var reason = checkInSync() - var i = 0 - while (reason != null) { - i++ - if (i % 10 == 0) LOG.debug { "Still waiting for the synchronization to finish: $reason" } - delay(100) - reason = checkInSync() - } - return lastSyncedVersion.getValue()!! - } - - private suspend fun CoroutineScope.syncJob() { - // initial sync - initialSync() - - // continuous sync to MPS - launchLoop { - val newHash = client().pollHash(branchRef, lastSyncedVersion.getValue()) - if (newHash != lastSyncedVersion.getValue()?.getContentHash()) { - LOG.debug { "New remote version detected: $newHash" } - syncToMPS() - } - } - - // continuous sync to server - syncToServerTask = launchValidation { - syncToServer() - } - } - - suspend fun ensureInitialized() { - if (lastSyncedVersion.getValue() == null) { - initialSync() - } - } - - private suspend fun initialSync() { - lastSyncedVersion.updateValue { oldVersion -> - LOG.debug { "Running initial synchronization" } - - val baseVersion = oldVersion - ?: initialVersionHash?.let { client().loadVersion(branchRef.repositoryId, it, null) } - if (baseVersion == null) { - // Binding was never activated before. Overwrite local changes or do initial upload. - - val remoteVersion = client().pullIfExists(branchRef) - if (remoteVersion == null) { - LOG.debug { "Repository don't exist. Will copy the local project to the server." } - // repository doesn't exist -> copy the local project to the server - val emptyVersion = client().initRepository(branchRef.repositoryId) - doSyncToServer(emptyVersion) ?: emptyVersion - } else { - LOG.debug { "Repository exists. Will checkout version $remoteVersion" } - doSyncToMPS(null, remoteVersion) - remoteVersion - } - } else { - // Binding was activated before. Preserve local changes. - - // push local changes that happened while the binding was deactivated - val localChanges = doSyncFromMPS(baseVersion) - val remoteVersion = if (localChanges != null) { - val mergedVersion = client().push(branchRef, localChanges, baseVersion) - doSyncToMPS(baseVersion, mergedVersion) - mergedVersion - } else { - client().pull(branchRef, baseVersion) - } - - // load remote changes into MPS - doSyncToMPS(baseVersion, remoteVersion) - - remoteVersion - } - } - } - - suspend fun syncToMPS(): IVersion { - return lastSyncedVersion.updateValue { oldVersion -> - client().pull(branchRef, oldVersion).also { newVersion -> - doSyncToMPS(oldVersion, newVersion) - } - } - } - - suspend fun syncToServer(): IVersion? { - return lastSyncedVersion.updateValue { oldVersion -> - if (oldVersion == null) { - // have to wait for initial sync - oldVersion - } else { - val newVersion = doSyncToServer(oldVersion) - newVersion ?: oldVersion - } - } - } - - private suspend fun doSyncToMPS(oldVersion: IVersion?, newVersion: IVersion) { - if (oldVersion?.getContentHash() == newVersion.getContentHash()) return - - LOG.debug { "Updating MPS project from $oldVersion to $newVersion" } - - val mpsProjects = listOf(mpsProject as MPSProject) - val baseVersion = oldVersion - val filter = if (baseVersion != null) { - val invalidationTree = InvalidationTree(100_000) - val newTree = newVersion.getTree() - newTree.visitChanges( - baseVersion.getTree(), - InvalidatingVisitor(newTree, invalidationTree), - ) - invalidationTree - } else { - FullSyncFilter() + return bindings.filter { it.serverConnection == this } } - val targetRoot = MPSRepositoryAsNode(repository) - writeToMPS { - if (invalidatingListener?.hasAnyInvalidations() == true) { - // Concurrent modification! - // Write changes from MPS to a new version first and try again after it is merged. - LOG.debug { "Skipping sync to MPS because there are pending changes in MPS" } - return@writeToMPS - } + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - getMPSListener().runSync { - val branch = TreePointer(newVersion.getTree()) - val nodeAssociation = NodeAssociationFromModelServer(branch, targetRoot.getModel()) - ModelSynchronizer( - filter = filter, - sourceRoot = branch.getRootNode().asWritableNode(), - targetRoot = targetRoot, - nodeAssociation = nodeAssociation, - sourceMask = MPSProjectSyncMask(mpsProjects, false), - targetMask = MPSProjectSyncMask(mpsProjects, true), - ).synchronize() - } - } - } + other as Connection - private suspend fun writeToMPS(body: () -> R): R { - val result = ArrayList() - ApplicationManager.getApplication().invokeAndWait({ - ApplicationManager.getApplication().runWriteAction { - repository.modelAccess.executeUndoTransparentCommand { - ModelixMpsApi.runWithProject(mpsProject) { - result += body() - } - } - } - }, ModalityState.NON_MODAL) -// withContext(Dispatchers.Main) { -// } - return result.single() - } - - private fun getMPSListener() = invalidatingListener ?: initializeListener() - - private fun initializeListener(): MyInvalidatingListener { - // Being inside a transaction ensure there are not writes, and we don't lose changes. - repository.modelAccess.checkReadAccess() - check(invalidatingListener == null) - return MyInvalidatingListener().also { - invalidatingListener = it - it.start(repository) + return connection == other.connection } - } - - /** - * @return null if nothing changed - */ - private suspend fun doSyncFromMPS(oldVersion: IVersion): IVersion? { - check(lastSyncedVersion.isLocked()) - - LOG.debug { "Commiting MPS changes" } - - val mpsProjects = listOf(mpsProject as MPSProject) - val client = client() - val newVersion = repository.computeRead { - fun sync(invalidationTree: IIncrementalUpdateInformation): IVersion? { - return oldVersion.runWrite(client) { branch -> - ModelixMpsApi.runWithProject(mpsProject) { - ModelSynchronizer( - filter = invalidationTree, - sourceRoot = MPSRepositoryAsNode(ModelixMpsApi.getRepository()), - targetRoot = branch.getRootNode().asWritableNode(), - nodeAssociation = NodeAssociationToModelServer(branch), - sourceMask = MPSProjectSyncMask(mpsProjects, true), - targetMask = MPSProjectSyncMask(mpsProjects, false), - ).synchronize() - } - } - } - - if (invalidatingListener == null) { - sync(FullSyncFilter()).also { - // registering the listener after the sync is sufficient - // because we are in a read action that prevents model changes - initializeListener() - } - } else { - invalidatingListener!!.runSync { sync(it) } - } - } - - LOG.debug { if (newVersion == null) "Nothing changed" else "New version created: $newVersion" } - - return newVersion - } - - /** - * @return null if nothing changed - */ - private suspend fun doSyncToServer(oldVersion: IVersion): IVersion? { - return doSyncFromMPS(oldVersion)?.let { client().push(branchRef, it, oldVersion) } - } - private inner class MyInvalidatingListener : MPSInvalidatingListener(repository) { - override fun onInvalidation() { - syncToServerTask?.invalidate() + override fun hashCode(): Int { + return connection.hashCode() } } } diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt index 5539bc70c8..72a499a13d 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt @@ -29,7 +29,7 @@ abstract class MPSTestBase : UsefulTestCase() { override fun runInDispatchThread() = false @OptIn(ExperimentalPathApi::class) - fun openTestProject(testDataName: String?): Project { + fun openTestProject(testDataName: String?, beforeOpen: (projectDir: Path) -> Unit = {}): Project { val projectDirParent = Path.of("build", "test-projects").absolute() projectDirParent.toFile().mkdirs() val projectDir = Files.createTempDirectory(projectDirParent, "mps-project") @@ -40,6 +40,7 @@ abstract class MPSTestBase : UsefulTestCase() { val project = if (testDataName != null) { val sourceDir = File("testdata/$testDataName") sourceDir.copyRecursively(projectDir.toFile(), overwrite = true) + beforeOpen(projectDir) ProjectManagerEx.getInstanceEx().openProject(projectDir, options)!! } else { projectDir.resolve(".mps").also { it.toFile().mkdirs() }.resolve("modules.xml").writeText( @@ -53,6 +54,7 @@ abstract class MPSTestBase : UsefulTestCase() { """.trimIndent(), ) + beforeOpen(projectDir) ProjectManagerEx.getInstanceEx().openProject(projectDir, options)!! } diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt index 6614a6ac85..8e87dd3bc4 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt @@ -28,7 +28,10 @@ private fun Map.contentsAsString(): String { private fun filterFiles(files: Map) = files.filter { val name = it.key if (name.startsWith(".mps/")) { - name == ".mps/modules.xml" + when (name.substringAfter("/")) { + ".gitignore", "migration.xml", "workspace.xml", "modelix.xml" -> false + else -> true + } } else if (name.contains("/source_gen") || name.contains("/classes_gen")) { false } else { diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index 3dc2fc4e2c..48ae0cbb6c 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -1,6 +1,7 @@ package org.modelix.mps.sync3 import com.badoo.reaktive.observable.toList +import com.intellij.configurationStore.saveSettings import com.intellij.testFramework.TestApplicationManager import jetbrains.mps.smodel.SNodeUtil import kotlinx.coroutines.runBlocking @@ -30,6 +31,8 @@ import org.testcontainers.images.builder.ImageFromDockerfile import java.nio.file.Path import java.util.concurrent.atomic.AtomicLong import kotlin.io.path.absolute +import kotlin.io.path.readText +import kotlin.io.path.writeText import kotlin.time.Duration.Companion.minutes import kotlin.time.toJavaDuration @@ -299,6 +302,67 @@ class ProjectSyncTest : MPSTestBase() { assertEquals(expectedSnapshot, project.captureSnapshot()) } + fun `test loading persisted binding`(): Unit = runWithModelServer { port -> + // The client is in sync ... + val branchRef = RepositoryId("sync-test").getBranchReference() + val version1 = syncProjectToServer("initial", port, branchRef) + + // ... and then closes the project while some other client continues making changes. + val version2 = syncProjectToServer("change1", port, branchRef, version1.getContentHash()) + val expectedSnapshot = lastSnapshotBeforeSync + + // Then the client opens the project again and reconnects using the persisted binding information. + openTestProject("initial") { projectDir -> + projectDir.resolve(".mps").resolve("modelix.xml").writeText( + """ + + + + + http://localhost:$port + ${branchRef.repositoryId.id} + ${branchRef.branchName} + ${version1.getContentHash()} + + + + """.trimIndent(), + ) + } + + val binding = IModelSyncService.getInstance(mpsProject).getServerConnections().flatMap { it.getBindings() }.single() + assertEquals(branchRef, binding.branchRef) + val version3 = binding.flush() + + assertEquals(version2.getContentHash(), version3.getContentHash()) + + // ... applies all the pending changes and is again in sync with the other client + assertEquals(expectedSnapshot, project.captureSnapshot()) + } + + fun `test storing persisted binding`(): Unit = runWithModelServer { port -> + val branchRef = RepositoryId("sync-test").getBranchReference() + openTestProject(null) + val binding = IModelSyncService.getInstance(project).addServer("http://localhost:$port").bind(branchRef) + val version1 = binding.flush() + saveSettings(project, true) + val actual = Path.of(project.basePath).resolve(".mps").resolve("modelix.xml").readText() + val expected = """ + + + + + http://localhost:$port + ${branchRef.repositoryId.id} + ${branchRef.branchName} + ${version1.getContentHash()} + + + + """.trimIndent() + assertEquals(expected, actual) + } + private fun runWithModelServer(body: suspend (port: Int) -> Unit) = runBlocking { withTimeout(3.minutes) { val modelServer: GenericContainer<*> = GenericContainer(modelServerImage) From bdadaaf8def99c7169899bd104f9676fa88971c0 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 20 Feb 2025 10:47:25 +0100 Subject: [PATCH 07/37] fix(model-datastructure): deserialization failed after addNewChildren with empty list of children --- .../org/modelix/model/sync/bulk/AbstractModelSyncTest.kt | 2 -- .../org/modelix/model/operations/OTWriteTransaction.kt | 8 +++----- .../org/modelix/model/persistent/OperationSerializer.kt | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/AbstractModelSyncTest.kt b/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/AbstractModelSyncTest.kt index 1e37c4558e..bdd2302a2b 100644 --- a/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/AbstractModelSyncTest.kt +++ b/bulk-model-sync-lib/src/commonTest/kotlin/org/modelix/model/sync/bulk/AbstractModelSyncTest.kt @@ -10,7 +10,6 @@ import org.modelix.model.data.ModelData import org.modelix.model.lazy.CLTree import org.modelix.model.lazy.ObjectStoreCache import org.modelix.model.operations.AddNewChildOp -import org.modelix.model.operations.AddNewChildrenOp import org.modelix.model.operations.DeleteNodeOp import org.modelix.model.operations.IOperation import org.modelix.model.operations.MoveNodeOp @@ -581,7 +580,6 @@ abstract class AbstractModelSyncTest { SetReferenceOp::class to 3, MoveNodeOp::class to 6, // could be done in 5, but finding that optimization makes the sync algorithm slower AddNewChildOp::class to 1, - AddNewChildrenOp::class to 1, DeleteNodeOp::class to 1, ) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/OTWriteTransaction.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/OTWriteTransaction.kt index bdad9d2f3a..7de6055293 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/OTWriteTransaction.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/OTWriteTransaction.kt @@ -69,11 +69,9 @@ class OTWriteTransaction( childIds: LongArray, concepts: Array, ) { - var index_ = index - if (index_ == -1) { - index_ = getChildren(parentId, role).count() - } - apply(AddNewChildrenOp(PositionInRole(parentId, role, index_), childIds, concepts)) + if (childIds.isEmpty()) return + val index = if (index != -1) index else getChildren(parentId, role).count() + apply(AddNewChildrenOp(PositionInRole(parentId, role, index), childIds, concepts)) } override fun addNewChild(parentId: Long, role: String?, index: Int, childId: Long, concept: IConcept?) { diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt index 243d075d52..dcc4e9b0fd 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/persistent/OperationSerializer.kt @@ -98,7 +98,7 @@ class OperationSerializer private constructor() { override fun deserialize(serialized: String): AddNewChildrenOp { val parts = serialized.split(SEPARATOR).toTypedArray() - val ids = parts[3].split(Separators.LEVEL4).map { longFromHex(it) }.toLongArray() + val ids = parts[3].split(Separators.LEVEL4).filter { it.isNotEmpty() }.map { longFromHex(it) }.toLongArray() val concepts = parts[4].split(Separators.LEVEL4).map { deserializeConcept(it) }.toTypedArray() return AddNewChildrenOp(PositionInRole(longFromHex(parts[0]), unescape(parts[1]), parts[2].toInt()), ids, concepts) } From 2f6cdf8f2ac0a9a6d8a86d05918719d8e3a812f0 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 20 Feb 2025 12:47:42 +0100 Subject: [PATCH 08/37] chore(mps-sync-plugin): move classes to separate files --- .../mps/sync3/AppLevelModelSyncService.kt | 66 +++++++++++ .../org/modelix/mps/sync3/BackoffStrategy.kt | 24 ++++ .../modelix/mps/sync3/IModelSyncService.kt | 4 +- ...ncForMPSProject.kt => ModelSyncService.kt} | 112 ------------------ .../org/modelix/mps/sync3/ValueWithMutex.kt | 34 ++++++ 5 files changed, 127 insertions(+), 113 deletions(-) create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BackoffStrategy.kt rename mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/{ModelSyncForMPSProject.kt => ModelSyncService.kt} (71%) create mode 100644 mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt new file mode 100644 index 0000000000..860c7edb42 --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt @@ -0,0 +1,66 @@ +package org.modelix.mps.sync3 + +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.components.Service +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancel +import org.modelix.model.client2.IModelClientV2 +import org.modelix.model.client2.ModelClientV2 + +@Service(Service.Level.APP) +class AppLevelModelSyncService() : Disposable { + + companion object { + fun getInstance(): AppLevelModelSyncService { + return ApplicationManager.getApplication().getService(AppLevelModelSyncService::class.java) + } + } + + private val connections = LinkedHashMap() + private val coroutinesScope = CoroutineScope(Dispatchers.Default) + private val connectionCheckingJob = coroutinesScope.launchLoop( + BackoffStrategy( + initialDelay = 3_000, + maxDelay = 10_000, + factor = 1.2, + ), + ) { + for (connection in synchronized(connections) { connections.values.toList() }) { + connection.checkConnection() + } + } + + @Synchronized + fun getConnections() = synchronized(connections) { connections.values.toList() } + + @Synchronized + fun addConnection(url: String): ServerConnection { + return synchronized(connections) { connections.getOrPut(url) { ServerConnection(url) } } + } + + override fun dispose() { + coroutinesScope.cancel("disposed") + } + + class ServerConnection(val url: String) { + private var client: ValueWithMutex = ValueWithMutex(null) + private var connected: Boolean = false + + suspend fun getClient(): IModelClientV2 { + return client.getValue() ?: client.updateValue { + it ?: ModelClientV2.Companion.builder().url(url).build().also { it.init() } + } + } + + suspend fun checkConnection() { + try { + getClient().getServerId() + connected = true + } catch (ex: Throwable) { + connected = false + } + } + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BackoffStrategy.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BackoffStrategy.kt new file mode 100644 index 0000000000..148e6e51cc --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BackoffStrategy.kt @@ -0,0 +1,24 @@ +package org.modelix.mps.sync3 + +import kotlinx.coroutines.delay +import kotlin.math.roundToLong + +class BackoffStrategy( + val initialDelay: Long = 500, + val maxDelay: Long = 10_000, + val factor: Double = 1.5, +) { + var currentDelay: Long = initialDelay + + fun failed() { + currentDelay = (currentDelay * factor).roundToLong().coerceAtMost(maxDelay) + } + + fun success() { + currentDelay = initialDelay + } + + suspend fun wait() { + delay(currentDelay) + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt index f414595e2a..742efc9b8f 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt @@ -25,12 +25,14 @@ interface IModelSyncService { fun getServerConnections(): List } -interface IServerConnection { +interface IServerConnection : Closeable { fun activate() fun deactivate() fun remove() fun getStatus(): Status + override fun close() = deactivate() + suspend fun pullVersion(branchRef: BranchReference): IVersion fun bind(branchRef: BranchReference): IBinding = bind(branchRef, null) diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt similarity index 71% rename from mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt rename to mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt index df074ec7c0..170c60e878 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncForMPSProject.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt @@ -4,7 +4,6 @@ package org.modelix.mps.sync3 import com.intellij.configurationStore.Property import com.intellij.openapi.Disposable -import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.components.PersistentStateComponent import com.intellij.openapi.components.Service import com.intellij.openapi.components.State @@ -16,76 +15,15 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock import org.jdom.Element import org.modelix.model.IVersion import org.modelix.model.client2.IModelClientV2 -import org.modelix.model.client2.ModelClientV2 import org.modelix.model.lazy.BranchReference import org.modelix.model.lazy.RepositoryId import org.modelix.mps.sync3.Binding.Companion.LOG -import kotlin.math.roundToLong import kotlin.time.ExperimentalTime -@Service(Service.Level.APP) -class AppLevelModelSyncService() : Disposable { - - companion object { - fun getInstance(): AppLevelModelSyncService { - return ApplicationManager.getApplication().getService(AppLevelModelSyncService::class.java) - } - } - - private val connections = LinkedHashMap() - private val coroutinesScope = CoroutineScope(Dispatchers.Default) - private val connectionCheckingJob = coroutinesScope.launchLoop( - BackoffStrategy( - initialDelay = 3_000, - maxDelay = 10_000, - factor = 1.2, - ), - ) { - for (connection in synchronized(connections) { connections.values.toList() }) { - connection.checkConnection() - } - } - - @Synchronized - fun getConnections() = synchronized(connections) { connections.values.toList() } - - @Synchronized - fun addConnection(url: String): ServerConnection { - return synchronized(connections) { connections.getOrPut(url) { ServerConnection(url) } } - } - - override fun dispose() { - coroutinesScope.cancel("disposed") - } - - class ServerConnection(val url: String) { - private var client: ValueWithMutex = ValueWithMutex(null) - private var connected: Boolean = false - - suspend fun getClient(): IModelClientV2 { - return client.getValue() ?: client.updateValue { - it ?: ModelClientV2.builder().url(url).build().also { it.init() } - } - } - - suspend fun checkConnection() { - try { - getClient().getServerId() - connected = true - } catch (ex: Throwable) { - connected = false - } - } - } -} - @Service(Service.Level.PROJECT) @State(name = "modelix-sync", storages = [Storage(value = "modelix.xml")]) class ModelSyncService(val project: Project) : @@ -294,53 +232,3 @@ suspend fun jobLoop( } } } - -class BackoffStrategy( - val initialDelay: Long = 500, - val maxDelay: Long = 10_000, - val factor: Double = 1.5, -) { - var currentDelay: Long = initialDelay - - fun failed() { - currentDelay = (currentDelay * factor).roundToLong().coerceAtMost(maxDelay) - } - - fun success() { - currentDelay = initialDelay - } - - suspend fun wait() { - delay(currentDelay) - } -} - -class ValueWithMutex(private var value: E) { - private val mutex = Mutex() - private var lastUpdateResult: Result? = null - - suspend fun updateValue(body: suspend (E) -> R): R { - return mutex.withLock { - val newValue = runCatching { - body(value) - } - lastUpdateResult = newValue - newValue.onFailure { - LOG.error(it) { "Value update failed. Keeping $value" } - } - newValue.getOrThrow().also { value = it } - } - } - - /** - * Blocks until any active update is done. - * @return The result of the most recent update attempt. - */ - suspend fun flush(): Result? { - return mutex.withLock { lastUpdateResult } - } - - fun isLocked() = mutex.isLocked - - fun getValue(): E = value -} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt new file mode 100644 index 0000000000..71fec5eca2 --- /dev/null +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt @@ -0,0 +1,34 @@ +package org.modelix.mps.sync3 + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +class ValueWithMutex(private var value: E) { + private val mutex = Mutex() + private var lastUpdateResult: Result? = null + + suspend fun updateValue(body: suspend (E) -> R): R { + return mutex.withLock { + val newValue = runCatching { + body(value) + } + lastUpdateResult = newValue + newValue.onFailure { + Binding.Companion.LOG.error(it) { "Value update failed. Keeping $value" } + } + newValue.getOrThrow().also { value = it } + } + } + + /** + * Blocks until any active update is done. + * @return The result of the most recent update attempt. + */ + suspend fun flush(): Result? { + return mutex.withLock { lastUpdateResult } + } + + fun isLocked() = mutex.isLocked + + fun getValue(): E = value +} From b7bc5e4e47f67a9656c43c28c47671d5545a985f Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 20 Feb 2025 16:46:05 +0100 Subject: [PATCH 09/37] feat(mps-sync-plugin): handle disabled bindings when loading from modelix.xml --- .../modelix/model/client2/ModelClientV2.kt | 14 +- mps-sync-plugin3/build.gradle.kts | 1 + .../sync3/{Binding.kt => BindingWorker.kt} | 15 +- .../modelix/mps/sync3/IModelSyncService.kt | 13 +- .../org/modelix/mps/sync3/ModelSyncService.kt | 267 +++++++++++++----- .../org/modelix/mps/sync3/ValueWithMutex.kt | 2 +- .../org/modelix/mps/sync3/ProjectSyncTest.kt | 32 ++- 7 files changed, 241 insertions(+), 103 deletions(-) rename mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/{Binding.kt => BindingWorker.kt} (98%) diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt index 36795496d0..092887a9ee 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt @@ -508,11 +508,7 @@ abstract class ModelClientV2Builder { } fun url(url: String): ModelClientV2Builder { - baseUrl = buildUrl { - takeFrom(url) - if (pathSegments.lastOrNull() == "") pathSegments = pathSegments.dropLast(1) - if (pathSegments.lastOrNull() != "v2") appendPathSegments("v2") - }.toString() + baseUrl = normalizeUrl(url) return this } @@ -586,6 +582,14 @@ abstract class ModelClientV2Builder { companion object { private val LOG = mu.KotlinLogging.logger {} + + fun normalizeUrl(url: String): String { + return buildUrl { + takeFrom(url) + if (pathSegments.lastOrNull() == "") pathSegments = pathSegments.dropLast(1) + if (pathSegments.lastOrNull() != "v2") appendPathSegments("v2") + }.toString() + } } } diff --git a/mps-sync-plugin3/build.gradle.kts b/mps-sync-plugin3/build.gradle.kts index 2c3842f6a0..f00835beb3 100644 --- a/mps-sync-plugin3/build.gradle.kts +++ b/mps-sync-plugin3/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { testImplementation(libs.testcontainers) testImplementation(libs.kotlin.coroutines.test) testImplementation(libs.logback.classic) + testImplementation(kotlin("test")) } tasks { diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt similarity index 98% rename from mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt rename to mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt index cb67a27caf..4cbbeaec5c 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/Binding.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt @@ -27,13 +27,13 @@ import org.modelix.mps.api.ModelixMpsApi import org.modelix.mps.model.sync.bulk.MPSProjectSyncMask import java.util.concurrent.atomic.AtomicBoolean -class Binding( +class BindingWorker( val coroutinesScope: CoroutineScope, - override val mpsProject: Project, + val mpsProject: Project, val serverConnection: ModelSyncService.Connection, - override val branchRef: BranchReference, + val branchRef: BranchReference, val initialVersionHash: String?, -) : IBinding { +) { companion object { val LOG = KotlinLogging.logger { } } @@ -48,13 +48,14 @@ class Binding( private suspend fun client() = serverConnection.getClient() fun getCurrentVersionHash(): String? = lastSyncedVersion.getValue()?.getContentHash() + fun isActive(): Boolean = activated.get() - override fun activate() { + fun activate() { if (activated.getAndSet(true)) return syncJob = coroutinesScope.launch { syncJob() } } - override fun deactivate() { + fun deactivate() { if (!activated.getAndSet(false)) return syncJob?.cancel() @@ -75,7 +76,7 @@ class Binding( return null } - override suspend fun flush(): IVersion { + suspend fun flush(): IVersion { check(syncJob?.isActive == true) { "Synchronization is not active" } var reason = checkInSync() var i = 0 diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt index 742efc9b8f..c8a5245d68 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt @@ -23,6 +23,7 @@ interface IModelSyncService { fun addServer(url: String): IServerConnection fun getServerConnections(): List + fun getBindings(): List } interface IServerConnection : Closeable { @@ -49,12 +50,14 @@ interface IServerConnection : Closeable { } interface IBinding : Closeable { - val mpsProject: org.jetbrains.mps.openapi.project.Project - val branchRef: BranchReference - fun activate() - fun deactivate() + fun getProject(): org.jetbrains.mps.openapi.project.Project + fun getBranchRef(): BranchReference + fun isEnabled(): Boolean + fun enable() + fun disable() + fun delete() - override fun close() = deactivate() + override fun close() = disable() /** * Blocks until both ends are in sync. diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt index 170c60e878..363e734cb1 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt @@ -2,7 +2,6 @@ package org.modelix.mps.sync3 -import com.intellij.configurationStore.Property import com.intellij.openapi.Disposable import com.intellij.openapi.components.PersistentStateComponent import com.intellij.openapi.components.Service @@ -21,9 +20,10 @@ import org.modelix.model.IVersion import org.modelix.model.client2.IModelClientV2 import org.modelix.model.lazy.BranchReference import org.modelix.model.lazy.RepositoryId -import org.modelix.mps.sync3.Binding.Companion.LOG import kotlin.time.ExperimentalTime +private val LOG = mu.KotlinLogging.logger { } + @Service(Service.Level.PROJECT) @State(name = "modelix-sync", storages = [Storage(value = "modelix.xml")]) class ModelSyncService(val project: Project) : @@ -32,11 +32,12 @@ class ModelSyncService(val project: Project) : PersistentStateComponent { private val mpsProject: MPSProject get() = ProjectHelper.fromIdeaProject(project)!! - private val bindings = ArrayList() + private var loadedState: SyncServiceState = SyncServiceState() + private val workers = LinkedHashMap() private val coroutinesScope = CoroutineScope(Dispatchers.IO) @Synchronized - override fun addServer(url: String): IServerConnection { + override fun addServer(url: String): Connection { return AppLevelModelSyncService.getInstance().addConnection(url).let { Connection(it) } } @@ -47,22 +48,13 @@ class ModelSyncService(val project: Project) : @Synchronized override fun dispose() { - bindings.forEach { it.deactivate() } + workers.values.forEach { it.deactivate() } coroutinesScope.cancel("disposed") } @Synchronized override fun getState(): Element? { - return State( - bindings = bindings.map { - BindingState( - url = it.serverConnection.getUrl(), - repository = it.branchRef.repositoryId.id, - branch = it.branchRef.branchName, - versionHash = it.getCurrentVersionHash(), - ) - }, - ).toXml() + return updateCurrentVersions().toXml() // Returning XML seems to be the most reliable way to get the state actually persisted. // Letting IntelliJ serialize the state sometimes fails silently. // Using kotlin.serialization is difficult because of version conflicts. @@ -70,85 +62,148 @@ class ModelSyncService(val project: Project) : @Synchronized override fun loadState(state: Element) { - val state = State.fromXml(state) - val statesById = state.bindings.associateBy { it.getId() } - val bindingsById = bindings.associateBy { it.getState().getId() } - val allBindingIds = statesById.keys + bindingsById.keys + loadState(SyncServiceState.fromXml(state)) + } + + override fun getBindings(): List { + return synchronized(this@ModelSyncService) { + loadedState.bindings.keys.map { Binding(it) } + } + } + + fun loadState(newState: SyncServiceState) { + val oldState: SyncServiceState = this.loadedState + val allBindingIds = newState.bindings.keys + workers.keys for (id in allBindingIds) { - val state = statesById[id] - val binding = bindingsById[id] - if (state == null) { + val newBindingState: BindingState? = newState.bindings[id] + val oldBindingState: BindingState? = oldState.bindings[id] + val binding: BindingWorker? = workers[id] + if (newBindingState == null) { if (binding == null) { // unreachable } else { binding.deactivate() - bindings.remove(binding) + workers.remove(id) } } else { if (binding == null) { - loadBinding(state) + loadBinding(id, newBindingState) } else { - if (binding.initialVersionHash != state.versionHash && binding.getCurrentVersionHash() != state.versionHash) { + if (newBindingState.versionHash != oldBindingState?.versionHash && + newBindingState.versionHash != binding.initialVersionHash && + newBindingState.versionHash != binding.getCurrentVersionHash() + ) { binding.deactivate() - bindings.remove(binding) - loadBinding(state) + workers.remove(id) + loadBinding(id, newBindingState) } } } } + + this.loadedState = newState } - private fun loadBinding(state: BindingState) { - addServer(state.url ?: return).bind( - branchRef = RepositoryId(state.repository ?: return).getBranchReference(state.branch), - lastSyncedVersionHash = state.versionHash, - ) + @Synchronized + private fun writeState(updater: (SyncServiceState) -> SyncServiceState): SyncServiceState { + val oldState: SyncServiceState = this.loadedState + val updatedState = updater(oldState) + if (updatedState != oldState) { + this.loadedState = updatedState + } + return this.loadedState } - private fun Binding.getState() = BindingState( - url = serverConnection.getUrl(), - repository = branchRef.repositoryId.id, - branch = branchRef.branchName, - versionHash = getCurrentVersionHash(), - ) + @Synchronized + private fun updateState(updater: (SyncServiceState) -> SyncServiceState): SyncServiceState { + return updater(this.loadedState).also { loadState(it) } + } - data class State( - @Property - val bindings: List = emptyList(), - ) { - fun toXml() = Element("model-sync").also { - it.children.addAll(bindings.map { it.toXml() }) + @Synchronized + private fun updateBindingState(id: BindingId, updater: (BindingState) -> BindingState) { + updateState { oldState -> + val oldBinding = oldState.bindings[id] + oldState.copy( + bindings = oldState.bindings + (id to updater(oldBinding ?: BindingState())), + ) } - companion object { - fun fromXml(element: Element) = State(element.getChildren("binding").map { BindingState.fromXml(it) }) + } + + private fun updateCurrentVersions(): SyncServiceState { + return writeState { oldState -> + oldState.copy( + oldState.bindings.mapValues { + it.value.copy( + versionHash = workers[it.key]?.getCurrentVersionHash() ?: it.value.versionHash, + ) + }, + ) } } - data class BindingState( - val url: String? = null, - val repository: String? = null, - val branch: String? = null, - val versionHash: String? = null, + private fun loadBinding(id: BindingId, state: BindingState) { + val binding = getOrCreateWorker(id, state) + if (state.enabled) { + binding.activate() + } else { + binding.deactivate() + } + } + + private fun getOrCreateWorker(id: BindingId, state: BindingState?): BindingWorker { + return workers.getOrPut(id) { + BindingWorker( + coroutinesScope, + mpsProject, + serverConnection = addServer(id.url), + branchRef = id.branchRef, + initialVersionHash = state?.versionHash, + ) + } + } + + data class SyncServiceState( + val bindings: Map = emptyMap(), ) { - fun getId(): Any = copy(versionHash = null) - fun toXml() = Element("binding").also { - it.children.add(Element("url").also { it.text = url }) - it.children.add(Element("repository").also { it.text = repository }) - it.children.add(Element("branch").also { it.text = branch }) - it.children.add(Element("versionHash").also { it.text = versionHash }) + fun toXml() = Element("model-sync").also { + it.children.addAll( + bindings.map { bindingEntry -> + Element("binding").also { + it.children.add(Element("enabled").also { it.text = bindingEntry.value.enabled.toString() }) + it.children.add(Element("url").also { it.text = bindingEntry.key.url }) + it.children.add(Element("repository").also { it.text = bindingEntry.key.branchRef.repositoryId.id }) + it.children.add(Element("branch").also { it.text = bindingEntry.key.branchRef.branchName }) + it.children.add(Element("versionHash").also { it.text = bindingEntry.value.versionHash }) + } + }, + ) } companion object { - fun fromXml(element: Element) = BindingState( - // separate elements instead of attributes so that each value has its own line in the .xml file - element.getChild("url")?.text, - element.getChild("repository")?.text, - element.getChild("branch")?.text, - element.getChild("versionHash")?.text, - ) + fun fromXml(element: Element): SyncServiceState { + return SyncServiceState( + element.getChildren("binding").mapNotNull> { element -> + BindingId( + url = element.getChild("url")?.text ?: return@mapNotNull null, + branchRef = BranchReference( + RepositoryId(element.getChild("repository")?.text ?: return@mapNotNull null), + element.getChild("branch")?.text ?: return@mapNotNull null, + ), + ) to BindingState( + versionHash = element.getChild("versionHash")?.text, + enabled = element.getChild("enabled")?.text.toBoolean(), + ) + }.toMap(), + ) + } } } + data class BindingState( + val versionHash: String? = null, + val enabled: Boolean = false, + ) + inner class Connection(val connection: AppLevelModelSyncService.ServerConnection) : IServerConnection { override fun getUrl(): String { return connection.url @@ -179,33 +234,85 @@ class ModelSyncService(val project: Project) : } override fun bind(branchRef: BranchReference, lastSyncedVersionHash: String?): IBinding { - val binding = Binding( - coroutinesScope = coroutinesScope, - mpsProject = mpsProject, - serverConnection = this, - branchRef = branchRef, - initialVersionHash = lastSyncedVersionHash, - ) - bindings.add(binding) - binding.activate() - return binding + val id = BindingId(connection.url, branchRef) + updateBindingState(id) { oldBinding -> + BindingState( + versionHash = lastSyncedVersionHash ?: oldBinding?.versionHash, + enabled = true, + ) + } + return Binding(id) } override fun getBindings(): List { - return bindings.filter { it.serverConnection == this } + return synchronized(this@ModelSyncService) { + loadedState.bindings.keys.map { Binding(it) }.filter { it.id.url == connection.url } + } } + private fun getService(): ModelSyncService = this@ModelSyncService + override fun equals(other: Any?): Boolean { if (this === other) return true if (javaClass != other?.javaClass) return false other as Connection - return connection == other.connection + return connection == other.connection && getService() == other.getService() + } + + override fun hashCode(): Int { + return connection.hashCode() + 31 * getService().hashCode() + } + } + + inner class Binding(val id: BindingId) : IBinding { + override fun toString(): String = id.toString() + + override fun getProject(): org.jetbrains.mps.openapi.project.Project { + return ProjectHelper.fromIdeaProject(project)!! + } + + override fun getBranchRef(): BranchReference { + return id.branchRef + } + + override fun isEnabled(): Boolean { + return synchronized(this@ModelSyncService) { loadedState.bindings[id]?.enabled == true } + } + + override fun enable() { + updateBindingState(id) { it.copy(enabled = true) } + } + + override fun disable() { + updateBindingState(id) { it.copy(enabled = false) } + } + + override fun delete() { + updateState { it.copy(bindings = it.bindings - id) } + } + + override suspend fun flush(): IVersion { + val worker = synchronized(this@ModelSyncService) { + getOrCreateWorker(id, loadedState.bindings[id]) + } + return worker.flush() + } + + private fun getService(): ModelSyncService = this@ModelSyncService + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Binding + + return id == other.id && getService() == other.getService() } override fun hashCode(): Int { - return connection.hashCode() + return id.hashCode() + 31 * getService().hashCode() } } } @@ -232,3 +339,9 @@ suspend fun jobLoop( } } } + +data class BindingId(val url: String, val branchRef: BranchReference) { + override fun toString(): String { + return "BindingId($url, ${branchRef.repositoryId}, ${branchRef.branchName})" + } +} diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt index 71fec5eca2..faf2a35415 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ValueWithMutex.kt @@ -14,7 +14,7 @@ class ValueWithMutex(private var value: E) { } lastUpdateResult = newValue newValue.onFailure { - Binding.Companion.LOG.error(it) { "Value update failed. Keeping $value" } + BindingWorker.Companion.LOG.error(it) { "Value update failed. Keeping $value" } } newValue.getOrThrow().also { value = it } } diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index 48ae0cbb6c..1fe35ca9cd 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicLong import kotlin.io.path.absolute import kotlin.io.path.readText import kotlin.io.path.writeText +import kotlin.test.assertFailsWith import kotlin.time.Duration.Companion.minutes import kotlin.time.toJavaDuration @@ -302,14 +303,19 @@ class ProjectSyncTest : MPSTestBase() { assertEquals(expectedSnapshot, project.captureSnapshot()) } - fun `test loading persisted binding`(): Unit = runWithModelServer { port -> + fun `test loading enabled persisted binding`(): Unit = runPersistedBindingTest(true) + fun `test loading disabled persisted binding`(): Unit = runPersistedBindingTest(false) + + fun runPersistedBindingTest(enabled: Boolean) = runWithModelServer { port -> // The client is in sync ... val branchRef = RepositoryId("sync-test").getBranchReference() val version1 = syncProjectToServer("initial", port, branchRef) + val snapshot1 = lastSnapshotBeforeSync // ... and then closes the project while some other client continues making changes. val version2 = syncProjectToServer("change1", port, branchRef, version1.getContentHash()) - val expectedSnapshot = lastSnapshotBeforeSync + val snapshot2 = lastSnapshotBeforeSync + val expectedSnapshot = if (enabled) snapshot2 else snapshot1 // Then the client opens the project again and reconnects using the persisted binding information. openTestProject("initial") { projectDir -> @@ -319,6 +325,7 @@ class ProjectSyncTest : MPSTestBase() { + $enabled http://localhost:$port ${branchRef.repositoryId.id} ${branchRef.branchName} @@ -330,14 +337,22 @@ class ProjectSyncTest : MPSTestBase() { ) } - val binding = IModelSyncService.getInstance(mpsProject).getServerConnections().flatMap { it.getBindings() }.single() - assertEquals(branchRef, binding.branchRef) - val version3 = binding.flush() + val binding = IModelSyncService.getInstance(mpsProject).getServerConnections() + .flatMap { it.getBindings() } + .single() + assertEquals(branchRef, binding.getBranchRef()) + if (enabled) { + val version3 = binding.flush() - assertEquals(version2.getContentHash(), version3.getContentHash()) + assertEquals(version2.getContentHash(), version3.getContentHash()) - // ... applies all the pending changes and is again in sync with the other client - assertEquals(expectedSnapshot, project.captureSnapshot()) + // ... applies all the pending changes and is again in sync with the other client + assertEquals(expectedSnapshot, project.captureSnapshot()) + } else { + assertFailsWith { + binding.flush() + } + } } fun `test storing persisted binding`(): Unit = runWithModelServer { port -> @@ -352,6 +367,7 @@ class ProjectSyncTest : MPSTestBase() { + true http://localhost:$port ${branchRef.repositoryId.id} ${branchRef.branchName} From 9e6610ea3330cbf986ec8c09e386b52040fb87f7 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Fri, 21 Feb 2025 20:59:21 +0100 Subject: [PATCH 10/37] feat(mps-sync-plugin): MPS 2020.3 support --- mps-sync-plugin3/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mps-sync-plugin3/build.gradle.kts b/mps-sync-plugin3/build.gradle.kts index f00835beb3..d3691c823d 100644 --- a/mps-sync-plugin3/build.gradle.kts +++ b/mps-sync-plugin3/build.gradle.kts @@ -37,7 +37,7 @@ dependencies { tasks { patchPluginXml { - sinceBuild.set("211") + sinceBuild.set("203") untilBuild.set("241.*") } From 16209d3ccd9a9ce1aeda907c6483a7955d543e18 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Fri, 21 Feb 2025 21:00:57 +0100 Subject: [PATCH 11/37] fix(mps-sync-plugin): binding couldn't be disabled --- .../modelix/mps/sync3/IModelSyncService.kt | 1 + .../org/modelix/mps/sync3/ModelSyncService.kt | 46 ++++++++++++------- .../org/modelix/mps/sync3/ProjectSyncTest.kt | 39 +++++++++++++++- 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt index c8a5245d68..9a03ddd3f2 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/IModelSyncService.kt @@ -66,4 +66,5 @@ interface IBinding : Closeable { */ suspend fun flush(): IVersion fun flushBlocking() = runBlocking { flush() } + suspend fun flushIfEnabled(): IVersion? } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt index 363e734cb1..9bc1efa944 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/ModelSyncService.kt @@ -65,40 +65,43 @@ class ModelSyncService(val project: Project) : loadState(SyncServiceState.fromXml(state)) } + @Synchronized override fun getBindings(): List { - return synchronized(this@ModelSyncService) { - loadedState.bindings.keys.map { Binding(it) } - } + return loadedState.bindings.keys.map { Binding(it) } } + @Synchronized fun loadState(newState: SyncServiceState) { val oldState: SyncServiceState = this.loadedState - val allBindingIds = newState.bindings.keys + workers.keys + val allBindingIds = newState.bindings.keys + oldState.bindings.keys + workers.keys for (id in allBindingIds) { val newBindingState: BindingState? = newState.bindings[id] val oldBindingState: BindingState? = oldState.bindings[id] - val binding: BindingWorker? = workers[id] + val worker: BindingWorker? = workers[id] if (newBindingState == null) { - if (binding == null) { - // unreachable + if (worker == null) { + // nothing to do } else { - binding.deactivate() + worker.deactivate() workers.remove(id) } } else { - if (binding == null) { - loadBinding(id, newBindingState) - } else { + if (worker != null) { if (newBindingState.versionHash != oldBindingState?.versionHash && - newBindingState.versionHash != binding.initialVersionHash && - newBindingState.versionHash != binding.getCurrentVersionHash() + newBindingState.versionHash != worker.initialVersionHash && + newBindingState.versionHash != worker.getCurrentVersionHash() ) { - binding.deactivate() + worker.deactivate() workers.remove(id) - loadBinding(id, newBindingState) } } + val newWorker = getOrCreateWorker(id, newBindingState) + if (newBindingState.enabled) { + newWorker.activate() + } else { + newWorker.deactivate() + } } } @@ -130,6 +133,7 @@ class ModelSyncService(val project: Project) : } } + @Synchronized private fun updateCurrentVersions(): SyncServiceState { return writeState { oldState -> oldState.copy( @@ -142,7 +146,8 @@ class ModelSyncService(val project: Project) : } } - private fun loadBinding(id: BindingId, state: BindingState) { + @Synchronized + private fun updateWorker(id: BindingId, state: BindingState) { val binding = getOrCreateWorker(id, state) if (state.enabled) { binding.activate() @@ -151,6 +156,7 @@ class ModelSyncService(val project: Project) : } } + @Synchronized private fun getOrCreateWorker(id: BindingId, state: BindingState?): BindingWorker { return workers.getOrPut(id) { BindingWorker( @@ -300,6 +306,14 @@ class ModelSyncService(val project: Project) : return worker.flush() } + override suspend fun flushIfEnabled(): IVersion? { + val worker = synchronized(this@ModelSyncService) { + if (!isEnabled()) return null + getOrCreateWorker(id, loadedState.bindings[id]) + } + return worker.flush() + } + private fun getService(): ModelSyncService = this@ModelSyncService override fun equals(other: Any?): Boolean { diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index 1fe35ca9cd..6a9c195847 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -6,6 +6,7 @@ import com.intellij.testFramework.TestApplicationManager import jetbrains.mps.smodel.SNodeUtil import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout +import org.jetbrains.mps.openapi.persistence.PersistenceFacade import org.junit.Assert import org.modelix.model.IVersion import org.modelix.model.api.TreePointer @@ -35,6 +36,7 @@ import kotlin.io.path.readText import kotlin.io.path.writeText import kotlin.test.assertFailsWith import kotlin.time.Duration.Companion.minutes +import kotlin.time.ExperimentalTime import kotlin.time.toJavaDuration private val modelServerDir = Path.of("../model-server").absolute().normalize() @@ -355,7 +357,7 @@ class ProjectSyncTest : MPSTestBase() { } } - fun `test storing persisted binding`(): Unit = runWithModelServer { port -> + fun `test saving binding state`(): Unit = runWithModelServer { port -> val branchRef = RepositoryId("sync-test").getBranchReference() openTestProject(null) val binding = IModelSyncService.getInstance(project).addServer("http://localhost:$port").bind(branchRef) @@ -379,7 +381,42 @@ class ProjectSyncTest : MPSTestBase() { assertEquals(expected, actual) } + fun `test binding can be disabled`(): Unit = runWithModelServer { port -> + // An MPS project is connected to a repository ... + val branchRef = RepositoryId("sync-test").getBranchReference() + openTestProject("initial") + val service = IModelSyncService.getInstance(mpsProject) + val connection = service.addServer("http://localhost:$port") + val binding = connection.bind(branchRef) + val version1 = binding.flush() + + // With the binding disabled ... + assertTrue(binding.isEnabled()) + binding.disable() + assertFalse(binding.isEnabled()) + + // ... the MPS user changes the name of a class ... + val nameProperty = SNodeUtil.property_INamedConcept_name + command { + val node = PersistenceFacade.getInstance() + .createNodeReference("r:cd78e6ac-0e34-490a-9b49-e5643f948d6d(NewSolution.a_model)/8281020627045237343") + .resolve(mpsProject.repository)!! + node.setProperty(nameProperty, "A") + } + + binding.flushIfEnabled() + + val version2 = connection.pullVersion(branchRef) + + // ... which should not be synchronized to the server + assertEquals(version1.getContentHash(), version2.getContentHash()) + + binding.enable() + val version3 = binding.flush() + } + private fun runWithModelServer(body: suspend (port: Int) -> Unit) = runBlocking { + @OptIn(ExperimentalTime::class) withTimeout(3.minutes) { val modelServer: GenericContainer<*> = GenericContainer(modelServerImage) .withExposedPorts(28101) From ee17b6b9c206f6866b4c37518410415bcd0a8db0 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 22 Feb 2025 06:23:47 +0100 Subject: [PATCH 12/37] feat(mps-sync-plugin): catch exceptions and continue synchronization An exception may heal itself if the user continues editing. Without ignoring exceptions, the synchronization will just stop. --- .../model/sync/bulk/ModelSynchronizer.kt | 100 +++++++++++------- 1 file changed, 63 insertions(+), 37 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 2d39211d41..869f024e70 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -1,5 +1,6 @@ package org.modelix.model.sync.bulk +import mu.KLogger import mu.KotlinLogging import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.INode @@ -21,6 +22,8 @@ import org.modelix.model.api.syncNewChildren import org.modelix.model.data.NodeData import org.modelix.model.sync.bulk.ModelSynchronizer.IIncrementalUpdateInformation +private val LOG: KLogger = mu.KotlinLogging.logger { } + /** * Similar to [ModelImporter], but the input is two [INode] instances instead of [INode] and [NodeData]. * @@ -39,11 +42,20 @@ class ModelSynchronizer( val nodeAssociation: INodeAssociation, val sourceMask: IModelMask = UnfilteredModelMask(), val targetMask: IModelMask = UnfilteredModelMask(), + val continueOnError: Boolean = true, ) { private val nodesToRemove: MutableSet = HashSet() private val pendingReferences: MutableList = ArrayList() private val logger = KotlinLogging.logger {} + private fun runSafe(body: () -> R): Result { + return if (continueOnError) { + runCatching(body).onFailure { LOG.warn(it) { "Ignoring exception during synchronization" } } + } else { + Result.success(body()) + } + } + fun synchronize() { synchronize(listOf(sourceRoot), listOf(targetRoot)) } @@ -55,8 +67,10 @@ class ModelSynchronizer( } logger.debug { "Synchronizing pending references..." } pendingReferences.forEach { - if (!it.trySyncReference()) { - it.copyTargetRef() + runSafe { + if (!it.trySyncReference()) { + it.copyTargetRef() + } } } logger.debug { "Removing extra nodes..." } @@ -68,24 +82,30 @@ class ModelSynchronizer( nodeAssociation.associate(sourceNode, targetNode) if (filter.needsSynchronization(sourceNode)) { logger.trace { "Synchronizing changed node. sourceNode = $sourceNode" } - synchronizeProperties(sourceNode, targetNode) - synchronizeReferences(sourceNode, targetNode) + runSafe { synchronizeProperties(sourceNode, targetNode) } + runSafe { synchronizeReferences(sourceNode, targetNode) } - val sourceConcept = sourceNode.getConceptReference() - val targetConcept = targetNode.getConceptReference() + val conceptCorrectedTargetNode = runSafe { + val sourceConcept = sourceNode.getConceptReference() + val targetConcept = targetNode.getConceptReference() - val conceptCorrectedTargetNode = if (sourceConcept != targetConcept) { - targetNode.changeConcept(sourceConcept) - } else { - targetNode - } + if (sourceConcept != targetConcept) { + targetNode.changeConcept(sourceConcept) + } else { + targetNode + } + }.getOrDefault(targetNode) - syncChildren(sourceNode, conceptCorrectedTargetNode) + runSafe { + syncChildren(sourceNode, conceptCorrectedTargetNode) + } } else if (filter.needsDescentIntoSubtree(sourceNode)) { for (sourceChild in sourceMask.filterChildren(sourceNode, sourceNode.getAllChildren())) { - val targetChild = nodeAssociation.resolveTarget(sourceChild) - ?: error("Expected target node was not found. sourceChild=${sourceChild.getNodeReference()}, originalId=${sourceChild.getOriginalReference()}") - synchronizeNode(sourceChild, targetChild) + runSafe { + val targetChild = nodeAssociation.resolveTarget(sourceChild) + ?: error("Expected target node was not found. sourceChild=${sourceChild.getNodeReference()}, originalId=${sourceChild.getOriginalReference()}") + synchronizeNode(sourceChild, targetChild) + } } } else { logger.trace { "Skipping subtree due to filter. root = $sourceNode" } @@ -97,12 +117,14 @@ class ModelSynchronizer( targetNode: IWritableNode, ) { iterateMergedRoles(sourceNode.getReferenceLinks(), targetNode.getReferenceLinks()) { role -> - val pendingReference = PendingReference(sourceNode, targetNode, role) + runSafe { + val pendingReference = PendingReference(sourceNode, targetNode, role) - // If the reference target already exist we can synchronize it immediately and save memory between the - // two synchronization phases. - if (!pendingReference.trySyncReference()) { - pendingReferences += pendingReference + // If the reference target already exist we can synchronize it immediately and save memory between the + // two synchronization phases. + if (!pendingReference.trySyncReference()) { + pendingReferences += pendingReference + } } } } @@ -112,10 +134,12 @@ class ModelSynchronizer( targetNode: IWritableNode, ) { iterateMergedRoles(sourceNode.getPropertyLinks(), targetNode.getPropertyLinks()) { role -> - val oldValue = targetNode.getPropertyValue(role) - val newValue = sourceNode.getPropertyValue(role) - if (oldValue != newValue) { - targetNode.setPropertyValue(role, newValue) + runSafe { + val oldValue = targetNode.getPropertyValue(role) + val newValue = sourceNode.getPropertyValue(role) + if (oldValue != newValue) { + targetNode.setPropertyValue(role, newValue) + } } } } @@ -255,22 +279,24 @@ class ModelSynchronizer( } fun trySyncReference(): Boolean { - val expectedRef = sourceNode.getReferenceTargetRef(role) - if (expectedRef == null) { - targetNode.setReferenceTargetRef(role, null) - return true - } - val actualRef = targetNode.getReferenceTargetRef(role) + return runSafe { + val expectedRef = sourceNode.getReferenceTargetRef(role) + if (expectedRef == null) { + targetNode.setReferenceTargetRef(role, null) + return@runSafe true + } + val actualRef = targetNode.getReferenceTargetRef(role) - val referenceTargetInSource = sourceNode.getReferenceTarget(role) ?: return false + val referenceTargetInSource = sourceNode.getReferenceTarget(role) ?: return@runSafe false - val referenceTargetInTarget = nodeAssociation.resolveTarget(referenceTargetInSource) - ?: return false // Target cannot be resolved right now but might become resolvable later. + val referenceTargetInTarget = nodeAssociation.resolveTarget(referenceTargetInSource) + ?: return@runSafe false // Target cannot be resolved right now but might become resolvable later. - if (referenceTargetInTarget.getNodeReference().serialize() != actualRef?.serialize()) { - targetNode.setReferenceTarget(role, referenceTargetInTarget) - } - return true + if (referenceTargetInTarget.getNodeReference().serialize() != actualRef?.serialize()) { + targetNode.setReferenceTarget(role, referenceTargetInTarget) + } + return@runSafe true + }.getOrDefault(false) } } From c1eead7cf735bb8a297c878916d7962645c7b3f6 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 22 Feb 2025 06:49:37 +0100 Subject: [PATCH 13/37] fix(mps-sync-plugin): descendants of new nodes where not synchronized --- .../model/sync/bulk/ModelSynchronizer.kt | 18 +++--- .../mps/sync3/MPSInvalidatingListener.kt | 28 +++++--- .../org/modelix/mps/sync3/ProjectSyncTest.kt | 64 +++++++++++++++++++ 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 869f024e70..9c6b1a35d2 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -63,7 +63,7 @@ class ModelSynchronizer( fun synchronize(sourceNodes: List, targetNodes: List) { logger.debug { "Synchronizing nodes..." } for ((sourceNode, targetNode) in sourceNodes.zip(targetNodes)) { - synchronizeNode(sourceNode, targetNode) + synchronizeNode(sourceNode, targetNode, false) } logger.debug { "Synchronizing pending references..." } pendingReferences.forEach { @@ -78,7 +78,7 @@ class ModelSynchronizer( logger.debug { "Synchronization finished." } } - private fun synchronizeNode(sourceNode: IReadableNode, targetNode: IWritableNode) { + private fun synchronizeNode(sourceNode: IReadableNode, targetNode: IWritableNode, forceSyncDescendants: Boolean) { nodeAssociation.associate(sourceNode, targetNode) if (filter.needsSynchronization(sourceNode)) { logger.trace { "Synchronizing changed node. sourceNode = $sourceNode" } @@ -97,14 +97,14 @@ class ModelSynchronizer( }.getOrDefault(targetNode) runSafe { - syncChildren(sourceNode, conceptCorrectedTargetNode) + syncChildren(sourceNode, conceptCorrectedTargetNode, forceSyncDescendants) } } else if (filter.needsDescentIntoSubtree(sourceNode)) { for (sourceChild in sourceMask.filterChildren(sourceNode, sourceNode.getAllChildren())) { runSafe { val targetChild = nodeAssociation.resolveTarget(sourceChild) ?: error("Expected target node was not found. sourceChild=${sourceChild.getNodeReference()}, originalId=${sourceChild.getOriginalReference()}") - synchronizeNode(sourceChild, targetChild) + synchronizeNode(sourceChild, targetChild, forceSyncDescendants) } } } else { @@ -152,7 +152,7 @@ class ModelSynchronizer( return parent.getChildren(role).let { targetMask.filterChildren(parent, role, it) } } - private fun syncChildren(sourceParent: IReadableNode, targetParent: IWritableNode) { + private fun syncChildren(sourceParent: IReadableNode, targetParent: IWritableNode, forceSyncDescendants: Boolean) { iterateMergedRoles( sourceParent.getAllChildren().map { it.getContainmentLink() }.distinct(), targetParent.getAllChildren().map { it.getContainmentLink() }.distinct(), @@ -173,7 +173,7 @@ class ModelSynchronizer( .zip(sourceNodes) .forEach { (newChild, sourceChild) -> nodeAssociation.associate(sourceChild, newChild) - synchronizeNode(sourceChild, newChild) + synchronizeNode(sourceChild, newChild, forceSyncDescendants) } return@iterateMergedRoles } @@ -181,7 +181,7 @@ class ModelSynchronizer( // optimization for when there is no change in the child list // size check first to avoid querying the original ID if (sourceNodes.size == targetNodes.size && sourceNodes.zip(targetNodes).all { nodeAssociation.matches(it.first, it.second) }) { - sourceNodes.zip(targetNodes).forEach { synchronizeNode(it.first, it.second) } + sourceNodes.zip(targetNodes).forEach { synchronizeNode(it.first, it.second, forceSyncDescendants) } return@iterateMergedRoles } @@ -206,11 +206,13 @@ class ModelSynchronizer( // existingChildren.getOrNull handles `-1` as needed by returning `null`. val nodeAtIndex = existingChildren.getOrNull(newIndex) val expectedConcept = expected.getConceptReference() + var isNewChild = false val childNode = if (nodeAtIndex?.getOriginalOrCurrentReference() != expectedId) { val existingNode = nodeAssociation.resolveTarget(expected) if (existingNode == null) { val newChild = targetParent.syncNewChild(role, newIndex, NewNodeSpec(expected)) nodeAssociation.associate(expected, newChild) + isNewChild = true newChild } else { // The existing child node is not only moved to a new index, @@ -234,7 +236,7 @@ class ModelSynchronizer( nodeAtIndex } - synchronizeNode(expected, childNode) + synchronizeNode(expected, childNode, forceSyncDescendants || isNewChild) } // Do not use existingNodes, but call node.getChildren(role) because diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt index 1de4addcae..e840700bbb 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt @@ -79,20 +79,20 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : onInvalidation() } - private fun invalidate(node: SNode) { - invalidate(node.asReadableNode()) + private fun invalidate(node: SNode, includingDescendants: Boolean = false) { + invalidate(node.asReadableNode(), includingDescendants) } - private fun invalidate(model: SModel) { - invalidate(MPSModelAsNode(model)) + private fun invalidate(model: SModel, includingDescendants: Boolean = false) { + invalidate(MPSModelAsNode(model), includingDescendants) } - private fun invalidate(module: SModule) { - invalidate(MPSModuleAsNode(module)) + private fun invalidate(module: SModule, includingDescendants: Boolean = false) { + invalidate(MPSModuleAsNode(module), includingDescendants) } - private fun invalidate(repository: SRepository) { - invalidate(MPSRepositoryAsNode(repository)) + private fun invalidate(repository: SRepository, includingDescendants: Boolean = false) { + invalidate(MPSRepositoryAsNode(repository), includingDescendants) } override fun addListener(model: SModel) { @@ -138,6 +138,7 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : } else { invalidate(e.model) } + invalidate(e.child, true) } override fun nodeRemoved(e: SNodeRemoveEvent) { @@ -153,7 +154,10 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : override fun beforeModelDisposed(model: SModel) {} override fun beforeModelRenamed(event: SModelRenamedEvent) {} override fun beforeRootRemoved(event: SModelRootEvent) {} - override fun childAdded(event: SModelChildEvent) { invalidate(event.parent) } + override fun childAdded(event: SModelChildEvent) { + invalidate(event.parent) + invalidate(event.child, true) + } override fun childRemoved(event: SModelChildEvent) { invalidate(event.parent) } override fun devkitAdded(event: SModelDevKitEvent) { invalidate(event.model) } override fun devkitRemoved(event: SModelDevKitEvent) { invalidate(event.model) } @@ -192,6 +196,7 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : override fun problemsDetected(model: SModel, problems: Iterable) {} override fun modelAdded(module: SModule, model: SModel) { invalidate(module) + invalidate(model, true) } override fun beforeModelRemoved(module: SModule, model: SModel) {} @@ -215,7 +220,10 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : * methods updateStarted and updateFinished in that version. */ private val srepositoryListener = object : SRepositoryListenerBase() { - override fun moduleAdded(module: SModule) { invalidate(repository) } + override fun moduleAdded(module: SModule) { + invalidate(repository) + invalidate(module, true) + } override fun beforeModuleRemoved(module: SModule) {} override fun moduleRemoved(reference: SModuleReference) { invalidate(repository) } override fun commandStarted(repository: SRepository) {} diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index 6a9c195847..a32e4dec6d 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -4,8 +4,13 @@ import com.badoo.reaktive.observable.toList import com.intellij.configurationStore.saveSettings import com.intellij.testFramework.TestApplicationManager import jetbrains.mps.smodel.SNodeUtil +import jetbrains.mps.smodel.adapter.ids.SConceptId +import jetbrains.mps.smodel.adapter.ids.SContainmentLinkId +import jetbrains.mps.smodel.adapter.structure.concept.SConceptAdapterById +import jetbrains.mps.smodel.adapter.structure.link.SContainmentLinkAdapterById import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout +import org.jetbrains.mps.openapi.model.SNode import org.jetbrains.mps.openapi.persistence.PersistenceFacade import org.junit.Assert import org.modelix.model.IVersion @@ -173,6 +178,65 @@ class ProjectSyncTest : MPSTestBase() { assertEquals("Changed", version2.getTree().getProperty(change.nodeId, change.role.key(version1.getTree()))) } + fun `test descendants of new node are synchronized`() = runChangeInMpsTest { classNode -> + val memberRole = SContainmentLinkAdapterById(SContainmentLinkId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1107461130800/5375687026011219971"), "member") + val visibilityRole = SContainmentLinkAdapterById(SContainmentLinkId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1178549954367/1178549979242"), "member") + val bodyRole = SContainmentLinkAdapterById(SContainmentLinkId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1178549954367/1068580123135"), "body") + val statementRole = SContainmentLinkAdapterById(SContainmentLinkId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1178549954367/1068581517665"), "statement") + val instanceMethodDeclarationConcept = SConceptAdapterById(SConceptId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1068580123165"), "InstanceMethodDeclaration") + val publicVisibilityConcept = SConceptAdapterById(SConceptId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1146644602865"), "PublicVisibility") + val statementListConcept = SConceptAdapterById(SConceptId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1068580123136"), "StatementList") + val returnStatementConcept = SConceptAdapterById(SConceptId.deserialize("f3061a53-9226-4cc5-a443-f952ceaf5816/1068581242878"), "ReturnStatement") + + val methodNode = jetbrains.mps.smodel.SNode(instanceMethodDeclarationConcept) + val visibilityNode = jetbrains.mps.smodel.SNode(publicVisibilityConcept).also { methodNode.addChild(visibilityRole, it) } + val statementListNode = jetbrains.mps.smodel.SNode(statementListConcept).also { methodNode.addChild(bodyRole, it) } + val returnStatementNode = jetbrains.mps.smodel.SNode(returnStatementConcept).also { statementListNode.addChild(statementRole, it) } + + // adding the new method when it already contains all the descendants will result in a single change event. + // There is no event for the other `addChild` calls, which is what this test is about. + classNode.addChild(memberRole, methodNode) + } + + private fun runChangeInMpsTest(mutator: (SNode) -> Unit): Unit = runWithModelServer { port -> + // An MPS project is connected to a repository ... + val branchRef = RepositoryId("sync-test").getBranchReference() + openTestProject("initial") + val binding1 = IModelSyncService.getInstance(mpsProject).addServer("http://localhost:$port").bind(branchRef) + val version1 = binding1.flush() + val snapshot1 = project.captureSnapshot() + + // ... and then an MPS user changes something in MPS ... + command { + val nameProperty = SNodeUtil.property_INamedConcept_name + val node = mpsProject.projectModules + .first { it.moduleName == "NewSolution" } + .models + .flatMap { it.rootNodes } + .first { it.getProperty(nameProperty) == "MyClass" } + mutator(node) + } + + // ... which is synchronized to the server. + val version2 = binding1.flush() + val snapshot2 = project.captureSnapshot() + project.close() + + // A second MPS client should end up in the same state. + + val branchRef2 = branchRef.repositoryId.getBranchReference("branchB") + val client = ModelClientV2.builder().url("http://localhost:$port").build() + client.push(branchRef2, version1, null) + + openTestProject("initial") + val binding2 = IModelSyncService.getInstance(mpsProject).addServer("http://localhost:$port").bind(branchRef2) + binding2.flush() + assertEquals(snapshot1, project.captureSnapshot()) + client.push(branchRef2, version2, version1) + binding2.flush() + assertEquals(snapshot2, project.captureSnapshot()) + } + fun `test sync after model-server change`(): Unit = runWithModelServer { port -> // An MPS project is connected to a repository ... val branchRef = RepositoryId("sync-test").getBranchReference() From 1eb2ad1f70167b4833c3a0d3b87ae663a1696999 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sun, 23 Feb 2025 10:57:54 +0100 Subject: [PATCH 14/37] fix(mps-sync-plugin): model-synchronizer didn't call ISyncTargetNode.isOrdered(IChildLinkReference) --- .../kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt | 3 +-- .../src/commonMain/kotlin/org/modelix/model/api/INode.kt | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 9c6b1a35d2..96ce1f7001 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -12,7 +12,6 @@ import org.modelix.model.api.NewNodeSpec import org.modelix.model.api.PNodeAdapter import org.modelix.model.api.getOriginalOrCurrentReference import org.modelix.model.api.getOriginalReference -import org.modelix.model.api.isChildRoleOrdered import org.modelix.model.api.isOrdered import org.modelix.model.api.matches import org.modelix.model.api.mergeWith @@ -185,7 +184,7 @@ class ModelSynchronizer( return@iterateMergedRoles } - val isOrdered = targetParent.isChildRoleOrdered(role) + val isOrdered = targetParent.isOrdered(role) sourceNodes.forEachIndexed { indexInImport, expected -> val existingChildren = getFilteredTargetChildren(targetParent, role) diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/INode.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/INode.kt index 4aa3fae574..7736f2c0ce 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/INode.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/INode.kt @@ -413,7 +413,7 @@ fun INode.tryResolveProperty(role: String): IProperty? { */ @Deprecated("provide a IChildLinkReference") fun INode.isChildRoleOrdered(role: String?): Boolean { - return asReadableNode().isChildRoleOrdered(IChildLinkReference.fromUnclassifiedString(role)) + return asReadableNode().isOrdered(IChildLinkReference.fromUnclassifiedString(role)) return if (role == null) { true } else { @@ -421,8 +421,9 @@ fun INode.isChildRoleOrdered(role: String?): Boolean { } } +@Deprecated("Use isOrdered", ReplaceWith("isOrdered(role)")) fun IReadableNode.isChildRoleOrdered(role: IChildLinkReference): Boolean { - return this.tryResolveChildLink(role)?.isOrdered ?: true + return isOrdered(role) } fun INode.resolvePropertyOrFallback(role: String): IProperty { From 63e6a29b06caab7a8716457f771898e89d3fb021 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sun, 23 Feb 2025 11:01:30 +0100 Subject: [PATCH 15/37] feat(mps-sync-plugin): ignore exception during synchronization --- .../modelix/model/sync/bulk/ModelSynchronizer.kt | 8 ++++---- .../kotlin/org/modelix/mps/sync3/BindingWorker.kt | 1 + .../modelix/mps/sync3/MPSInvalidatingListener.kt | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 96ce1f7001..ec59205a83 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -41,17 +41,17 @@ class ModelSynchronizer( val nodeAssociation: INodeAssociation, val sourceMask: IModelMask = UnfilteredModelMask(), val targetMask: IModelMask = UnfilteredModelMask(), - val continueOnError: Boolean = true, + private val exceptionHandler: ((Throwable) -> Unit)? = { LOG.warn(it) { "Ignoring exception during synchronization" } }, ) { private val nodesToRemove: MutableSet = HashSet() private val pendingReferences: MutableList = ArrayList() private val logger = KotlinLogging.logger {} private fun runSafe(body: () -> R): Result { - return if (continueOnError) { - runCatching(body).onFailure { LOG.warn(it) { "Ignoring exception during synchronization" } } - } else { + return if (exceptionHandler == null) { Result.success(body()) + } else { + runCatching(body).onFailure { exceptionHandler(it) } } } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt index 4cbbeaec5c..9125f9d6cc 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt @@ -213,6 +213,7 @@ class BindingWorker( nodeAssociation = nodeAssociation, sourceMask = MPSProjectSyncMask(mpsProjects, false), targetMask = MPSProjectSyncMask(mpsProjects, true), + exceptionHandler = { getMPSListener().synchronizationErrorHappened() }, ).synchronize() } } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt index e840700bbb..3d7cc0cd54 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/MPSInvalidatingListener.kt @@ -48,16 +48,26 @@ abstract class MPSInvalidatingListener(val repository: SRepository) : private val syncActive = AtomicBoolean(false) private val invalidationTree: DefaultInvalidationTree = DefaultInvalidationTree(MPSRepositoryAsNode(repository).getNodeReference().toSerialized()) + private var synchronizationErrorHappened: Boolean = false fun hasAnyInvalidations() = synchronized(invalidationTree) { invalidationTree.hasAnyInvalidations() } + fun synchronizationErrorHappened() { + synchronizationErrorHappened = true + } + fun runSync(body: (DefaultInvalidationTree) -> R): R { check(!syncActive.getAndSet(true)) { "Synchronization is already running" } try { synchronized(invalidationTree) { + synchronizationErrorHappened = false return body(invalidationTree).also { - LOG.trace { "Resetting invalidations" } - invalidationTree.reset() + if (synchronizationErrorHappened) { + LOG.trace { "Synchronization wasn't successful. Preserving invalidations." } + } else { + LOG.trace { "Resetting invalidations" } + invalidationTree.reset() + } } } } catch (ex: Throwable) { From 6509a81a3e66f3566ddfa370e87550587d4e57a9 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sun, 23 Feb 2025 11:02:19 +0100 Subject: [PATCH 16/37] fix(model-datastructure): AddNewChildrenOp.toString() --- .../kotlin/org/modelix/model/operations/AddNewChildOp.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt index 188b8dc228..5955f7397d 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt @@ -35,7 +35,7 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIds: LongArra } override fun toString(): String { - return "AddNewChildrenOp ${childIds.map { SerializationUtil.longToHex(it) }}, $position, $concepts" + return "AddNewChildrenOp ${childIds.map { SerializationUtil.longToHex(it) }}, $position, ${concepts.map { it?.getUID() }}" } inner class Applied : AbstractOperation.Applied(), IAppliedOperation { From 8f4d1485c7cfe3a5b38609da2f1252288dc66cb7 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 10:46:15 +0100 Subject: [PATCH 17/37] feat(mps-sync-plugin): ignore exception during synchronization --- .../model/sync/bulk/ModelSynchronizer.kt | 169 ++++++++++-------- 1 file changed, 92 insertions(+), 77 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index ec59205a83..09288c6a2c 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -41,7 +41,7 @@ class ModelSynchronizer( val nodeAssociation: INodeAssociation, val sourceMask: IModelMask = UnfilteredModelMask(), val targetMask: IModelMask = UnfilteredModelMask(), - private val exceptionHandler: ((Throwable) -> Unit)? = { LOG.warn(it) { "Ignoring exception during synchronization" } }, + private val exceptionHandler: ((Throwable) -> Unit)? = {}, ) { private val nodesToRemove: MutableSet = HashSet() private val pendingReferences: MutableList = ArrayList() @@ -51,7 +51,9 @@ class ModelSynchronizer( return if (exceptionHandler == null) { Result.success(body()) } else { - runCatching(body).onFailure { exceptionHandler(it) } + runCatching(body) + .onFailure { LOG.error(it) { "Ignoring exception during synchronization" } } + .onFailure { exceptionHandler(it) } } } @@ -156,93 +158,106 @@ class ModelSynchronizer( sourceParent.getAllChildren().map { it.getContainmentLink() }.distinct(), targetParent.getAllChildren().map { it.getContainmentLink() }.distinct(), ) { role -> - val sourceNodes = getFilteredSourceChildren(sourceParent, role) - val targetNodes = getFilteredTargetChildren(targetParent, role) - val unusedTargetChildren = targetNodes.toMutableSet() - - val allExpectedNodesDoNotExist by lazy { - sourceNodes.all { sourceNode -> - nodeAssociation.resolveTarget(sourceNode) == null - } + runSafe { + syncChildrenInRole(sourceParent, role, targetParent, forceSyncDescendants) } + } + // tryFixCrossRoleOrder(sourceParent, targetParent) + } - // optimization that uses the bulk operation .syncNewChildren - if (targetNodes.isEmpty() && allExpectedNodesDoNotExist) { - targetParent.syncNewChildren(role, -1, sourceNodes.map { NewNodeSpec(it) }) - .zip(sourceNodes) - .forEach { (newChild, sourceChild) -> - nodeAssociation.associate(sourceChild, newChild) - synchronizeNode(sourceChild, newChild, forceSyncDescendants) - } - return@iterateMergedRoles - } + private fun syncChildrenInRole( + sourceParent: IReadableNode, + role: IChildLinkReference, + targetParent: IWritableNode, + forceSyncDescendants: Boolean, + ) { + val sourceNodes = getFilteredSourceChildren(sourceParent, role) + val targetNodes = getFilteredTargetChildren(targetParent, role) + val unusedTargetChildren = targetNodes.toMutableSet() - // optimization for when there is no change in the child list - // size check first to avoid querying the original ID - if (sourceNodes.size == targetNodes.size && sourceNodes.zip(targetNodes).all { nodeAssociation.matches(it.first, it.second) }) { - sourceNodes.zip(targetNodes).forEach { synchronizeNode(it.first, it.second, forceSyncDescendants) } - return@iterateMergedRoles + val allExpectedNodesDoNotExist by lazy { + sourceNodes.all { sourceNode -> + nodeAssociation.resolveTarget(sourceNode) == null } + } - val isOrdered = targetParent.isOrdered(role) + // optimization that uses the bulk operation .syncNewChildren + if (targetNodes.isEmpty() && allExpectedNodesDoNotExist) { + targetParent.syncNewChildren(role, -1, sourceNodes.map { NewNodeSpec(it) }) + .zip(sourceNodes) + .forEach { (newChild, sourceChild) -> + nodeAssociation.associate(sourceChild, newChild) + synchronizeNode(sourceChild, newChild, forceSyncDescendants) + } + return + } - sourceNodes.forEachIndexed { indexInImport, expected -> - val existingChildren = getFilteredTargetChildren(targetParent, role) - val expectedId = checkNotNull(expected.originalIdOrFallback()) { "Specified node '$expected' has no id" } - // newIndex is the index on which to import the expected child. - // It might be -1 if the child does not exist and should be added at the end. - val newIndex = if (isOrdered) { - indexInImport + // optimization for when there is no change in the child list + // size check first to avoid querying the original ID + if (sourceNodes.size == targetNodes.size && sourceNodes.zip(targetNodes) + .all { nodeAssociation.matches(it.first, it.second) } + ) { + sourceNodes.zip(targetNodes).forEach { synchronizeNode(it.first, it.second, forceSyncDescendants) } + return + } + + val isOrdered = targetParent.isOrdered(role) + + sourceNodes.forEachIndexed { indexInImport, expected -> + val existingChildren = getFilteredTargetChildren(targetParent, role) + val expectedId = checkNotNull(expected.originalIdOrFallback()) { "Specified node '$expected' has no id" } + // newIndex is the index on which to import the expected child. + // It might be -1 if the child does not exist and should be added at the end. + val newIndex = if (isOrdered) { + indexInImport + } else { + // The `existingChildren` are only searched once for the expected element before changing. + // Therefore, indexing existing children will not be more efficient than iterating once. + // (For the moment, this is fine because as we expect unordered children to be the exception, + // Reusable indexing would be possible if we switch from + // a depth-first import to a breadth-first import.) + existingChildren + .indexOfFirst { existingChild -> existingChild.getOriginalOrCurrentReference() == expectedId } + } + // existingChildren.getOrNull handles `-1` as needed by returning `null`. + val nodeAtIndex = existingChildren.getOrNull(newIndex) + val expectedConcept = expected.getConceptReference() + var isNewChild = false + val childNode = if (nodeAtIndex?.getOriginalOrCurrentReference() != expectedId) { + val existingNode = nodeAssociation.resolveTarget(expected) + if (existingNode == null) { + val newChild = targetParent.syncNewChild(role, newIndex, NewNodeSpec(expected)) + nodeAssociation.associate(expected, newChild) + isNewChild = true + newChild } else { - // The `existingChildren` are only searched once for the expected element before changing. - // Therefore, indexing existing children will not be more efficient than iterating once. - // (For the moment, this is fine because as we expect unordered children to be the exception, - // Reusable indexing would be possible if we switch from - // a depth-first import to a breadth-first import.) - existingChildren - .indexOfFirst { existingChild -> existingChild.getOriginalOrCurrentReference() == expectedId } - } - // existingChildren.getOrNull handles `-1` as needed by returning `null`. - val nodeAtIndex = existingChildren.getOrNull(newIndex) - val expectedConcept = expected.getConceptReference() - var isNewChild = false - val childNode = if (nodeAtIndex?.getOriginalOrCurrentReference() != expectedId) { - val existingNode = nodeAssociation.resolveTarget(expected) - if (existingNode == null) { - val newChild = targetParent.syncNewChild(role, newIndex, NewNodeSpec(expected)) - nodeAssociation.associate(expected, newChild) - isNewChild = true - newChild - } else { - // The existing child node is not only moved to a new index, - // it is potentially moved to a new parent and role. - if (existingNode.getParent() != targetParent || - !existingNode.getContainmentLink().matches(role) || - targetParent.isOrdered(role) - ) { - targetParent.moveChild(role, newIndex, existingNode) - } - // If the old parent and old role synchronized before the move operation, - // the existing child node would have been marked as to be deleted. - // Now that it is used, it should not be deleted. - unusedTargetChildren.remove(existingNode) - nodesToRemove.remove(existingNode) - existingNode + // The existing child node is not only moved to a new index, + // it is potentially moved to a new parent and role. + if (existingNode.getParent() != targetParent || + !existingNode.getContainmentLink().matches(role) || + targetParent.isOrdered(role) + ) { + targetParent.moveChild(role, newIndex, existingNode) } - } else { - unusedTargetChildren.remove(nodeAtIndex) - nodesToRemove.remove(nodeAtIndex) - nodeAtIndex + // If the old parent and old role synchronized before the move operation, + // the existing child node would have been marked as to be deleted. + // Now that it is used, it should not be deleted. + unusedTargetChildren.remove(existingNode) + nodesToRemove.remove(existingNode) + existingNode } - - synchronizeNode(expected, childNode, forceSyncDescendants || isNewChild) + } else { + unusedTargetChildren.remove(nodeAtIndex) + nodesToRemove.remove(nodeAtIndex) + nodeAtIndex } - // Do not use existingNodes, but call node.getChildren(role) because - // the recursive synchronization in the meantime already removed some nodes from node.getChildren(role). - nodesToRemove += getFilteredTargetChildren(targetParent, role).intersect(unusedTargetChildren) + synchronizeNode(expected, childNode, forceSyncDescendants || isNewChild) } - // tryFixCrossRoleOrder(sourceParent, targetParent) + + // Do not use existingNodes, but call node.getChildren(role) because + // the recursive synchronization in the meantime already removed some nodes from node.getChildren(role). + nodesToRemove += getFilteredTargetChildren(targetParent, role).intersect(unusedTargetChildren) } /** From 64879041f5f13f392ee8f3d222bcac25db4dedc8 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:04:30 +0100 Subject: [PATCH 18/37] fix(mps-sync-plugin): descendants of new nodes where not synchronized --- .../org/modelix/model/sync/bulk/ModelSynchronizer.kt | 10 +++++----- .../main/kotlin/org/modelix/mps/sync3/BindingWorker.kt | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 09288c6a2c..5a1d9e31b7 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -41,19 +41,19 @@ class ModelSynchronizer( val nodeAssociation: INodeAssociation, val sourceMask: IModelMask = UnfilteredModelMask(), val targetMask: IModelMask = UnfilteredModelMask(), - private val exceptionHandler: ((Throwable) -> Unit)? = {}, + private val onException: ((Throwable) -> Unit)? = {}, ) { private val nodesToRemove: MutableSet = HashSet() private val pendingReferences: MutableList = ArrayList() private val logger = KotlinLogging.logger {} private fun runSafe(body: () -> R): Result { - return if (exceptionHandler == null) { + return if (onException == null) { Result.success(body()) } else { runCatching(body) .onFailure { LOG.error(it) { "Ignoring exception during synchronization" } } - .onFailure { exceptionHandler(it) } + .onFailure { onException(it) } } } @@ -81,7 +81,7 @@ class ModelSynchronizer( private fun synchronizeNode(sourceNode: IReadableNode, targetNode: IWritableNode, forceSyncDescendants: Boolean) { nodeAssociation.associate(sourceNode, targetNode) - if (filter.needsSynchronization(sourceNode)) { + if (forceSyncDescendants || filter.needsSynchronization(sourceNode)) { logger.trace { "Synchronizing changed node. sourceNode = $sourceNode" } runSafe { synchronizeProperties(sourceNode, targetNode) } runSafe { synchronizeReferences(sourceNode, targetNode) } @@ -187,7 +187,7 @@ class ModelSynchronizer( .zip(sourceNodes) .forEach { (newChild, sourceChild) -> nodeAssociation.associate(sourceChild, newChild) - synchronizeNode(sourceChild, newChild, forceSyncDescendants) + synchronizeNode(sourceChild, newChild, forceSyncDescendants = true) } return } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt index 9125f9d6cc..a514edc970 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt @@ -213,7 +213,7 @@ class BindingWorker( nodeAssociation = nodeAssociation, sourceMask = MPSProjectSyncMask(mpsProjects, false), targetMask = MPSProjectSyncMask(mpsProjects, true), - exceptionHandler = { getMPSListener().synchronizationErrorHappened() }, + onException = { getMPSListener().synchronizationErrorHappened() }, ).synchronize() } } From 298a71ff0eb30509408e317b92b5acd3aa05daf5 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:23:43 +0100 Subject: [PATCH 19/37] fix(mps-sync-plugin): synchronization of used devkits failed --- .../org/modelix/model/mpsadapters/MPSModelAsNode.kt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt index c62143f901..976d41a463 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSModelAsNode.kt @@ -161,9 +161,16 @@ data class MPSModelAsNode(val model: SModel) : MPSGenericNodeAdapter() { override fun remove(element: SModel, child: IWritableNode) { check(element is SModelDescriptorStub) { "Model '$element' is not a SModelDescriptor." } - require(child is MPSSingleLanguageDependencyAsNode) { "Node $child to be removed is not a single language dependency." } - val languageToRemove = MetaAdapterFactory.getLanguage(child.moduleReference.sourceModuleReference) - element.deleteLanguageId(languageToRemove) + when (child) { + is MPSSingleLanguageDependencyAsNode -> { + val languageToRemove = MetaAdapterFactory.getLanguage(child.moduleReference.sourceModuleReference) + element.deleteLanguageId(languageToRemove) + } + is MPSDevKitDependencyAsNode -> { + element.deleteDevKit(child.moduleReference) + } + else -> throw UnsupportedOperationException("Unsupported type: ${child.getConceptReference()}") + } } }, ) From a0039a2a233bffbded40cf4b4aeee613602518cb Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:35:45 +0100 Subject: [PATCH 20/37] chore(mps-sync-plugin): duplicate logger --- .../modelix/model/sync/bulk/ModelSynchronizer.kt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 5a1d9e31b7..7c67eb6af6 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -1,7 +1,6 @@ package org.modelix.model.sync.bulk import mu.KLogger -import mu.KotlinLogging import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.INode import org.modelix.model.api.IReadableNode @@ -45,7 +44,6 @@ class ModelSynchronizer( ) { private val nodesToRemove: MutableSet = HashSet() private val pendingReferences: MutableList = ArrayList() - private val logger = KotlinLogging.logger {} private fun runSafe(body: () -> R): Result { return if (onException == null) { @@ -62,11 +60,11 @@ class ModelSynchronizer( } fun synchronize(sourceNodes: List, targetNodes: List) { - logger.debug { "Synchronizing nodes..." } + LOG.debug { "Synchronizing nodes..." } for ((sourceNode, targetNode) in sourceNodes.zip(targetNodes)) { synchronizeNode(sourceNode, targetNode, false) } - logger.debug { "Synchronizing pending references..." } + LOG.debug { "Synchronizing pending references..." } pendingReferences.forEach { runSafe { if (!it.trySyncReference()) { @@ -74,15 +72,15 @@ class ModelSynchronizer( } } } - logger.debug { "Removing extra nodes..." } + LOG.debug { "Removing extra nodes..." } nodesToRemove.filter { it.isValid() }.forEach { it.remove() } - logger.debug { "Synchronization finished." } + LOG.debug { "Synchronization finished." } } private fun synchronizeNode(sourceNode: IReadableNode, targetNode: IWritableNode, forceSyncDescendants: Boolean) { nodeAssociation.associate(sourceNode, targetNode) if (forceSyncDescendants || filter.needsSynchronization(sourceNode)) { - logger.trace { "Synchronizing changed node. sourceNode = $sourceNode" } + LOG.trace { "Synchronizing changed node. sourceNode = $sourceNode" } runSafe { synchronizeProperties(sourceNode, targetNode) } runSafe { synchronizeReferences(sourceNode, targetNode) } @@ -109,7 +107,7 @@ class ModelSynchronizer( } } } else { - logger.trace { "Skipping subtree due to filter. root = $sourceNode" } + LOG.trace { "Skipping subtree due to filter. root = $sourceNode" } } } From e3a0303fab18b4cde5595f0a4f0904144392231e Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:41:13 +0100 Subject: [PATCH 21/37] fix(mps-sync-plugin): exceptions weren't logged (missing appender) --- mps-sync-plugin3/build.gradle.kts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mps-sync-plugin3/build.gradle.kts b/mps-sync-plugin3/build.gradle.kts index d3691c823d..8ed5e86452 100644 --- a/mps-sync-plugin3/build.gradle.kts +++ b/mps-sync-plugin3/build.gradle.kts @@ -15,12 +15,16 @@ intellij { } dependencies { - implementation(project(":bulk-model-sync-lib")) - implementation(project(":bulk-model-sync-mps")) - implementation(project(":mps-model-adapters")) - implementation(project(":model-client")) - implementation(libs.modelix.mpsApi) - implementation(libs.kotlin.logging) + val excludeMPSLibraries: (ModuleDependency).() -> Unit = { + exclude("org.slf4j", "slf4j-api") + } + + implementation(project(":bulk-model-sync-lib"), excludeMPSLibraries) + implementation(project(":bulk-model-sync-mps"), excludeMPSLibraries) + implementation(project(":mps-model-adapters"), excludeMPSLibraries) + implementation(project(":model-client"), excludeMPSLibraries) + implementation(libs.modelix.mpsApi, excludeMPSLibraries) + implementation(libs.kotlin.logging, excludeMPSLibraries) compileOnly( fileTree(mpsHomeDir).matching { From e91031517403fb3f4f6ca7aa254094f4a97f92f6 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:41:53 +0100 Subject: [PATCH 22/37] build(mps-sync-plugin): fixed installMpsPlugin task --- mps-sync-plugin3/build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mps-sync-plugin3/build.gradle.kts b/mps-sync-plugin3/build.gradle.kts index 8ed5e86452..51069db73f 100644 --- a/mps-sync-plugin3/build.gradle.kts +++ b/mps-sync-plugin3/build.gradle.kts @@ -72,12 +72,12 @@ tasks { } } - val mpsPluginDir = project.findProperty("mps.plugins.dir")?.toString()?.let { file(it) } + val mpsPluginDir = project.findProperty("mps$mpsPlatformVersion.plugins.dir")?.toString()?.let { file(it) } if (mpsPluginDir != null && mpsPluginDir.isDirectory) { create("installMpsPlugin") { dependsOn(prepareSandbox) - from(project.layout.buildDirectory.dir("idea-sandbox/plugins/mps-model-adapters-plugin")) - into(mpsPluginDir.resolve("mps-model-adapters-plugin")) + from(project.layout.buildDirectory.dir("idea-sandbox/plugins/mps-sync-plugin3")) + into(mpsPluginDir.resolve("mps-sync-plugin3")) } } } From 327bfe8d0e984b7cbdb97d42dcd2b9dae7df4f50 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:42:37 +0100 Subject: [PATCH 23/37] test(mps-sync-plugin): some cleanup --- .../src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt | 5 +++++ .../src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt | 2 +- .../src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt | 2 -- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt index 72a499a13d..00588bfdac 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/MPSTestBase.kt @@ -8,6 +8,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.project.ProjectManager import com.intellij.openapi.project.ex.ProjectManagerEx import com.intellij.openapi.util.Disposer +import com.intellij.testFramework.TestApplicationManager import com.intellij.testFramework.UsefulTestCase import com.intellij.util.io.delete import jetbrains.mps.ide.ThreadUtils @@ -27,6 +28,10 @@ abstract class MPSTestBase : UsefulTestCase() { protected lateinit var project: Project override fun runInDispatchThread() = false + override fun setUp() { + super.setUp() + TestApplicationManager.getInstance() + } @OptIn(ExperimentalPathApi::class) fun openTestProject(testDataName: String?, beforeOpen: (projectDir: Path) -> Unit = {}): Project { diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt index 8e87dd3bc4..92a0350346 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt @@ -29,7 +29,7 @@ private fun filterFiles(files: Map) = files.filter { val name = it.key if (name.startsWith(".mps/")) { when (name.substringAfter("/")) { - ".gitignore", "migration.xml", "workspace.xml", "modelix.xml" -> false + ".gitignore", "migration.xml", "workspace.xml", "modelix.xml", "vcs.xml" -> false else -> true } } else if (name.contains("/source_gen") || name.contains("/classes_gen")) { diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index a32e4dec6d..f14ff9b38f 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -2,7 +2,6 @@ package org.modelix.mps.sync3 import com.badoo.reaktive.observable.toList import com.intellij.configurationStore.saveSettings -import com.intellij.testFramework.TestApplicationManager import jetbrains.mps.smodel.SNodeUtil import jetbrains.mps.smodel.adapter.ids.SConceptId import jetbrains.mps.smodel.adapter.ids.SContainmentLinkId @@ -55,7 +54,6 @@ class ProjectSyncTest : MPSTestBase() { override fun setUp() { super.setUp() - TestApplicationManager.getInstance() } override fun tearDown() { From 48899e25a8d15cb086fe2923f74e327db96f511f Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Mon, 24 Feb 2025 14:43:27 +0100 Subject: [PATCH 24/37] test(mps-sync-plugin): variation with active binding --- .../org/modelix/mps/sync3/ProjectSyncTest.kt | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index f14ff9b38f..6df4c845db 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -343,7 +343,7 @@ class ProjectSyncTest : MPSTestBase() { assertEquals(expected.asNormalizedJson(), version2.asNormalizedJson()) } - fun `test sync to MPS after non-trivial commit`(): Unit = runWithModelServer { port -> + fun `test sync to MPS after non-trivial commit at startup`(): Unit = runWithModelServer { port -> // Two clients are in sync with the same version ... val branchRef = RepositoryId("sync-test").getBranchReference() val version1 = syncProjectToServer("initial", port, branchRef) @@ -367,6 +367,37 @@ class ProjectSyncTest : MPSTestBase() { assertEquals(expectedSnapshot, project.captureSnapshot()) } + fun `test sync to MPS after non-trivial commit with active binding`(): Unit = runWithModelServer { port -> + // Two clients are in sync with the same version ... + val branchRef = RepositoryId("sync-test").getBranchReference("branchA") + val version1 = syncProjectToServer("initial", port, branchRef) + + // ... and while one client is disconnected, the other client continues making changes. + val version2 = syncProjectToServer("change1", port, branchRef, version1.getContentHash()) + val expectedSnapshot = lastSnapshotBeforeSync + + println("initial two versions pushed") + + val branchRef2 = branchRef.repositoryId.getBranchReference("branchB") + val client = ModelClientV2.builder().url("http://localhost:$port").build() + client.push(branchRef2, version1, null) + + // The second client then reconnects ... + openTestProject("initial") + val snap1 = project.captureSnapshot() + val binding = IModelSyncService.getInstance(mpsProject) + .addServer("http://localhost:$port") + .bind(branchRef2, null) + binding.flush() + assertEquals(snap1, project.captureSnapshot()) + + client.push(branchRef2, version2, version1) + binding.flush() + + // ... applies all the pending changes and is again in sync with the other client + assertEquals(expectedSnapshot, project.captureSnapshot()) + } + fun `test loading enabled persisted binding`(): Unit = runPersistedBindingTest(true) fun `test loading disabled persisted binding`(): Unit = runPersistedBindingTest(false) From 4a7f31746b711d9660a732b1321157f6a4374e14 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 26 Feb 2025 09:49:12 +0100 Subject: [PATCH 25/37] feat(model-server): include oauth endpoints in WWW-Authenticate on 401 --- authorization/build.gradle.kts | 1 + .../authorization/AccessTokenPrincipal.kt | 9 ++- .../authorization/AuthorizationConfig.kt | 2 +- .../authorization/AuthorizationPlugin.kt | 64 ++++++++++++------- .../modelix/authorization/ModelixJWTUtil.kt | 12 ++-- .../org/modelix/kotlin/utils/Collections.kt | 4 ++ 6 files changed, 59 insertions(+), 33 deletions(-) create mode 100644 kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/Collections.kt diff --git a/authorization/build.gradle.kts b/authorization/build.gradle.kts index 9554a66a46..8120c202d1 100644 --- a/authorization/build.gradle.kts +++ b/authorization/build.gradle.kts @@ -10,6 +10,7 @@ java { } dependencies { + implementation(project(":kotlin-utils")) implementation(libs.kotlin.serialization.json) implementation(libs.kotlin.serialization.yaml) implementation(libs.guava) diff --git a/authorization/src/main/kotlin/org/modelix/authorization/AccessTokenPrincipal.kt b/authorization/src/main/kotlin/org/modelix/authorization/AccessTokenPrincipal.kt index ca155a5fb8..a59d37acdd 100644 --- a/authorization/src/main/kotlin/org/modelix/authorization/AccessTokenPrincipal.kt +++ b/authorization/src/main/kotlin/org/modelix/authorization/AccessTokenPrincipal.kt @@ -1,17 +1,16 @@ package org.modelix.authorization -import com.auth0.jwt.interfaces.DecodedJWT -import io.ktor.server.auth.Principal +import com.auth0.jwt.interfaces.Payload -class AccessTokenPrincipal(val jwt: DecodedJWT) : Principal { +class AccessTokenPrincipal(val jwt: Payload) { fun getUserName(): String? = ModelixJWTUtil.extractUserId(jwt) override fun equals(other: Any?): Boolean { if (other !is AccessTokenPrincipal) return false - return other.jwt.token.equals(jwt.token) + return other.jwt.claims == jwt.claims } override fun hashCode(): Int { - return jwt.token.hashCode() + return jwt.claims.hashCode() } } diff --git a/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationConfig.kt b/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationConfig.kt index 260ff39e01..964775ae0d 100644 --- a/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationConfig.kt +++ b/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationConfig.kt @@ -188,7 +188,7 @@ class ModelixAuthorizationConfig : IModelixAuthorizationConfig { * * The fake token is generated so that we always have a username that can be used in the server logic. */ - fun shouldGenerateFakeTokens() = generateFakeTokens ?: !jwtUtil.canVerifyTokens() + fun shouldGenerateFakeTokens() = generateFakeTokens ?: !permissionCheckingEnabled() /** * Whether permission checking should be enabled based on the configuration values provided. diff --git a/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationPlugin.kt b/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationPlugin.kt index eb96c26252..53beab289e 100644 --- a/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationPlugin.kt +++ b/authorization/src/main/kotlin/org/modelix/authorization/AuthorizationPlugin.kt @@ -7,20 +7,23 @@ import com.auth0.jwt.interfaces.DecodedJWT import com.auth0.jwt.interfaces.JWTVerifier import com.google.common.cache.CacheBuilder import com.nimbusds.jose.jwk.JWKSet +import com.nimbusds.jwt.proc.BadJWTException import io.ktor.http.ContentType import io.ktor.http.HttpStatusCode +import io.ktor.http.auth.HttpAuthHeader import io.ktor.server.application.Application import io.ktor.server.application.ApplicationCall import io.ktor.server.application.ApplicationCallPipeline import io.ktor.server.application.BaseRouteScopedPlugin -import io.ktor.server.application.call import io.ktor.server.application.install import io.ktor.server.application.plugin import io.ktor.server.auth.Authentication import io.ktor.server.auth.AuthenticationContext import io.ktor.server.auth.AuthenticationProvider +import io.ktor.server.auth.UnauthorizedResponse import io.ktor.server.auth.authenticate import io.ktor.server.auth.jwt.jwt +import io.ktor.server.auth.parseAuthorizationHeader import io.ktor.server.auth.principal import io.ktor.server.plugins.forwardedheaders.XForwardedHeaders import io.ktor.server.plugins.statuspages.StatusPages @@ -40,6 +43,7 @@ import org.modelix.authorization.permissions.PermissionParts import org.modelix.authorization.permissions.SchemaInstance import org.modelix.authorization.permissions.recordKnownRoles import org.modelix.authorization.permissions.recordKnownUser +import org.modelix.kotlin.utils.filterNotNullValues import java.util.concurrent.TimeUnit private val LOG = mu.KotlinLogging.logger { } @@ -79,29 +83,41 @@ object ModelixAuthorization : BaseRouteScopedPlugin + call.request.parseAuthorizationHeader() + ?: call.request.headers["X-Forwarded-Access-Token"] + ?.let { HttpAuthHeader.Single("Bearer", it) } + } + verifier(config.getVerifier()) - challenge { _, _ -> - call.respond(status = HttpStatusCode.Unauthorized, "No or invalid JWT token provided") + challenge { scheme, realm -> + call.respond( + UnauthorizedResponse( + HttpAuthHeader.Parameterized( + scheme, + mapOf( + HttpAuthHeader.Parameters.Realm to realm, + "error" to "invalid_token", + "authorization_uri" to System.getenv("MODELIX_AUTHORIZATION_URI")?.takeIf { it.isNotBlank() }, + "token_uri" to System.getenv("MODELIX_TOKEN_URI")?.takeIf { it.isNotBlank() }, + ).filterNotNullValues(), + ), + ), + ) + // login and token generation is done by OAuth proxy. Only validation is required here. } - validate { - try { + validate { credential -> + val jwt = credential.payload + application.launch(Dispatchers.IO) { val authPlugin = application.plugin(ModelixAuthorization) val authConfig = authPlugin.config - jwtFromHeaders() - ?.let { authConfig.nullIfInvalid(it) } - ?.also { jwt -> - application.launch(Dispatchers.IO) { - val accessControlPersistence = authConfig.accessControlPersistence - accessControlPersistence.recordKnownUser(authConfig.jwtUtil.extractUserId(jwt)) - accessControlPersistence.recordKnownRoles(authConfig.jwtUtil.extractUserRoles(jwt)) - } - } - ?.let(::AccessTokenPrincipal) - } catch (e: Exception) { - LOG.warn(e) { "Failed to read JWT token" } - null + val accessControlPersistence = authConfig.accessControlPersistence + accessControlPersistence.recordKnownUser(authConfig.jwtUtil.extractUserId(jwt)) + accessControlPersistence.recordKnownRoles(authConfig.jwtUtil.extractUserRoles(jwt)) } + AccessTokenPrincipal(jwt) } } } @@ -135,7 +151,7 @@ object ModelixAuthorization : BaseRouteScopedPlugin()?.jwt ?: call.jwtFromHeaders() + val jwt = call.jwtFromHeaders() if (jwt == null) { call.respondText("No JWT token available") } else { @@ -226,9 +242,13 @@ internal fun ModelixAuthorizationConfig.getVerifier() = object : JWTVerifier { } override fun verify(jwt: DecodedJWT?): DecodedJWT { - if (jwt == null) { - throw JWTVerificationException("No JWT provided.") + try { + if (jwt == null) { + throw JWTVerificationException("No JWT provided.") + } + return this@getVerifier.nullIfInvalid(jwt)?.also { println("Valid token: ${jwt.token}") } ?: throw JWTVerificationException("JWT invalid.") + } catch (ex: BadJWTException) { + throw JWTVerificationException("Invalid token: ${jwt?.token}", ex) } - return this@getVerifier.nullIfInvalid(jwt) ?: throw JWTVerificationException("JWT invalid.") } } diff --git a/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt b/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt index 6dc5f3d9e3..06baf5be3d 100644 --- a/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt +++ b/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt @@ -1,6 +1,7 @@ package org.modelix.authorization import com.auth0.jwt.interfaces.DecodedJWT +import com.auth0.jwt.interfaces.Payload import com.nimbusds.jose.JWSAlgorithm import com.nimbusds.jose.JWSHeader import com.nimbusds.jose.JWSObject @@ -18,6 +19,7 @@ import com.nimbusds.jose.jwk.KeyType import com.nimbusds.jose.jwk.KeyUse import com.nimbusds.jose.jwk.RSAKey import com.nimbusds.jose.jwk.gen.RSAKeyGenerator +import com.nimbusds.jose.jwk.source.DefaultJWKSetCache import com.nimbusds.jose.jwk.source.ImmutableJWKSet import com.nimbusds.jose.jwk.source.JWKSource import com.nimbusds.jose.jwk.source.RemoteJWKSet @@ -244,12 +246,12 @@ class ModelixJWTUtil { return PermissionEvaluator(schema).also { loadGrantedPermissions(token, it) } } - fun extractPermissions(token: DecodedJWT): List? { + fun extractPermissions(token: Payload): List? { return token.claims[ModelixTokenConstants.PERMISSIONS]?.asList(String::class.java) } @Synchronized - fun loadGrantedPermissions(token: DecodedJWT, evaluator: PermissionEvaluator) { + fun loadGrantedPermissions(token: Payload, evaluator: PermissionEvaluator) { val permissions = extractPermissions(token) // There is a difference between access tokens and identity tokens. @@ -271,11 +273,11 @@ class ModelixJWTUtil { } } - fun extractUserId(jwt: DecodedJWT): String? { + fun extractUserId(jwt: Payload): String? { return Companion.extractUserId(jwt) } - fun extractUserRoles(jwt: DecodedJWT): List { + fun extractUserRoles(jwt: Payload): List { val keycloakRoles = jwt .getClaim(KeycloakTokenConstants.REALM_ACCESS)?.asMap() ?.get(KeycloakTokenConstants.REALM_ACCESS_ROLES) @@ -372,7 +374,7 @@ class ModelixJWTUtil { } companion object { - fun extractUserId(jwt: DecodedJWT): String? { + fun extractUserId(jwt: Payload): String? { return jwt.getClaim(KeycloakTokenConstants.EMAIL)?.asString() ?: jwt.getClaim(KeycloakTokenConstants.PREFERRED_USERNAME)?.asString() } diff --git a/kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/Collections.kt b/kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/Collections.kt new file mode 100644 index 0000000000..a135d2bce6 --- /dev/null +++ b/kotlin-utils/src/commonMain/kotlin/org/modelix/kotlin/utils/Collections.kt @@ -0,0 +1,4 @@ +package org.modelix.kotlin.utils + +@Suppress("UNCHECKED_CAST") +fun Map.filterNotNullValues(): Map = filterValues { it != null } as Map From 66a17adbce86fb157e9b780e55e65592fa4893e3 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 26 Feb 2025 09:51:26 +0100 Subject: [PATCH 26/37] fix(model-server): some versions were missing on the history page --- .../server/handlers/ui/HistoryHandler.kt | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/ui/HistoryHandler.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/ui/HistoryHandler.kt index edffda64ef..33760af7f8 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/ui/HistoryHandler.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/ui/HistoryHandler.kt @@ -257,37 +257,49 @@ class HistoryHandler(private val repositoriesManager: IRepositoriesManager) { } } tbody { - var version: CLVersion? = headVersion - while (version != null) { - if (rowIndex >= skip) { - createTableRow(repositoryAndBranch.repositoryId, version, latestVersion) + val versions = sequence { + var version: CLVersion? = headVersion + while (version != null) { + yield(version) if (version.isMerge()) { for (v in LinearHistory(version.baseVersion!!.getContentHash()).load(version.getMergedVersion1()!!, version.getMergedVersion2()!!)) { - createTableRow(repositoryAndBranch.repositoryId, v, latestVersion) - rowIndex++ - if (rowIndex >= skip + limit) { - break - } + yield(v) + v.baseVersion?.let { yield(it) } // to include merge commits } } + version = version.baseVersion } - rowIndex++ - if (rowIndex >= skip + limit) { - break + }.distinct().drop(skip).take(limit) + + var previous: CLVersion? = null + for (version in versions) { + if (previous != null) { + createTableRow(repositoryAndBranch.repositoryId, previous, version, latestVersion) } - version = version.baseVersion + previous = version + } + if (previous != null) { + createTableRow(repositoryAndBranch.repositoryId, previous, null, latestVersion) } } } buttons() } - private fun TBODY.createTableRow(repositoryId: RepositoryId, version: CLVersion, latestVersion: CLVersion) { + private fun TBODY.createTableRow(repositoryId: RepositoryId, version: CLVersion, nextVersion: CLVersion?, latestVersion: CLVersion) { tr { td { +version.id.toString(16) br { } span(classes = "hash") { +version.getContentHash() } + + if (nextVersion != null && version.baseVersion?.getContentHash() != nextVersion.getContentHash()) { + br { } + span(classes = "hash") { + +"parent: " + +version.baseVersion?.getContentHash().orEmpty() + } + } } td { style = "white-space: nowrap;" @@ -299,7 +311,7 @@ class HistoryHandler(private val repositoriesManager: IRepositoriesManager) { } td { if (version.isMerge()) { - +"merge ${version.getMergedVersion1()!!.id} + ${version.getMergedVersion2()!!.id} (base ${version.baseVersion})" + +"merge ${version.getMergedVersion1()} + ${version.getMergedVersion2()} (base ${version.baseVersion})" } else { if (version.operationsInlined()) { ul { From 73cc5c57ac8e6bb1a00f3f4eae9998c0b1b32dfa Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 26 Feb 2025 09:52:45 +0100 Subject: [PATCH 27/37] fix(model-server): don't require login for the /headers endpoint Otherwise, it can't be used for debugging authentication issues. --- .../handlers/KeyValueLikeModelServer.kt | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/KeyValueLikeModelServer.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/KeyValueLikeModelServer.kt index 56850b360f..c7f98dbd62 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/KeyValueLikeModelServer.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/KeyValueLikeModelServer.kt @@ -72,23 +72,23 @@ class KeyValueLikeModelServer( private fun Application.modelServerModule() { routing { - requiresLogin { - get { - val headers = call.request.headers.entries().flatMap { e -> e.value.map { e.key to it } } - call.respondHtmlTemplate(PageWithMenuBar("headers", ".")) { - bodyContent { - h1 { +"HTTP Headers" } - div { - headers.forEach { - span { - +"${it.first}: ${it.second}" - } - br { } + get { + val headers = call.request.headers.entries().flatMap { e -> e.value.map { e.key to it } } + call.respondHtmlTemplate(PageWithMenuBar("headers", ".")) { + bodyContent { + h1 { +"HTTP Headers" } + div { + headers.forEach { + span { + +"${it.first}: ${it.second}" } + br { } } } } } + } + requiresLogin { get { val key = call.parameters["key"]!! checkKeyPermission(key, EPermissionType.READ) From 8a07819689fa6239382fe047cd7162ef7d2eecbd Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 26 Feb 2025 09:56:22 +0100 Subject: [PATCH 28/37] fix(mps-sync-plugin): handle exceptions during initial sync --- .../kotlin/org/modelix/mps/sync3/BindingWorker.kt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt index a514edc970..0f6eabc057 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt @@ -2,6 +2,7 @@ package org.modelix.mps.sync3 import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.ModalityState +import io.ktor.utils.io.CancellationException import jetbrains.mps.project.MPSProject import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -91,7 +92,17 @@ class BindingWorker( private suspend fun CoroutineScope.syncJob() { // initial sync - initialSync() + while (isActive()) { + try { + initialSync() + break + } catch (ex: CancellationException) { + break + } catch (ex: Exception) { + LOG.error(ex) { "Initial synchronization failed" } + delay(5_000) + } + } // continuous sync to MPS launchLoop { From c4c67874667c47402ef20dfb2c3911148ad68c12 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 26 Feb 2025 10:55:43 +0100 Subject: [PATCH 29/37] fix(model-client): OAuth login --- .../modelix/authorization/ModelixJWTUtil.kt | 1 - model-client/build.gradle.kts | 11 ++ .../modelix/model/client2/ModelClientV2.kt | 8 +- .../modelix/model/oauth/ModelixAuthClient.kt | 7 +- .../modelix/model/oauth/ModelixAuthClient.kt | 7 +- .../modelix/model/oauth/ModelixAuthClient.kt | 103 +++++++--- .../org/modelix/model/client2/OAuthTest.kt | 126 +++++++++++++ .../src/jvmTest/resources/logback-test.xml | 21 +++ model-server-with-auth/compose.yaml | 66 +++++++ model-server-with-auth/realm.json | 177 ++++++++++++++++++ mps-sync-plugin3/compose.yaml | 66 +++++++ mps-sync-plugin3/realm.json | 177 ++++++++++++++++++ 12 files changed, 745 insertions(+), 25 deletions(-) create mode 100644 model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt create mode 100644 model-client/src/jvmTest/resources/logback-test.xml create mode 100644 model-server-with-auth/compose.yaml create mode 100644 model-server-with-auth/realm.json create mode 100644 mps-sync-plugin3/compose.yaml create mode 100644 mps-sync-plugin3/realm.json diff --git a/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt b/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt index 06baf5be3d..08e9f89256 100644 --- a/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt +++ b/authorization/src/main/kotlin/org/modelix/authorization/ModelixJWTUtil.kt @@ -19,7 +19,6 @@ import com.nimbusds.jose.jwk.KeyType import com.nimbusds.jose.jwk.KeyUse import com.nimbusds.jose.jwk.RSAKey import com.nimbusds.jose.jwk.gen.RSAKeyGenerator -import com.nimbusds.jose.jwk.source.DefaultJWKSetCache import com.nimbusds.jose.jwk.source.ImmutableJWKSet import com.nimbusds.jose.jwk.source.JWKSource import com.nimbusds.jose.jwk.source.RemoteJWKSet diff --git a/model-client/build.gradle.kts b/model-client/build.gradle.kts index da3144d437..57a06e159c 100644 --- a/model-client/build.gradle.kts +++ b/model-client/build.gradle.kts @@ -66,6 +66,12 @@ kotlin { implementation(libs.ktor.serialization.json) } } + jvmTest { + dependencies { + implementation(libs.logback.classic) + implementation(libs.testcontainers) + } + } val jsMain by getting { languageSettings.optIn("kotlin.js.ExperimentalJsExport") dependencies { @@ -156,3 +162,8 @@ npmPublish { tasks.withType(NodeExecTask::class) { dependsOn(":setupNodeEverywhere") } + +tasks.jvmTest { + dependsOn(":model-server:assemble") + environment("KEYCLOAK_VERSION", libs.versions.keycloak.get()) +} diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt index 092887a9ee..db3d5a2054 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt @@ -492,6 +492,7 @@ abstract class ModelClientV2Builder { protected var httpClient: HttpClient? = null protected var baseUrl: String = "https://localhost/model/v2" protected var authTokenProvider: (suspend () -> String?)? = null + protected var authRequestBrowser: ((url: String) -> Unit)? = null protected var userId: String? = null protected var connectTimeout: Duration = 1.seconds protected var requestTimeout: Duration = 30.seconds @@ -522,6 +523,11 @@ abstract class ModelClientV2Builder { return this } + fun authRequestBrowser(browser: ((url: String) -> Unit)?): ModelClientV2Builder { + authRequestBrowser = browser + return this + } + fun userId(userId: String?): ModelClientV2Builder { this.userId = userId return this @@ -574,7 +580,7 @@ abstract class ModelClientV2Builder { } } } - ModelixAuthClient.installAuth(this, baseUrl, authTokenProvider) + ModelixAuthClient.installAuth(this, baseUrl, authTokenProvider, authRequestBrowser) } } diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt b/model-client/src/commonMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt index 0038c49368..04940f0ca3 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt @@ -32,7 +32,12 @@ expect object ModelixAuthClient { * and to refresh it when the old one expired. * Returning `null` cause the client to attempt the request without a token. */ - fun installAuth(config: HttpClientConfig<*>, baseUrl: String, authTokenProvider: (suspend () -> String?)? = null) + fun installAuth( + config: HttpClientConfig<*>, + baseUrl: String, + authTokenProvider: (suspend () -> String?)? = null, + authRequestBrowser: ((url: String) -> Unit)? = null, + ) } internal fun installAuthWithAuthTokenProvider(config: HttpClientConfig<*>, authTokenProvider: suspend () -> String?) { diff --git a/model-client/src/jsMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt b/model-client/src/jsMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt index b4aafc1b74..97125afcac 100644 --- a/model-client/src/jsMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt +++ b/model-client/src/jsMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt @@ -5,7 +5,12 @@ import io.ktor.client.HttpClientConfig @Suppress("UndocumentedPublicClass") // already documented in the expected declaration actual object ModelixAuthClient { @Suppress("UndocumentedPublicFunction") // already documented in the expected declaration - actual fun installAuth(config: HttpClientConfig<*>, baseUrl: String, authTokenProvider: (suspend () -> String?)?) { + actual fun installAuth( + config: HttpClientConfig<*>, + baseUrl: String, + authTokenProvider: (suspend () -> String?)?, + authRequestBrowser: ((url: String) -> Unit)?, + ) { if (authTokenProvider != null) { installAuthWithAuthTokenProvider(config, authTokenProvider) } diff --git a/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt b/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt index f736b5f5bf..2d36a71e7f 100644 --- a/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt +++ b/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt @@ -18,13 +18,16 @@ import io.ktor.client.HttpClientConfig import io.ktor.client.plugins.auth.Auth import io.ktor.client.plugins.auth.providers.BearerTokens import io.ktor.client.plugins.auth.providers.bearer +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpMessage +import io.ktor.http.auth.HttpAuthHeader +import io.ktor.http.auth.parseAuthorizationHeader import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @Suppress("UndocumentedPublicClass") // already documented in the expected declaration actual object ModelixAuthClient { private var DATA_STORE_FACTORY: DataStoreFactory = MemoryDataStoreFactory() - private val SCOPE = "email" private val HTTP_TRANSPORT: HttpTransport = NetHttpTransport() private val JSON_FACTORY: JsonFactory = GsonFactory() @@ -33,37 +36,68 @@ actual object ModelixAuthClient { } suspend fun authorize(modelixServerUrl: String): Credential { + val oidcUrl = modelixServerUrl.trimEnd('/') + "/realms/modelix/protocol/openid-connect" + return authorize( + clientId = "external-mps", + scopes = listOf("email"), + authUrl = "$oidcUrl/auth", + tokenUrl = "$oidcUrl/token", + authRequestBrowser = null, + ) + } + + suspend fun authorize( + clientId: String, + scopes: List, + authUrl: String, + tokenUrl: String, + authRequestBrowser: ((url: String) -> Unit)?, + ): Credential { return withContext(Dispatchers.IO) { - val oidcUrl = modelixServerUrl.trimEnd('/') + "/realms/modelix/protocol/openid-connect" - val clientId = "external-mps" val flow = AuthorizationCodeFlow.Builder( BearerToken.authorizationHeaderAccessMethod(), HTTP_TRANSPORT, JSON_FACTORY, - GenericUrl("$oidcUrl/token"), + GenericUrl(tokenUrl), ClientParametersAuthentication(clientId, null), clientId, - "$oidcUrl/auth", + authUrl, ) - .setScopes(listOf(SCOPE)) + .setScopes(scopes) .enablePKCE() .setDataStoreFactory(DATA_STORE_FACTORY) .build() val receiver: LocalServerReceiver = LocalServerReceiver.Builder().setHost("127.0.0.1").build() - AuthorizationCodeInstalledApp(flow, receiver).authorize("user") + val browser = authRequestBrowser?.let { + object : AuthorizationCodeInstalledApp.Browser { + override fun browse(url: String) { + it(url) + } + } + } ?: AuthorizationCodeInstalledApp.DefaultBrowser() + AuthorizationCodeInstalledApp(flow, receiver, browser).authorize("user") } } @Suppress("UndocumentedPublicFunction") // already documented in the expected declaration - actual fun installAuth(config: HttpClientConfig<*>, baseUrl: String, authTokenProvider: (suspend () -> String?)?) { + actual fun installAuth( + config: HttpClientConfig<*>, + baseUrl: String, + authTokenProvider: (suspend () -> String?)?, + authRequestBrowser: ((url: String) -> Unit)?, + ) { if (authTokenProvider != null) { installAuthWithAuthTokenProvider(config, authTokenProvider) } else { - installAuthWithPKCEFlow(config, baseUrl) + installAuthWithPKCEFlow(config, baseUrl, authRequestBrowser) } } - private fun installAuthWithPKCEFlow(config: HttpClientConfig<*>, baseUrl: String) { + private fun installAuthWithPKCEFlow( + config: HttpClientConfig<*>, + baseUrl: String, + authRequestBrowser: ((url: String) -> Unit)?, + ) { config.apply { install(Auth) { bearer { @@ -71,21 +105,48 @@ actual object ModelixAuthClient { getTokens()?.let { BearerTokens(it.accessToken, it.refreshToken) } } refreshTokens { - var url = baseUrl - if (!url.endsWith("/")) url += "/" - // XXX Detecting and removing "/model/" is workaround for when the model server - // is used in Modelix workspaces and reachable behind the sub path /model/". - // When the model server is reachable at https://example.org/model/, - // Keycloak is expected to be reachable under https://example.org/realms/ - // See https://github.com/modelix/modelix.kubernetes/blob/60f7db6533c3fb82209b1a6abb6836923f585672/proxy/nginx.conf#L14 - // and https://github.com/modelix/modelix.kubernetes/blob/60f7db6533c3fb82209b1a6abb6836923f585672/proxy/nginx.conf#L41 - // TODO MODELIX-975 remove this check and replace with configuration. - if (url.endsWith("/model/")) url = url.substringBeforeLast("/model/") - val tokens = authorize(url) + val tokens = response.parseWWWAuthenticate()?.let { wwwAuthenticate -> + // The model server tells the client where to get a token + + if (wwwAuthenticate.parameter("error") != "invalid_token") return@let null + val authUrl = wwwAuthenticate.parameter("authorization_uri") ?: return@let null + val tokenUrl = wwwAuthenticate.parameter("token_uri") ?: return@let null + val realm = wwwAuthenticate.parameter("realm") + val description = wwwAuthenticate.parameter("error_description") + authorize( + clientId = "modelix-sync-plugin", + scopes = listOf("sync"), + authUrl = authUrl, + tokenUrl = tokenUrl, + authRequestBrowser = authRequestBrowser, + ) + } ?: let { + // legacy keycloak specific URLs + + var url = baseUrl + if (!url.endsWith("/")) url += "/" + // XXX Detecting and removing "/model/" is workaround for when the model server + // is used in Modelix workspaces and reachable behind the sub path /model/". + // When the model server is reachable at https://example.org/model/, + // Keycloak is expected to be reachable under https://example.org/realms/ + // See https://github.com/modelix/modelix.kubernetes/blob/60f7db6533c3fb82209b1a6abb6836923f585672/proxy/nginx.conf#L14 + // and https://github.com/modelix/modelix.kubernetes/blob/60f7db6533c3fb82209b1a6abb6836923f585672/proxy/nginx.conf#L41 + // TODO MODELIX-975 remove this check and replace with configuration. + if (url.endsWith("/model/")) url = url.substringBeforeLast("/model/") + authorize(url) + } + + println("Access token: ${tokens.accessToken}") + println("Refresh token: ${tokens.refreshToken}") BearerTokens(tokens.accessToken, tokens.refreshToken) } } } } } + + fun HttpMessage.parseWWWAuthenticate(): HttpAuthHeader.Parameterized? { + return headers[HttpHeaders.WWWAuthenticate] + ?.let { parseAuthorizationHeader(it) as? HttpAuthHeader.Parameterized } + } } diff --git a/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt b/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt new file mode 100644 index 0000000000..c94ecc371d --- /dev/null +++ b/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt @@ -0,0 +1,126 @@ +package org.modelix.model.client2 + +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.cookies.AcceptAllCookiesStorage +import io.ktor.client.plugins.cookies.CookiesStorage +import io.ktor.client.plugins.cookies.HttpCookies +import io.ktor.client.request.forms.submitForm +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.Cookie +import io.ktor.http.HttpHeaders +import io.ktor.http.Url +import io.ktor.http.parameters +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import org.modelix.model.api.ITree +import org.modelix.model.lazy.RepositoryId +import org.testcontainers.containers.ComposeContainer +import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.Network +import org.testcontainers.images.builder.ImageFromDockerfile +import org.testcontainers.utility.MountableFile +import java.nio.file.Path +import kotlin.io.path.absolute +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.minutes +import kotlin.time.ExperimentalTime + +private val modelServerDir = Path.of("../model-server").absolute().normalize() +private val modelServerImage = ImageFromDockerfile() + .withDockerfile(modelServerDir.resolve("Dockerfile")) + +class OAuthTest { + + @Test + fun test() = runWithModelServer { url -> + val client = ModelClientV2.builder() + .url(url) + .retries(1U) + .authRequestBrowser { authUrl -> + runBlocking { + handleOAuthLogin(authUrl, "user1", "abc") + } + } + .build() + client.init() + + val version = client.initRepository(RepositoryId("oauth-test-repo")) + assertEquals(0, version.getTree().getAllChildren(ITree.ROOT_ID).count()) + } + + private suspend fun handleOAuthLogin(authUrl: String, user: String, password: String) { + val acceptAllCookiesStorage = AcceptAllCookiesStorage() + val cookiesStorage = object : CookiesStorage by acceptAllCookiesStorage { + override suspend fun addCookie(requestUrl: Url, cookie: Cookie) { + acceptAllCookiesStorage.addCookie(requestUrl, cookie.copy(secure = false)) + } + } + val httpClient = HttpClient(CIO) { + install(HttpCookies) { + storage = cookiesStorage + } + } + val html = httpClient.get(authUrl).also { println(it.headers.entries()) }.bodyAsText() + + val loginUrl = Regex("""[^"]+/login-actions/authenticate[^"]+""").find(html)!!.value + + val callbackUrl = httpClient.submitForm( + url = loginUrl, + formParameters = parameters { + set("username", user) + set("password", password) + }, + ).headers[HttpHeaders.Location]!! + + httpClient.get(callbackUrl).bodyAsText() + } + + private fun runWithModelServer(body: suspend (url: String) -> Unit) = runBlocking { + @OptIn(ExperimentalTime::class) + withTimeout(3.minutes) { + val network = Network.newNetwork() + + val keycloak: GenericContainer<*> = GenericContainer("quay.io/keycloak/keycloak:${System.getenv("KEYCLOAK_VERSION")}") + .withExposedPorts(8080) + .withCommand("start-dev", "--import-realm") + .withEnv("KEYCLOAK_ADMIN", "admin") + .withEnv("KEYCLOAK_ADMIN_PASSWORD", "admin") + .withEnv("KC_HTTP_PORT", "8080") + .withEnv("KC_HOSTNAME", "localhost") + .withCopyFileToContainer(MountableFile.forHostPath("../model-server-with-auth/realm.json"), "/opt/keycloak/data/import/realm.json") + .withNetwork(network) + .withNetworkAliases("keycloak") + .withLogConsumer { println("[KEYCLOAK] " + it.utf8StringWithoutLineEnding) } + keycloak.start() + val keycloakPort = keycloak.getMappedPort(8080) + + val modelServer: GenericContainer<*> = GenericContainer(modelServerImage) + .withExposedPorts(28101) + .withCommand("--inmemory") + .withEnv("MODELIX_AUTHORIZATION_URI", "http://localhost:$keycloakPort/realms/modelix/protocol/openid-connect/auth") + .withEnv("MODELIX_TOKEN_URI", "http://localhost:$keycloakPort/realms/modelix/protocol/openid-connect/token") + .withEnv("MODELIX_PERMISSION_CHECKS_ENABLED", "true") + .withEnv("MODELIX_JWK_URI_KEYCLOAK", "http://keycloak:8080/realms/modelix/protocol/openid-connect/certs") + .withNetwork(network) + .withNetworkAliases("model-server") + .withLogConsumer { println("[MODEL] " + it.utf8StringWithoutLineEnding) } + modelServer.start() + + try { + body("http://localhost:${modelServer.firstMappedPort}/") + } finally { + modelServer.stop() + keycloak.stop() + } + } + } +} + +private fun ComposeContainer.getServiceUrl(name: String, port: Int): String { + val h = getServiceHost(name, port) + val p = getServicePort(name, port) + return "http://$h:$p/" +} diff --git a/model-client/src/jvmTest/resources/logback-test.xml b/model-client/src/jvmTest/resources/logback-test.xml new file mode 100644 index 0000000000..2ea72810ee --- /dev/null +++ b/model-client/src/jvmTest/resources/logback-test.xml @@ -0,0 +1,21 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/model-server-with-auth/compose.yaml b/model-server-with-auth/compose.yaml new file mode 100644 index 0000000000..753fab1740 --- /dev/null +++ b/model-server-with-auth/compose.yaml @@ -0,0 +1,66 @@ +services: + keycloak: + image: quay.io/keycloak/keycloak:latest + command: start-dev --import-realm + environment: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KC_HTTP_PORT: 8080 + KC_HOSTNAME: localhost + volumes: + - ./realm.json:/opt/keycloak/data/import/realm.json + ports: + - "28180:8080" + networks: + - app-network + + oauth2-proxy: + image: quay.io/oauth2-proxy/oauth2-proxy:latest + environment: + OAUTH2_PROXY_PROVIDER: keycloak-oidc + OAUTH2_PROXY_OIDC_ISSUER_URL: http://keycloak:8080/realms/modelix + OAUTH2_PROXY_SKIP_OIDC_DISCOVERY: "true" + OAUTH2_PROXY_LOGIN_URL: http://localhost:28180/realms/modelix/protocol/openid-connect/auth + OAUTH2_PROXY_REDEEM_URL: http://keycloak:8080/realms/modelix/protocol/openid-connect/token + OAUTH2_PROXY_OIDC_JWKS_URL: http://keycloak:8080/realms/modelix/protocol/openid-connect/certs + OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true" + OAUTH2_PROXY_INSECURE_OIDC_SKIP_ISSUER_VERIFICATION: "true" + OAUTH2_PROXY_CLIENT_ID: oauth2-proxy + OAUTH2_PROXY_CLIENT_SECRET: 8g!.4RpL2tPxdXMMK6mb + OAUTH2_PROXY_COOKIE_SECRET: jLTKkbMwRJpsS7ZW + OAUTH2_PROXY_UPSTREAMS: http://model-server:28101/ + OAUTH2_PROXY_REDIRECT_URL: http://localhost:4180/oauth2/callback + OAUTH2_PROXY_EMAIL_DOMAINS: "*" + OAUTH2_PROXY_COOKIE_HTTPONLY: "false" + OAUTH2_PROXY_COOKIE_SECURE: "false" + OAUTH2_PROXY_SET_AUTHORIZATION_HEADER: "true" + OAUTH2_PROXY_PASS_ACCESS_TOKEN: "true" + OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180" + OAUTH2_PROXY_SCOPE: "sync openid" + ports: + - "4180:4180" + depends_on: + - keycloak + networks: + - app-network + + model-server: + build: + context: ../model-server + dockerfile: Dockerfile + command: ["--inmemory"] + environment: + MODELIX_AUTHORIZATION_URI: http://localhost:28180/realms/modelix/protocol/openid-connect/auth + MODELIX_TOKEN_URI: http://localhost:28180/realms/modelix/protocol/openid-connect/token + MODELIX_PERMISSION_CHECKS_ENABLED: true + MODELIX_JWK_URI_KEYCLOAK: http://keycloak:8080/realms/modelix/protocol/openid-connect/certs + ports: + - "28101:28101" + networks: + - app-network + depends_on: + - keycloak + +networks: + app-network: + driver: bridge diff --git a/model-server-with-auth/realm.json b/model-server-with-auth/realm.json new file mode 100644 index 0000000000..ef498144a1 --- /dev/null +++ b/model-server-with-auth/realm.json @@ -0,0 +1,177 @@ +{ + "realm": "modelix", + "enabled": true, + "clients": [ + { + "clientId": "oauth2-proxy", + "secret": "8g!.4RpL2tPxdXMMK6mb", + "enabled": true, + "directAccessGrantsEnabled": false, + "publicClient": false, + "defaultClientScopes": ["sync", "email"], + "redirectUris": [ + "http://localhost:4180/oauth2/callback" + ], + "protocolMappers": [ + { + "name": "aud-mapper-oauth2-proxy", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "oauth2-proxy", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] + }, + { + "clientId": "modelix-sync-plugin", + "enabled": true, + "directAccessGrantsEnabled": false, + "publicClient": true, + "defaultClientScopes": ["sync"], + "redirectUris": [ + "*" + ] + } + ], + "clientScopes": [ + { + "name": "sync", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "Read and write access for model synchronization" + }, + "protocolMappers": [ + { + "name": "sync-permissions-mapper", + "protocol": "openid-connect", + "protocolMapper": "oidc-hardcoded-claim-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "claim.value": "[\"model-server/admin\"]", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "permissions", + "jsonType.label": "JSON", + "access.tokenResponse.claim": "false" + } + } + ] + }, + { + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + }, + { + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + } + ] + } + ], + "users": [ + { + "username": "user1", + "email": "authorization-test-user@authorization-test-user.com", + "firstName": "authorization-test-user", + "lastName": "authorization-test-user", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "abc" + } + ] + } + ], + "components": { + "org.keycloak.keys.KeyProvider": [ + { + "name": "rsa-256-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "keySize": [ + "2048" + ], + "active": [ + "true" + ], + "priority": [ + "100" + ], + "enabled": [ + "true" + ], + "algorithm": [ + "RS256" + ] + } + }, + { + "name": "rsa-512-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "keySize": [ + "2048" + ], + "active": [ + "true" + ], + "priority": [ + "0" + ], + "enabled": [ + "true" + ], + "algorithm": [ + "RS512" + ] + } + } + ] + } +} diff --git a/mps-sync-plugin3/compose.yaml b/mps-sync-plugin3/compose.yaml new file mode 100644 index 0000000000..753fab1740 --- /dev/null +++ b/mps-sync-plugin3/compose.yaml @@ -0,0 +1,66 @@ +services: + keycloak: + image: quay.io/keycloak/keycloak:latest + command: start-dev --import-realm + environment: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KC_HTTP_PORT: 8080 + KC_HOSTNAME: localhost + volumes: + - ./realm.json:/opt/keycloak/data/import/realm.json + ports: + - "28180:8080" + networks: + - app-network + + oauth2-proxy: + image: quay.io/oauth2-proxy/oauth2-proxy:latest + environment: + OAUTH2_PROXY_PROVIDER: keycloak-oidc + OAUTH2_PROXY_OIDC_ISSUER_URL: http://keycloak:8080/realms/modelix + OAUTH2_PROXY_SKIP_OIDC_DISCOVERY: "true" + OAUTH2_PROXY_LOGIN_URL: http://localhost:28180/realms/modelix/protocol/openid-connect/auth + OAUTH2_PROXY_REDEEM_URL: http://keycloak:8080/realms/modelix/protocol/openid-connect/token + OAUTH2_PROXY_OIDC_JWKS_URL: http://keycloak:8080/realms/modelix/protocol/openid-connect/certs + OAUTH2_PROXY_INSECURE_OIDC_ALLOW_UNVERIFIED_EMAIL: "true" + OAUTH2_PROXY_INSECURE_OIDC_SKIP_ISSUER_VERIFICATION: "true" + OAUTH2_PROXY_CLIENT_ID: oauth2-proxy + OAUTH2_PROXY_CLIENT_SECRET: 8g!.4RpL2tPxdXMMK6mb + OAUTH2_PROXY_COOKIE_SECRET: jLTKkbMwRJpsS7ZW + OAUTH2_PROXY_UPSTREAMS: http://model-server:28101/ + OAUTH2_PROXY_REDIRECT_URL: http://localhost:4180/oauth2/callback + OAUTH2_PROXY_EMAIL_DOMAINS: "*" + OAUTH2_PROXY_COOKIE_HTTPONLY: "false" + OAUTH2_PROXY_COOKIE_SECURE: "false" + OAUTH2_PROXY_SET_AUTHORIZATION_HEADER: "true" + OAUTH2_PROXY_PASS_ACCESS_TOKEN: "true" + OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180" + OAUTH2_PROXY_SCOPE: "sync openid" + ports: + - "4180:4180" + depends_on: + - keycloak + networks: + - app-network + + model-server: + build: + context: ../model-server + dockerfile: Dockerfile + command: ["--inmemory"] + environment: + MODELIX_AUTHORIZATION_URI: http://localhost:28180/realms/modelix/protocol/openid-connect/auth + MODELIX_TOKEN_URI: http://localhost:28180/realms/modelix/protocol/openid-connect/token + MODELIX_PERMISSION_CHECKS_ENABLED: true + MODELIX_JWK_URI_KEYCLOAK: http://keycloak:8080/realms/modelix/protocol/openid-connect/certs + ports: + - "28101:28101" + networks: + - app-network + depends_on: + - keycloak + +networks: + app-network: + driver: bridge diff --git a/mps-sync-plugin3/realm.json b/mps-sync-plugin3/realm.json new file mode 100644 index 0000000000..0396cffcaf --- /dev/null +++ b/mps-sync-plugin3/realm.json @@ -0,0 +1,177 @@ +{ + "realm": "modelix", + "enabled": true, + "clients": [ + { + "clientId": "oauth2-proxy", + "secret": "8g!.4RpL2tPxdXMMK6mb", + "enabled": true, + "directAccessGrantsEnabled": false, + "publicClient": false, + "defaultClientScopes": ["sync", "email"], + "redirectUris": [ + "http://localhost:4180/oauth2/callback" + ], + "protocolMappers": [ + { + "name": "aud-mapper-oauth2-proxy", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "oauth2-proxy", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] + }, + { + "clientId": "modelix-sync-plugin", + "enabled": true, + "directAccessGrantsEnabled": false, + "publicClient": true, + "defaultClientScopes": ["sync"], + "redirectUris": [ + "*" + ] + } + ], + "clientScopes": [ + { + "name": "sync", + "description": "", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "gui.order": "", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "name": "sync-permissions-mapper", + "protocol": "openid-connect", + "protocolMapper": "oidc-hardcoded-claim-mapper", + "consentRequired": false, + "config": { + "introspection.token.claim": "true", + "claim.value": "[\"model-server/admin\"]", + "userinfo.token.claim": "true", + "id.token.claim": "true", + "lightweight.claim": "false", + "access.token.claim": "true", + "claim.name": "permissions", + "jsonType.label": "JSON", + "access.tokenResponse.claim": "false" + } + } + ] + }, + { + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + }, + { + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + } + ] + } + ], + "users": [ + { + "username": "user1", + "email": "authorization-test-user@authorization-test-user.com", + "firstName": "authorization-test-user", + "lastName": "authorization-test-user", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "abc" + } + ] + } + ], + "components": { + "org.keycloak.keys.KeyProvider": [ + { + "name": "rsa-256-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "keySize": [ + "2048" + ], + "active": [ + "true" + ], + "priority": [ + "100" + ], + "enabled": [ + "true" + ], + "algorithm": [ + "RS256" + ] + } + }, + { + "name": "rsa-512-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "keySize": [ + "2048" + ], + "active": [ + "true" + ], + "priority": [ + "0" + ], + "enabled": [ + "true" + ], + "algorithm": [ + "RS512" + ] + } + } + ] + } +} From 2dbdbbc76209d79247a1c6a929a0989600789f47 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Wed, 26 Feb 2025 14:38:09 +0100 Subject: [PATCH 30/37] fix(model-client): expired access token wasn't refreshed --- .../modelix/model/oauth/ModelixAuthClient.kt | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt b/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt index 2d36a71e7f..897db3f16f 100644 --- a/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt +++ b/model-client/src/jvmMain/kotlin/org/modelix/model/oauth/ModelixAuthClient.kt @@ -4,7 +4,6 @@ import com.google.api.client.auth.oauth2.AuthorizationCodeFlow import com.google.api.client.auth.oauth2.BearerToken import com.google.api.client.auth.oauth2.ClientParametersAuthentication import com.google.api.client.auth.oauth2.Credential -import com.google.api.client.auth.oauth2.StoredCredential import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver import com.google.api.client.http.GenericUrl @@ -30,9 +29,20 @@ actual object ModelixAuthClient { private var DATA_STORE_FACTORY: DataStoreFactory = MemoryDataStoreFactory() private val HTTP_TRANSPORT: HttpTransport = NetHttpTransport() private val JSON_FACTORY: JsonFactory = GsonFactory() + private val userId = "modelix-user" + private var lastCredentials: Credential? = null - fun getTokens(): StoredCredential? { - return StoredCredential.getDefaultDataStore(DATA_STORE_FACTORY).get("user") + fun getTokens(): Credential? { + return lastCredentials?.refreshIfExpired()?.takeIf { !it.isExpired() } + } + + private fun Credential.isExpired() = (expiresInSeconds ?: 0) < 60 + + private fun Credential.refreshIfExpired(): Credential { + if (isExpired()) { + refreshToken() + } + return this } suspend fun authorize(modelixServerUrl: String): Credential { @@ -67,6 +77,10 @@ actual object ModelixAuthClient { .enablePKCE() .setDataStoreFactory(DATA_STORE_FACTORY) .build() + + val existingTokens = flow.loadCredential(userId)?.refreshIfExpired() + if (existingTokens?.isExpired() == false) return@withContext existingTokens + val receiver: LocalServerReceiver = LocalServerReceiver.Builder().setHost("127.0.0.1").build() val browser = authRequestBrowser?.let { object : AuthorizationCodeInstalledApp.Browser { @@ -75,7 +89,11 @@ actual object ModelixAuthClient { } } } ?: AuthorizationCodeInstalledApp.DefaultBrowser() - AuthorizationCodeInstalledApp(flow, receiver, browser).authorize("user") + val tokens = AuthorizationCodeInstalledApp(flow, receiver, browser).authorize(userId) + if ((tokens.expiresInSeconds ?: 0) < 60) { + tokens.refreshToken() + } + tokens } } @@ -136,8 +154,6 @@ actual object ModelixAuthClient { authorize(url) } - println("Access token: ${tokens.accessToken}") - println("Refresh token: ${tokens.refreshToken}") BearerTokens(tokens.accessToken, tokens.refreshToken) } } From cd457fde2f1d9851b845de90e2fe9a8b16de35c6 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 27 Feb 2025 10:08:29 +0100 Subject: [PATCH 31/37] fix(model-client): only use OAuth if explicitly enabled Otherwise, the client is blocked forever if there is no user doing the login. --- .../org/modelix/model/client2/ModelClientV2.kt | 13 ++++++++++--- .../modelix/mps/sync3/AppLevelModelSyncService.kt | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt index db3d5a2054..05434c3622 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/ModelClientV2.kt @@ -493,6 +493,7 @@ abstract class ModelClientV2Builder { protected var baseUrl: String = "https://localhost/model/v2" protected var authTokenProvider: (suspend () -> String?)? = null protected var authRequestBrowser: ((url: String) -> Unit)? = null + protected var oauthEnabled = false protected var userId: String? = null protected var connectTimeout: Duration = 1.seconds protected var requestTimeout: Duration = 30.seconds @@ -523,9 +524,13 @@ abstract class ModelClientV2Builder { return this } - fun authRequestBrowser(browser: ((url: String) -> Unit)?): ModelClientV2Builder { + fun authRequestBrowser(browser: ((url: String) -> Unit)?) = also { authRequestBrowser = browser - return this + enableOAuth() + } + + fun enableOAuth() = also { + oauthEnabled = true } fun userId(userId: String?): ModelClientV2Builder { @@ -580,7 +585,9 @@ abstract class ModelClientV2Builder { } } } - ModelixAuthClient.installAuth(this, baseUrl, authTokenProvider, authRequestBrowser) + if (authTokenProvider != null || oauthEnabled) { + ModelixAuthClient.installAuth(this, baseUrl, authTokenProvider, authRequestBrowser) + } } } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt index 860c7edb42..840c6779a7 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/AppLevelModelSyncService.kt @@ -50,7 +50,7 @@ class AppLevelModelSyncService() : Disposable { suspend fun getClient(): IModelClientV2 { return client.getValue() ?: client.updateValue { - it ?: ModelClientV2.Companion.builder().url(url).build().also { it.init() } + it ?: ModelClientV2.builder().url(url).enableOAuth().build().also { it.init() } } } From a252e6bc2e5a670b3a096d3e0736281954d0b68f Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 27 Feb 2025 10:59:26 +0100 Subject: [PATCH 32/37] test(model-client): increase timeout for starting containers --- .../kotlin/org/modelix/model/client2/OAuthTest.kt | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt b/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt index c94ecc371d..5a46686496 100644 --- a/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt +++ b/model-client/src/jvmTest/kotlin/org/modelix/model/client2/OAuthTest.kt @@ -16,9 +16,9 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout import org.modelix.model.api.ITree import org.modelix.model.lazy.RepositoryId -import org.testcontainers.containers.ComposeContainer import org.testcontainers.containers.GenericContainer import org.testcontainers.containers.Network +import org.testcontainers.containers.wait.strategy.Wait import org.testcontainers.images.builder.ImageFromDockerfile import org.testcontainers.utility.MountableFile import java.nio.file.Path @@ -27,6 +27,7 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.time.Duration.Companion.minutes import kotlin.time.ExperimentalTime +import kotlin.time.toJavaDuration private val modelServerDir = Path.of("../model-server").absolute().normalize() private val modelServerImage = ImageFromDockerfile() @@ -80,7 +81,7 @@ class OAuthTest { private fun runWithModelServer(body: suspend (url: String) -> Unit) = runBlocking { @OptIn(ExperimentalTime::class) - withTimeout(3.minutes) { + withTimeout(5.minutes) { val network = Network.newNetwork() val keycloak: GenericContainer<*> = GenericContainer("quay.io/keycloak/keycloak:${System.getenv("KEYCLOAK_VERSION")}") @@ -93,6 +94,7 @@ class OAuthTest { .withCopyFileToContainer(MountableFile.forHostPath("../model-server-with-auth/realm.json"), "/opt/keycloak/data/import/realm.json") .withNetwork(network) .withNetworkAliases("keycloak") + .waitingFor(Wait.forListeningPort().withStartupTimeout(3.minutes.toJavaDuration())) .withLogConsumer { println("[KEYCLOAK] " + it.utf8StringWithoutLineEnding) } keycloak.start() val keycloakPort = keycloak.getMappedPort(8080) @@ -106,6 +108,7 @@ class OAuthTest { .withEnv("MODELIX_JWK_URI_KEYCLOAK", "http://keycloak:8080/realms/modelix/protocol/openid-connect/certs") .withNetwork(network) .withNetworkAliases("model-server") + .waitingFor(Wait.forListeningPort().withStartupTimeout(3.minutes.toJavaDuration())) .withLogConsumer { println("[MODEL] " + it.utf8StringWithoutLineEnding) } modelServer.start() @@ -118,9 +121,3 @@ class OAuthTest { } } } - -private fun ComposeContainer.getServiceUrl(name: String, port: Int): String { - val h = getServiceHost(name, port) - val p = getServicePort(name, port) - return "http://$h:$p/" -} From e5188ab26b5087f4aaf3e7e13a1e241d02dc4f13 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 27 Feb 2025 13:24:31 +0100 Subject: [PATCH 33/37] fix(model-client): OAuth login --- .../model/sync/bulk/IncludedModulesFilter.kt | 1 + model-server-with-auth/compose.yaml | 12 ++ model-server-with-auth/realm.json | 2 +- .../model/mpsadapters/MPSProjectAsNode.kt | 14 +- mps-sync-plugin3/compose.yaml | 66 ------- mps-sync-plugin3/realm.json | 177 ------------------ 6 files changed, 27 insertions(+), 245 deletions(-) delete mode 100644 mps-sync-plugin3/compose.yaml delete mode 100644 mps-sync-plugin3/realm.json diff --git a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt index 65a228c5db..00778fb3d2 100644 --- a/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt +++ b/bulk-model-sync-mps/src/main/kotlin/org/modelix/mps/model/sync/bulk/IncludedModulesFilter.kt @@ -14,6 +14,7 @@ import org.modelix.model.sync.bulk.isModuleIncluded * Note: This is currently not meant to be used standalone. * It should be used with other filters in a [CompositeFilter]. */ +@Deprecated("Use IModelMask") class IncludedModulesFilter( val includedModules: Collection, val includedModulePrefixes: Collection, diff --git a/model-server-with-auth/compose.yaml b/model-server-with-auth/compose.yaml index 753fab1740..2349af4c3d 100644 --- a/model-server-with-auth/compose.yaml +++ b/model-server-with-auth/compose.yaml @@ -14,6 +14,11 @@ services: networks: - app-network + redis: + image: redis:latest + networks: + - app-network + oauth2-proxy: image: quay.io/oauth2-proxy/oauth2-proxy:latest environment: @@ -35,12 +40,19 @@ services: OAUTH2_PROXY_COOKIE_SECURE: "false" OAUTH2_PROXY_SET_AUTHORIZATION_HEADER: "true" OAUTH2_PROXY_PASS_ACCESS_TOKEN: "true" + OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER: "true" OAUTH2_PROXY_HTTP_ADDRESS: "0.0.0.0:4180" OAUTH2_PROXY_SCOPE: "sync openid" + OAUTH2_PROXY_SESSION_STORE_TYPE: redis + OAUTH2_PROXY_REDIS_CONNECTION_URL: redis://redis/ + OAUTH2_PROXY_COOKIE_REFRESH: 60s + OAUTH2_PROXY_API_ROUTES: \/v2\/.* ports: - "4180:4180" depends_on: - keycloak + - model-server + - redis networks: - app-network diff --git a/model-server-with-auth/realm.json b/model-server-with-auth/realm.json index ef498144a1..b5f0be83e1 100644 --- a/model-server-with-auth/realm.json +++ b/model-server-with-auth/realm.json @@ -114,7 +114,7 @@ "users": [ { "username": "user1", - "email": "authorization-test-user@authorization-test-user.com", + "email": "user1@example.com", "firstName": "authorization-test-user", "lastName": "authorization-test-user", "enabled": true, diff --git a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt index 06fcad565f..3336a8ef69 100644 --- a/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt +++ b/mps-model-adapters/src/main/kotlin/org/modelix/model/mpsadapters/MPSProjectAsNode.kt @@ -1,7 +1,9 @@ package org.modelix.model.mpsadapters +import jetbrains.mps.project.ModuleId import jetbrains.mps.project.ProjectBase import org.jetbrains.mps.openapi.module.SRepository +import org.jetbrains.mps.openapi.persistence.PersistenceFacade import org.modelix.model.api.BuiltinLanguages import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.IConcept @@ -30,7 +32,17 @@ data class MPSProjectAsNode(val project: ProjectBase) : MPSGenericNodeAdapter Date: Thu, 27 Feb 2025 17:15:08 +0100 Subject: [PATCH 34/37] test(model-server): use keycloak version from gradle dependencies --- model-server/build.gradle.kts | 3 ++- .../test/kotlin/org/modelix/model/server/AuthorizationTest.kt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/model-server/build.gradle.kts b/model-server/build.gradle.kts index d9d2fac67e..ec98b2f7d9 100644 --- a/model-server/build.gradle.kts +++ b/model-server/build.gradle.kts @@ -221,5 +221,6 @@ project.registerVersionGenerationTask("org.modelix.model.server") tasks.test { // Workaround Ignite failing locally because the autogenerated node ID results in an invalid path name. // See https://stackoverflow.com/questions/76387714/apache-ignite-failing-on-startup - setEnvironment("IGNITE_OVERRIDE_CONSISTENT_ID" to "node00") + environment("IGNITE_OVERRIDE_CONSISTENT_ID", "node00") + environment("KEYCLOAK_VERSION", libs.versions.keycloak.get()) } diff --git a/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt index 02e7ba030d..611d719f4b 100644 --- a/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt +++ b/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt @@ -160,7 +160,7 @@ class AuthorizationTest { """ // Reuse on container across all tests. The configuration and state does not change in between. - private val keycloak: GenericContainer<*> = GenericContainer("quay.io/keycloak/keycloak:25.0.4") + private val keycloak: GenericContainer<*> = GenericContainer("quay.io/keycloak/keycloak:${System.getenv("KEYCLOAK_VERSION")}") .withEnv("KEYCLOAK_ADMIN", ADMIN_USER) .withEnv("KEYCLOAK_ADMIN_PASSWORD", ADMIN_PASSWORD) .withExposedPorts(8080) From 9530a0c46e6baf7a0ebce938febe7c1e53fe2f02 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Thu, 27 Feb 2025 17:18:09 +0100 Subject: [PATCH 35/37] test(model-server): increase timeout in AuthorizationTest --- .../test/kotlin/org/modelix/model/server/AuthorizationTest.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt index 611d719f4b..877ceb72c5 100644 --- a/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt +++ b/model-server/src/test/kotlin/org/modelix/model/server/AuthorizationTest.kt @@ -23,10 +23,13 @@ import org.modelix.model.server.handlers.RepositoriesManager import org.modelix.model.server.store.InMemoryStoreClient import org.modelix.model.server.store.StoreManager import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait import org.testcontainers.images.builder.Transferable import java.net.URI import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.time.Duration.Companion.minutes +import kotlin.time.toJavaDuration private const val ADMIN_USER = "admin" private const val ADMIN_PASSWORD = "admin" @@ -165,6 +168,7 @@ class AuthorizationTest { .withEnv("KEYCLOAK_ADMIN_PASSWORD", ADMIN_PASSWORD) .withExposedPorts(8080) .withCopyToContainer(Transferable.of(REALM_CONFIGURATION), "/opt/keycloak/data/import/realm.json") + .waitingFor(Wait.forListeningPort().withStartupTimeout(3.minutes.toJavaDuration())) .withCommand("start-dev", "--import-realm", "--verbose") private var keycloakBaseUrl: String From a8b6fcc988b56e58e75e8208c656f815c2c9ff4e Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Fri, 28 Feb 2025 08:51:48 +0100 Subject: [PATCH 36/37] fix(model-server): use output of gradle application plugin to run the model-server Instead of copying all the jar libraries and having our own shell script we can just use the gradle plugin, that already assembles everything properly. --- model-server/Dockerfile | 11 ++++++----- model-server/build.gradle.kts | 7 +------ model-server/run-model-server.sh | 3 --- 3 files changed, 7 insertions(+), 14 deletions(-) delete mode 100755 model-server/run-model-server.sh diff --git a/model-server/Dockerfile b/model-server/Dockerfile index 6345658e46..e94ae1ac69 100644 --- a/model-server/Dockerfile +++ b/model-server/Dockerfile @@ -1,9 +1,10 @@ FROM registry.access.redhat.com/ubi8/openjdk-11:1.21-1.1736337912 USER root -WORKDIR /usr/modelix-model +WORKDIR /model-server/ EXPOSE 28101 HEALTHCHECK CMD curl --fail http://localhost:28101/health || exit 1 -COPY run-model-server.sh /usr/modelix-model/ -COPY build/dependency-libs/ /usr/modelix-model/model-server/build/libs/ -COPY build/libs/ /usr/modelix-model/model-server/build/libs/ -ENTRYPOINT ["./run-model-server.sh"] + +COPY build/install/model-server/ /model-server/ + +ENV MODEL_SERVER_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5071 -XX:MaxRAMPercentage=75 -Djdbc.url=$jdbc_url" +ENTRYPOINT ["/model-server/bin/model-server"] diff --git a/model-server/build.gradle.kts b/model-server/build.gradle.kts index ec98b2f7d9..e62fe136a7 100644 --- a/model-server/build.gradle.kts +++ b/model-server/build.gradle.kts @@ -113,13 +113,8 @@ val fatJarArtifact = artifacts.add("archives", fatJarFile) { builtBy("shadowJar") } -task("copyLibs", Sync::class) { - into(project.layout.buildDirectory.dir("dependency-libs")) - from(configurations.runtimeClasspath) -} - tasks.named("assemble") { - finalizedBy("copyLibs") + dependsOn("installDist") } application { diff --git a/model-server/run-model-server.sh b/model-server/run-model-server.sh deleted file mode 100755 index eeff06d4ce..0000000000 --- a/model-server/run-model-server.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -exec java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5071 -XX:MaxRAMPercentage=75 -Djdbc.url=$jdbc_url -cp "model-server/build/libs/*" org.modelix.model.server.Main "$@" From 1187b88e56d875cc5ea83887a713762e54e5a9ed Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Fri, 28 Feb 2025 13:22:03 +0100 Subject: [PATCH 37/37] fix(mps-sync-plugin): removed last usages of originalId in ModelSynchronizer --- .../model/sync/bulk/ModelSynchronizer.kt | 152 ++++++++---------- .../org/modelix/mps/sync3/ProjectSnapshot.kt | 3 + 2 files changed, 72 insertions(+), 83 deletions(-) diff --git a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt index 7c67eb6af6..4b776b4323 100644 --- a/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt +++ b/bulk-model-sync-lib/src/commonMain/kotlin/org/modelix/model/sync/bulk/ModelSynchronizer.kt @@ -8,8 +8,6 @@ import org.modelix.model.api.IReferenceLinkReference import org.modelix.model.api.IRoleReference import org.modelix.model.api.IWritableNode import org.modelix.model.api.NewNodeSpec -import org.modelix.model.api.PNodeAdapter -import org.modelix.model.api.getOriginalOrCurrentReference import org.modelix.model.api.getOriginalReference import org.modelix.model.api.isOrdered import org.modelix.model.api.matches @@ -171,16 +169,10 @@ class ModelSynchronizer( ) { val sourceNodes = getFilteredSourceChildren(sourceParent, role) val targetNodes = getFilteredTargetChildren(targetParent, role) - val unusedTargetChildren = targetNodes.toMutableSet() - - val allExpectedNodesDoNotExist by lazy { - sourceNodes.all { sourceNode -> - nodeAssociation.resolveTarget(sourceNode) == null - } - } + val associatedChildren = associateChildren(sourceNodes, targetNodes) // optimization that uses the bulk operation .syncNewChildren - if (targetNodes.isEmpty() && allExpectedNodesDoNotExist) { + if (associatedChildren.all { it.hasToCreate() }) { targetParent.syncNewChildren(role, -1, sourceNodes.map { NewNodeSpec(it) }) .zip(sourceNodes) .forEach { (newChild, sourceChild) -> @@ -190,74 +182,49 @@ class ModelSynchronizer( return } + val isOrdered = targetParent.isOrdered(role) + // optimization for when there is no change in the child list - // size check first to avoid querying the original ID - if (sourceNodes.size == targetNodes.size && sourceNodes.zip(targetNodes) - .all { nodeAssociation.matches(it.first, it.second) } - ) { - sourceNodes.zip(targetNodes).forEach { synchronizeNode(it.first, it.second, forceSyncDescendants) } + if (associatedChildren.all { it.alreadyMatches(isOrdered) }) { + associatedChildren.forEach { + synchronizeNode(it.getSource(), it.getTarget(), forceSyncDescendants) + } return } - val isOrdered = targetParent.isOrdered(role) - - sourceNodes.forEachIndexed { indexInImport, expected -> - val existingChildren = getFilteredTargetChildren(targetParent, role) - val expectedId = checkNotNull(expected.originalIdOrFallback()) { "Specified node '$expected' has no id" } - // newIndex is the index on which to import the expected child. - // It might be -1 if the child does not exist and should be added at the end. - val newIndex = if (isOrdered) { - indexInImport + val unusedTargetChildren: List = associatedChildren + .asSequence() + .filter { it.hasToDelete() } + .map { it.getTarget() } + .toList() + + nodesToRemove += unusedTargetChildren + + val recursiveSyncTasks = ArrayList() + + for (associatedChild in associatedChildren) { + if (associatedChild.hasToCreate()) { + val newChild = targetParent.syncNewChild(role, associatedChild.sourceIndex, NewNodeSpec(associatedChild.getSource())) + nodeAssociation.associate(associatedChild.getSource(), newChild) + recursiveSyncTasks += RecursiveSyncTask(associatedChild.getSource(), newChild, true) + } else if (associatedChild.hasToMove(isOrdered)) { + targetParent.moveChild(role, associatedChild.sourceIndex, associatedChild.getTarget()) + nodesToRemove.remove(associatedChild.getTarget()) + recursiveSyncTasks += RecursiveSyncTask(associatedChild.getSource(), associatedChild.getTarget(), false) + } else if (associatedChild.hasToDelete()) { + // no sync between child nodes needed } else { - // The `existingChildren` are only searched once for the expected element before changing. - // Therefore, indexing existing children will not be more efficient than iterating once. - // (For the moment, this is fine because as we expect unordered children to be the exception, - // Reusable indexing would be possible if we switch from - // a depth-first import to a breadth-first import.) - existingChildren - .indexOfFirst { existingChild -> existingChild.getOriginalOrCurrentReference() == expectedId } + recursiveSyncTasks += RecursiveSyncTask(associatedChild.getSource(), associatedChild.getTarget(), false) } - // existingChildren.getOrNull handles `-1` as needed by returning `null`. - val nodeAtIndex = existingChildren.getOrNull(newIndex) - val expectedConcept = expected.getConceptReference() - var isNewChild = false - val childNode = if (nodeAtIndex?.getOriginalOrCurrentReference() != expectedId) { - val existingNode = nodeAssociation.resolveTarget(expected) - if (existingNode == null) { - val newChild = targetParent.syncNewChild(role, newIndex, NewNodeSpec(expected)) - nodeAssociation.associate(expected, newChild) - isNewChild = true - newChild - } else { - // The existing child node is not only moved to a new index, - // it is potentially moved to a new parent and role. - if (existingNode.getParent() != targetParent || - !existingNode.getContainmentLink().matches(role) || - targetParent.isOrdered(role) - ) { - targetParent.moveChild(role, newIndex, existingNode) - } - // If the old parent and old role synchronized before the move operation, - // the existing child node would have been marked as to be deleted. - // Now that it is used, it should not be deleted. - unusedTargetChildren.remove(existingNode) - nodesToRemove.remove(existingNode) - existingNode - } - } else { - unusedTargetChildren.remove(nodeAtIndex) - nodesToRemove.remove(nodeAtIndex) - nodeAtIndex - } - - synchronizeNode(expected, childNode, forceSyncDescendants || isNewChild) } - // Do not use existingNodes, but call node.getChildren(role) because - // the recursive synchronization in the meantime already removed some nodes from node.getChildren(role). - nodesToRemove += getFilteredTargetChildren(targetParent, role).intersect(unusedTargetChildren) + for (task in recursiveSyncTasks) { + synchronizeNode(task.source, task.target, forceSyncDescendants || task.isNew) + } } + private class RecursiveSyncTask(val source: IReadableNode, val target: IWritableNode, val isNew: Boolean) + /** * In MPS and also in Modelix nodes internally are stored in a single list that is filtered when a specific role is * accessed. The information about this internal order is visible when using getAllChildren(). @@ -281,6 +248,40 @@ class ModelSynchronizer( } } + private fun associateChildren(sourceChildren: List, targetChildren: List): List { + val unassociatedTargetNodes = targetChildren.withIndex().toMutableList() + return sourceChildren.mapIndexed { sourceIndex, sourceChild -> + val foundAt = unassociatedTargetNodes.indexOfFirst { targetChild -> + nodeAssociation.matches(sourceChild, targetChild.value) + } + if (foundAt == -1) { + AssociatedChild(sourceIndex, -1, sourceChild, null, nodeAssociation.resolveTarget(sourceChild)) + } else { + val foundTarget = unassociatedTargetNodes.removeAt(foundAt) + AssociatedChild(sourceIndex, foundTarget.index, sourceChild, foundTarget.value, null) + } + } + unassociatedTargetNodes.map { AssociatedChild(-1, it.index, null, it.value, null) } + } + + private class AssociatedChild( + val sourceIndex: Int, + private val targetIndex: Int, + private val source: IReadableNode?, + private val existingTarget: IWritableNode?, + private val resolvedTarget: IWritableNode?, + ) { + fun hasToCreate() = existingTarget == null && resolvedTarget == null + fun hasToMoveFromDifferentContainment() = source != null && resolvedTarget != null + fun hasToMoveWithinSameContainment() = source != null && existingTarget != null + fun hasToMove(ordered: Boolean) = hasToMoveFromDifferentContainment() || ordered && hasToMoveWithinSameContainment() + fun hasToDelete() = source == null + fun alreadyMatchesOrdered() = source != null && existingTarget != null && sourceIndex == targetIndex + fun alreadyMatchesUnordered() = source != null && existingTarget != null + fun alreadyMatches(ordered: Boolean) = if (ordered) alreadyMatchesOrdered() else alreadyMatchesUnordered() + fun getTarget() = existingTarget ?: resolvedTarget!! + fun getSource() = source!! + } + inner class PendingReference(val sourceNode: IReadableNode, val targetNode: IWritableNode, val role: IReferenceLinkReference) { override fun toString(): String = "${sourceNode.getNodeReference()}[$role] : ${targetNode.getAllReferenceTargetRefs()}" @@ -371,18 +372,3 @@ class AndFilter(val filter1: IIncrementalUpdateInformation, val filter2: IIncrem return filter1.needsSynchronization(node) && filter2.needsSynchronization(node) } } - -private fun INode.originalIdOrFallback(): String? { - val originalRef = getOriginalReference() - if (originalRef != null) return originalRef - - if (this is PNodeAdapter) return reference.serialize() - return null -} - -private fun IReadableNode.originalIdOrFallback(): String? { - val originalRef = getOriginalReference() - if (originalRef != null) return originalRef - - return getNodeReference().serialize() -} diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt index 92a0350346..066246bfe9 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSnapshot.kt @@ -100,6 +100,9 @@ private fun normalizeXmlFile(content: String): String { "dev-kit" -> { node.childElements("exported-language").sortByAttribute("name") } + "dependencies" -> { + node.childElements("dependency").sortBy { it.textContent } + } "sourceRoot" -> { val location = node.getAttribute("location") val path = node.getAttribute("path")