feat(mps-sync-plugin): MPS sync plugin inital commit

commitlint.config.js

@@ -8,7 +8,9 @@ module.exports = {
         "deps",
         "mps-legacy-sync-plugin",
         "mps-diff-plugin",
-        "generator"
+        "generator",
+        "mps-sync-plugin",
+        "mps-sync-plugin-lib",
       ],
     ],
     "subject-case": [0, 'never'],

gradle/libs.versions.toml

@@ -6,6 +6,7 @@ npm-publish = { id = "dev.petuska.npm.publish", version = "3.4.2" }
 
 [versions]
 modelixCore = "4.11.5"
+kotlinCoroutines="1.7.3"
 
 [libraries]
 modelix-model-api = { group = "org.modelix", name = "model-api", version.ref = "modelixCore" }
@@ -19,6 +20,11 @@ modelix-modelql-untyped = { group = "org.modelix", name = "modelql-untyped", ver
 modelix-modelql-html = { group = "org.modelix", name = "modelql-html", version.ref = "modelixCore" }
 modelix-model-datastructure = { group = "org.modelix", name = "model-datastructure", version.ref = "modelixCore" }
 modelix-mps-model-adapters = { group = "org.modelix.mps", name = "model-adapters", version.ref = "modelixCore" }
+
 kotlin-logging = { group = "io.github.oshai", name = "kotlin-logging", version = "6.0.3" }
+kotlin-logging-microutils = { group = "io.github.microutils", name = "kotlin-logging", version = "3.0.5" }
 kotlin-html = "org.jetbrains.kotlinx:kotlinx-html:0.11.0"
+kotlin-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinCoroutines" }
+
 zt-zip = { group = "org.zeroturnaround", name = "zt-zip", version = "1.17" }
+logback-classic = { group = "ch.qos.logback", name = "logback-classic", version = "1.4.14" }

mps-sync-plugin-lib/build.gradle.kts

@@ -0,0 +1,48 @@
+plugins {
+    kotlin("jvm")
+    kotlin("plugin.serialization")
+}
+
+repositories {
+    maven { url = uri("https://www.jetbrains.com/intellij-repository/releases") }
+}
+
+kotlin {
+    compilerOptions {
+        apiVersion.set(org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_1_6)
+    }
+}
+
+// use the given MPS version, or 2022.2 (last version with JAVA 11) as default
+val mpsVersion = project.findProperty("mps.version")?.toString().takeIf { !it.isNullOrBlank() } ?: "2020.3.6"
+
+val mpsZip by configurations.creating
+
+dependencies {
+    implementation(kotlin("stdlib-jdk8"))
+    implementation(libs.kotlin.logging.microutils)
+
+    implementation(libs.modelix.model.api)
+    implementation(libs.modelix.model.client)
+    implementation(libs.modelix.mps.model.adapters)
+
+    // extracting jars from zipped products
+    mpsZip("com.jetbrains:mps:$mpsVersion")
+    compileOnly(
+        zipTree({ mpsZip.singleFile }).matching {
+            include("lib/**/*.jar")
+        },
+    )
+}
+
+group = "org.modelix.mps"
+description = "Generic helper library to sync model-server content with MPS"
+
+publishing {
+    publications {
+        create<MavenPublication>("maven") {
+            artifactId = "sync-plugin-lib"
+            from(components["kotlin"])
+        }
+    }
+}

mps-sync-plugin-lib/src/main/kotlin/org/modelix/mps/sync/SyncService.kt

@@ -0,0 +1,38 @@
+package org.modelix.mps.sync
+
+import com.intellij.openapi.project.Project
+import org.modelix.kotlin.utils.UnstableModelixFeature
+import org.modelix.model.api.INode
+import org.modelix.model.client2.ModelClientV2
+import org.modelix.model.lazy.BranchReference
+import java.net.URL
+import java.util.concurrent.CompletableFuture
+
+@UnstableModelixFeature(reason = "The new modelix MPS plugin is under construction", intendedFinalization = "2024.1")
+interface SyncService {
+
+    suspend fun bindModule(
+        client: ModelClientV2,
+        branchReference: BranchReference,
+        module: INode,
+        callback: (() -> Unit)? = null,
+    ): Iterable<IBinding>
+
+    suspend fun connectModelServer(serverURL: URL, jwt: String, callback: (() -> Unit)? = null): ModelClientV2
+
+    fun disconnectModelServer(client: ModelClientV2, callback: (() -> Unit)? = null)
+
+    fun setActiveProject(project: Project)
+
+    fun dispose()
+}
+
+@UnstableModelixFeature(reason = "The new modelix MPS plugin is under construction", intendedFinalization = "2024.1")
+interface IBinding {
+
+    fun activate(callback: Runnable? = null)
+
+    fun deactivate(removeFromServer: Boolean, callback: Runnable? = null): CompletableFuture<Any?>
+
+    fun name(): String
+}

mps-sync-plugin-lib/src/main/kotlin/org/modelix/mps/sync/SyncServiceImpl.kt

@@ -0,0 +1,181 @@
+package org.modelix.mps.sync
+
+import com.intellij.openapi.project.Project
+import jetbrains.mps.project.MPSProject
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.cancel
+import mu.KotlinLogging
+import org.modelix.kotlin.utils.UnstableModelixFeature
+import org.modelix.model.api.IBranchListener
+import org.modelix.model.api.ILanguageRepository
+import org.modelix.model.api.INode
+import org.modelix.model.client2.ModelClientV2
+import org.modelix.model.client2.ReplicatedModel
+import org.modelix.model.client2.getReplicatedModel
+import org.modelix.model.lazy.BranchReference
+import org.modelix.model.mpsadapters.MPSLanguageRepository
+import org.modelix.mps.sync.bindings.BindingsRegistry
+import org.modelix.mps.sync.modelix.ModelixBranchListener
+import org.modelix.mps.sync.modelix.ReplicatedModelRegistry
+import org.modelix.mps.sync.mps.ActiveMpsProjectInjector
+import org.modelix.mps.sync.mps.RepositoryChangeListener
+import org.modelix.mps.sync.tasks.FuturesWaitQueue
+import org.modelix.mps.sync.tasks.SyncQueue
+import org.modelix.mps.sync.transformation.modelixToMps.initial.ITreeToSTreeTransformer
+import java.net.ConnectException
+import java.net.URL
+
+@UnstableModelixFeature(reason = "The new modelix MPS plugin is under construction", intendedFinalization = "2024.1")
+class SyncServiceImpl : SyncService {
+
+    private val logger = KotlinLogging.logger {}
+    private val mpsProjectInjector = ActiveMpsProjectInjector
+
+    private val coroutineScope = CoroutineScope(Dispatchers.Default)
+    val activeClients = mutableSetOf<ModelClientV2>()
+    private val replicatedModelByBranchReference = mutableMapOf<BranchReference, ReplicatedModel>()
+    private val changeListenerByReplicatedModel = mutableMapOf<ReplicatedModel, IBranchListener>()
+
+    private var projectWithChangeListener: Pair<MPSProject, RepositoryChangeListener>? = null
+
+    init {
+        logger.info { "============================================ Registering builtin languages" }
+        // just a dummy call, the initializer of ILanguageRegistry takes care of the rest...
+        ILanguageRepository.default.javaClass
+    }
+
+    // todo add afterActivate to allow async refresh
+    override suspend fun connectModelServer(
+        serverURL: URL,
+        jwt: String,
+        callback: (() -> Unit)?,
+    ): ModelClientV2 {
+        // avoid reconnect to existing server
+        val client = activeClients.find { it.baseUrl == serverURL.toString() }
+        client?.let {
+            logger.info { "Using already existing connection to $serverURL" }
+            return it
+        }
+
+        // TODO: use JWT here
+        val modelClientV2: ModelClientV2 = ModelClientV2.builder().url(serverURL.toString()).build()
+        try {
+            logger.info { "Connecting to $serverURL" }
+            modelClientV2.init()
+        } catch (e: ConnectException) {
+            logger.warn { "Unable to connect: ${e.message} / ${e.cause}" }
+            throw e
+        }
+
+        logger.info { "Connection to $serverURL successful" }
+        activeClients.add(modelClientV2)
+
+        callback?.invoke()
+
+        return modelClientV2
+    }
+
+    override fun disconnectModelServer(
+        client: ModelClientV2,
+        callback: (() -> Unit)?,
+    ) {
+        // TODO what shall happen with the bindings if we disconnect from model server?
+        activeClients.remove(client)
+        client.close()
+        callback?.invoke()
+    }
+
+    override suspend fun bindModule(
+        client: ModelClientV2,
+        branchReference: BranchReference,
+        module: INode,
+        callback: (() -> Unit)?,
+    ): Iterable<IBinding> {
+        // fetch replicated model and branch content
+        // TODO how to handle multiple replicated models at the same time?
+        val replicatedModel =
+            replicatedModelByBranchReference.getOrDefault(branchReference, client.getReplicatedModel(branchReference))
+        val replicateModelIsAlreadySynched = replicatedModelByBranchReference.containsKey(branchReference)
+
+        /*
+         * TODO fixme:
+         * (1) How to propagate replicated model to other places of code?
+         * (2) How to know to which replicated model we want to upload? (E.g. when connecting to multiple model servers?)
+         * (3) How to replace the outdated replicated models that are already used from the registry?
+         *
+         * Possible answers:
+         * (1) via the registry
+         * (2) Base the selection on the parent project and the active model server connections we have. E.g. let the user select to which model server they want to upload the changes and so they get the corresponding replicated model.
+         * (3) We don't. We have to make sure that the places always have the latest replicated models from the registry. E.g. if we disconnect from the model server then we remove the replicated model (and thus break the registered event handlers), otherwise the event handlers as for the replicated model from the registry (based on some identifying metainfo for example, so to know which replicated model they need).
+         */
+        ReplicatedModelRegistry.model = replicatedModel
+        replicatedModelByBranchReference[branchReference] = replicatedModel
+
+        // TODO when and how to dispose the replicated model and everything that depends on it?
+        val branch = if (replicateModelIsAlreadySynched) {
+            replicatedModel.getBranch()
+        } else {
+            replicatedModel.start()
+        }
+
+        // transform the model
+        val targetProject = mpsProjectInjector.activeMpsProject!!
+        val languageRepository = registerLanguages(targetProject)
+        val bindings = ITreeToSTreeTransformer(branch, languageRepository).transform(module)
+
+        // register replicated model change listener
+        if (!replicateModelIsAlreadySynched) {
+            val listener = ModelixBranchListener(replicatedModel, languageRepository, branch)
+            branch.addListener(listener)
+            changeListenerByReplicatedModel[replicatedModel] = listener
+        }
+
+        // register MPS project change listener
+        if (projectWithChangeListener == null) {
+            val repositoryChangeListener = RepositoryChangeListener(branch)
+            targetProject.repository.addRepositoryListener(repositoryChangeListener)
+            projectWithChangeListener = Pair(targetProject, repositoryChangeListener)
+        }
+
+        // trigger callback after activation
+        callback?.invoke()
+
+        return bindings
+    }
+
+    override fun setActiveProject(project: Project) {
+        mpsProjectInjector.setActiveProject(project)
+    }
+
+    override fun dispose() {
+        // cancel all running coroutines
+        coroutineScope.cancel()
+        SyncQueue.close()
+        FuturesWaitQueue.close()
+        // unregister change listeners
+        resetProjectWithChangeListener()
+        changeListenerByReplicatedModel.forEach { it.key.getBranch().removeListener(it.value) }
+        // dispose the clients
+        activeClients.forEach { it.close() }
+        // dispose all bindings
+        BindingsRegistry.getModuleBindings().forEach { it.deactivate(removeFromServer = false) }
+        BindingsRegistry.getModelBindings().forEach { it.deactivate(removeFromServer = false) }
+    }
+
+    private fun registerLanguages(project: MPSProject): MPSLanguageRepository {
+        val repository = project.repository
+        val mpsLanguageRepo = MPSLanguageRepository(repository)
+        ILanguageRepository.register(mpsLanguageRepo)
+        return mpsLanguageRepo
+    }
+
+    private fun resetProjectWithChangeListener() {
+        projectWithChangeListener?.let {
+            val project = it.first
+            val listener = it.second
+            project.repository.removeRepositoryListener(listener)
+            projectWithChangeListener = null
+        }
+    }
+}

mps-sync-plugin-lib/src/main/kotlin/org/modelix/mps/sync/bindings/BindingSortComparator.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2024.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.modelix.mps.sync.bindings
+
+import org.modelix.kotlin.utils.UnstableModelixFeature
+import org.modelix.mps.sync.IBinding
+
+@UnstableModelixFeature(reason = "The new modelix MPS plugin is under construction", intendedFinalization = "2024.1")
+class BindingSortComparator : Comparator<IBinding> {
+    /**
+     * ModelBindings should come first, then ModuleBindings. If both bindings have the same type, then they are sorted lexicographically.
+     */
+    override fun compare(left: IBinding, right: IBinding): Int {
+        val leftName = left.name()
+        val rightName = right.name()
+
+        if (left is ModelBinding) {
+            if (right is ModelBinding) {
+                return leftName.compareTo(rightName)
+            } else if (right is ModuleBinding) {
+                return -1
+            }
+        } else if (left is ModuleBinding) {
+            if (right is ModelBinding) {
+                return 1
+            } else if (right is ModuleBinding) {
+                return leftName.compareTo(rightName)
+            }
+        }
+        return 0
+    }
+}