diff --git a/core/src/main/kotlin/org/utbot/jcdb/api/Api.kt b/core/src/main/kotlin/org/utbot/jcdb/api/Api.kt index fbadc3980..08a5b8e8d 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/api/Api.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/api/Api.kt @@ -311,6 +311,8 @@ enum class FieldUsageMode { */ interface CompilationDatabase : Closeable { + val globalIdStore: GlobalIdsStore + /** * create classpath instance * @@ -361,4 +363,12 @@ interface CompilationDatabase : Closeable { * await background jobs */ suspend fun awaitBackgroundJobs() -} \ No newline at end of file +} + +/** + * database store for storing indexes mappings between ids -> name + */ +interface GlobalIdsStore { + suspend fun getId(name: String): Int + suspend fun getName(id: Int): String? +} diff --git a/core/src/main/kotlin/org/utbot/jcdb/api/Index.kt b/core/src/main/kotlin/org/utbot/jcdb/api/Index.kt index 15671cabe..b574eb5d8 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/api/Index.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/api/Index.kt @@ -10,9 +10,9 @@ import java.io.OutputStream */ interface ByteCodeLocationIndexBuilder> { - fun index(classNode: ClassNode) + suspend fun index(classNode: ClassNode) - fun index(classNode: ClassNode, methodNode: MethodNode) + suspend fun index(classNode: ClassNode, methodNode: MethodNode) fun build(location: ByteCodeLocation): INDEX @@ -22,17 +22,17 @@ interface ByteCodeLocationIndex { val location: ByteCodeLocation - fun query(term: String): Sequence + suspend fun query(term: String): Sequence } interface Feature> { val key: String - fun newBuilder() : ByteCodeLocationIndexBuilder + fun newBuilder(globalIdsStore: GlobalIdsStore) : ByteCodeLocationIndexBuilder fun serialize(index: INDEX, out: OutputStream) - fun deserialize(location: ByteCodeLocation, stream: InputStream): INDEX? + fun deserialize(globalIdsStore: GlobalIdsStore, location: ByteCodeLocation, stream: InputStream): INDEX? } diff --git a/core/src/main/kotlin/org/utbot/jcdb/impl/CompilationDatabaseImpl.kt b/core/src/main/kotlin/org/utbot/jcdb/impl/CompilationDatabaseImpl.kt index 354e6f7f7..bd146a5eb 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/impl/CompilationDatabaseImpl.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/impl/CompilationDatabaseImpl.kt @@ -5,6 +5,7 @@ import mu.KLogging import org.utbot.jcdb.CompilationDatabaseSettings import org.utbot.jcdb.api.* import org.utbot.jcdb.impl.fs.* +import org.utbot.jcdb.impl.index.GlobalIds import org.utbot.jcdb.impl.storage.PersistentEnvironment import org.utbot.jcdb.impl.storage.scheme.LocationEntity import org.utbot.jcdb.impl.tree.ClassTree @@ -18,7 +19,8 @@ import java.util.concurrent.atomic.AtomicInteger class CompilationDatabaseImpl( private val persistentEnvironment: PersistentEnvironment? = null, - private val settings: CompilationDatabaseSettings + private val settings: CompilationDatabaseSettings, + override val globalIdStore: GlobalIds = GlobalIds() ) : CompilationDatabase { companion object : KLogging() @@ -27,7 +29,7 @@ class CompilationDatabaseImpl( internal val javaRuntime = JavaRuntime(settings.jre) private val hooks = settings.hooks.map { it(this) } - internal val featureRegistry = FeaturesRegistry(persistentEnvironment, settings.fullFeatures) + internal val featureRegistry = FeaturesRegistry(persistentEnvironment, globalIdStore, settings.fullFeatures) internal val locationsRegistry = LocationsRegistry(featureRegistry) private val backgroundJobs = ConcurrentHashMap() @@ -122,7 +124,7 @@ class CompilationDatabaseImpl( } } }.joinAll() - persistentEnvironment?.globalIds?.sync() + persistentEnvironment?.globalIds?.sync(globalIdStore) backgroundJobs.remove(backgroundJobId) } } @@ -176,7 +178,7 @@ class CompilationDatabaseImpl( internal suspend fun restoreDataFrom(locations: Map) { val env = persistentEnvironment ?: return - env.globalIds.restore() + env.globalIds.restore(globalIdStore) val trees = withContext(Dispatchers.IO) { locations.map { (entity, location) -> async { @@ -206,7 +208,7 @@ class CompilationDatabaseImpl( val data = entity.index(key) if (data != null) { val index = try { - deserialize(location, data) + deserialize(globalIdStore, location, data) } catch (e: Exception) { logger.warn(e) { "can't parse location" } null diff --git a/core/src/main/kotlin/org/utbot/jcdb/impl/FeaturesRegistry.kt b/core/src/main/kotlin/org/utbot/jcdb/impl/FeaturesRegistry.kt index 095522217..d65a5d5a0 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/impl/FeaturesRegistry.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/impl/FeaturesRegistry.kt @@ -3,6 +3,7 @@ package org.utbot.jcdb.impl import org.utbot.jcdb.api.ByteCodeLocation import org.utbot.jcdb.api.ByteCodeLocationIndex import org.utbot.jcdb.api.Feature +import org.utbot.jcdb.api.GlobalIdsStore import org.utbot.jcdb.impl.index.index import org.utbot.jcdb.impl.storage.PersistentEnvironment import org.utbot.jcdb.impl.tree.ClassNode @@ -13,6 +14,7 @@ import java.util.concurrent.ConcurrentHashMap class FeaturesRegistry( private val persistence: PersistentEnvironment? = null, + val globalIdsStore: GlobalIdsStore, val features: List> ) : Closeable { private val indexes: ConcurrentHashMap>> = ConcurrentHashMap() @@ -38,7 +40,7 @@ class FeaturesRegistry( location: ByteCodeLocation, classes: Collection ) { - val builder = newBuilder() + val builder = newBuilder(globalIdsStore) classes.forEach { node -> index(node, builder) } diff --git a/core/src/main/kotlin/org/utbot/jcdb/impl/index/HierarchyIndex.kt b/core/src/main/kotlin/org/utbot/jcdb/impl/index/HierarchyIndex.kt index 2784457bd..fa82d1704 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/impl/index/HierarchyIndex.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/impl/index/HierarchyIndex.kt @@ -4,19 +4,17 @@ import kotlinx.collections.immutable.toImmutableMap import org.objectweb.asm.Type import org.objectweb.asm.tree.ClassNode import org.objectweb.asm.tree.MethodNode -import org.utbot.jcdb.api.ByteCodeLocation -import org.utbot.jcdb.api.ByteCodeLocationIndex -import org.utbot.jcdb.api.ByteCodeLocationIndexBuilder -import org.utbot.jcdb.api.Feature +import org.utbot.jcdb.api.* import java.io.InputStream import java.io.OutputStream -class HierarchyIndexBuilder : ByteCodeLocationIndexBuilder { +class HierarchyIndexBuilder(private val globalIdsStore: GlobalIdsStore) : + ByteCodeLocationIndexBuilder { // super class -> implementations private val parentToSubClasses = hashMapOf>() - override fun index(classNode: ClassNode) { + override suspend fun index(classNode: ClassNode) { val superClass = classNode.superName?.takeIf { it != "java/lang/Object" } @@ -28,27 +26,27 @@ class HierarchyIndexBuilder : ByteCodeLocationIndexBuilder> ) : ByteCodeLocationIndex { - override fun query(term: String): Sequence { - val parentClassId = GlobalIds.getId(term) + override suspend fun query(term: String): Sequence { + val parentClassId = globalIdsStore.getId(term) return parentToSubClasses[parentClassId]?.map { - GlobalIds.getName(it) + globalIdsStore.getName(it) }?.asSequence()?.filterNotNull() ?: emptySequence() } @@ -75,9 +74,13 @@ object Hierarchy : Feature { override val key = "hierarchy" - override fun newBuilder() = HierarchyIndexBuilder() + override fun newBuilder(globalIdsStore: GlobalIdsStore) = HierarchyIndexBuilder(globalIdsStore) - override fun deserialize(location: ByteCodeLocation, stream: InputStream): HierarchyIndex { + override fun deserialize( + globalIdsStore: GlobalIdsStore, + location: ByteCodeLocation, + stream: InputStream + ): HierarchyIndex { val reader = stream.reader() val result = hashMapOf>() reader.use { @@ -86,7 +89,7 @@ object Hierarchy : Feature { result[key] = value } } - return HierarchyIndex(location, result) + return HierarchyIndex(globalIdsStore, location, result) } override fun serialize(index: HierarchyIndex, out: OutputStream) { diff --git a/core/src/main/kotlin/org/utbot/jcdb/impl/index/Indexes.kt b/core/src/main/kotlin/org/utbot/jcdb/impl/index/Indexes.kt index 89c05a1ad..2d2a8fb41 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/impl/index/Indexes.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/impl/index/Indexes.kt @@ -1,6 +1,7 @@ package org.utbot.jcdb.impl.index import org.utbot.jcdb.api.ByteCodeLocationIndexBuilder +import org.utbot.jcdb.api.GlobalIdsStore import org.utbot.jcdb.impl.tree.ClassNode import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger @@ -14,7 +15,7 @@ suspend fun index(node: ClassNode, builder: ByteCodeLocationIndexBuilder<*, *>) } -object GlobalIds { +class GlobalIds : GlobalIdsStore { private val counter = AtomicInteger() @@ -24,7 +25,7 @@ object GlobalIds { @Volatile private var locked = false - fun getId(name: String): Int { + override suspend fun getId(name: String): Int { val id = all.getOrPut(name) { if (locked) { throw IllegalStateException("writing is locked") @@ -35,7 +36,7 @@ object GlobalIds { return id } - fun getName(id: Int): String? { + override suspend fun getName(id: Int): String? { return reversed.get(id) } diff --git a/core/src/main/kotlin/org/utbot/jcdb/impl/index/ReversedUsageIndex.kt b/core/src/main/kotlin/org/utbot/jcdb/impl/index/ReversedUsageIndex.kt index 14581435b..5f2862797 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/impl/index/ReversedUsageIndex.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/impl/index/ReversedUsageIndex.kt @@ -8,36 +8,33 @@ import org.objectweb.asm.tree.ClassNode import org.objectweb.asm.tree.FieldInsnNode import org.objectweb.asm.tree.MethodInsnNode import org.objectweb.asm.tree.MethodNode -import org.utbot.jcdb.api.ByteCodeLocation -import org.utbot.jcdb.api.ByteCodeLocationIndex -import org.utbot.jcdb.api.ByteCodeLocationIndexBuilder -import org.utbot.jcdb.api.Feature +import org.utbot.jcdb.api.* import java.io.InputStream import java.io.OutputStream -class ReversedUsageIndexBuilder : ByteCodeLocationIndexBuilder { +class ReversedUsageIndexBuilder(private val globalIdsStore: GlobalIdsStore) : ByteCodeLocationIndexBuilder { // class method -> usages of methods|fields private val fieldsUsages = hashMapOf>() private val methodsUsages = hashMapOf>() - override fun index(classNode: ClassNode) { + override suspend fun index(classNode: ClassNode) { } - override fun index(classNode: ClassNode, methodNode: MethodNode) { + override suspend fun index(classNode: ClassNode, methodNode: MethodNode) { val pureName = Type.getObjectType(classNode.name).className - val id = GlobalIds.getId(pureName) + val id = globalIdsStore.getId(pureName) methodNode.instructions.forEach { when (it) { is FieldInsnNode -> { val owner = Type.getObjectType(it.owner).className - val key = GlobalIds.getId(owner + "#" + it.name) + val key = globalIdsStore.getId(owner + "#" + it.name) fieldsUsages.getOrPut(key) { hashSetOf() }.add(id) } is MethodInsnNode -> { val owner = Type.getObjectType(it.owner).className - val key = GlobalIds.getId(owner + "#" + it.name) + val key = globalIdsStore.getId(owner + "#" + it.name) methodsUsages.getOrPut(key) { hashSetOf() }.add(id) } } @@ -48,6 +45,7 @@ class ReversedUsageIndexBuilder : ByteCodeLocationIndexBuilder>, internal val methodsUsages: ImmutableMap>, ) : ByteCodeLocationIndex { - override fun query(term: String): Sequence { - val usages = fieldsUsages[GlobalIds.getId(term)].orEmpty() + - methodsUsages[GlobalIds.getId(term)].orEmpty() - return usages.map { GlobalIds.getName(it) }.asSequence().filterNotNull() + override suspend fun query(term: String): Sequence { + val usages = fieldsUsages[globalIdsStore.getId(term)].orEmpty() + + methodsUsages[globalIdsStore.getId(term)].orEmpty() + return usages.map { globalIdsStore.getName(it) }.asSequence().filterNotNull() } } @@ -75,9 +74,9 @@ object ReversedUsages : Feature { override val key = "reversed-usages" - override fun newBuilder() = ReversedUsageIndexBuilder() + override fun newBuilder(globalIdsStore: GlobalIdsStore) = ReversedUsageIndexBuilder(globalIdsStore) - override fun deserialize(location: ByteCodeLocation, stream: InputStream): ReversedUsageIndex { + override fun deserialize(globalIdsStore: GlobalIdsStore, location: ByteCodeLocation, stream: InputStream): ReversedUsageIndex { val reader = stream.reader() val fields = hashMapOf>() @@ -93,7 +92,7 @@ object ReversedUsages : Feature { } } } - return ReversedUsageIndex(location, fields.toPersistentMap(), methods.toPersistentMap()) + return ReversedUsageIndex(location, globalIdsStore, fields.toPersistentMap(), methods.toPersistentMap()) } diff --git a/core/src/main/kotlin/org/utbot/jcdb/impl/storage/scheme/GlobalIdStore.kt b/core/src/main/kotlin/org/utbot/jcdb/impl/storage/scheme/GlobalIdStore.kt index c9d690ce4..cd1c1aa66 100644 --- a/core/src/main/kotlin/org/utbot/jcdb/impl/storage/scheme/GlobalIdStore.kt +++ b/core/src/main/kotlin/org/utbot/jcdb/impl/storage/scheme/GlobalIdStore.kt @@ -20,11 +20,11 @@ class GlobalIdStore(private val env: Environment) { ) } - fun sync() = synchronized(this) { + fun sync(globalIds: GlobalIds) = synchronized(this) { val current = counter.get() - val inMemoryCounter = GlobalIds.count + val inMemoryCounter = globalIds.count if (inMemoryCounter != current) { - val keys = GlobalIds.all(current, inMemoryCounter).toList() + val keys = globalIds.all(current, inMemoryCounter).toList() keys.windowed(10_000, step = 10_000, partialWindows = true) { env.executeInTransaction { txn -> it.forEach { @@ -35,23 +35,23 @@ class GlobalIdStore(private val env: Environment) { } } } - counter.set(GlobalIds.count) + counter.set(globalIds.count) } } - fun restore() { - GlobalIds.restore { + fun restore(globalIds: GlobalIds) { + globalIds.restore { env.executeInTransaction { txn -> store.openCursor(txn).use { cursor -> while (cursor.next) { - GlobalIds.append( + globalIds.append( IntegerBinding.entryToInt(cursor.key), StringBinding.entryToString(cursor.value) ) } } } - counter.set(GlobalIds.count) + counter.set(globalIds.count) } } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/utbot/jcdb/impl/IndexSerializationTest.kt b/core/src/test/kotlin/org/utbot/jcdb/impl/IndexSerializationTest.kt index 27f60454c..48a122486 100644 --- a/core/src/test/kotlin/org/utbot/jcdb/impl/IndexSerializationTest.kt +++ b/core/src/test/kotlin/org/utbot/jcdb/impl/IndexSerializationTest.kt @@ -5,20 +5,19 @@ import kotlinx.collections.immutable.toImmutableMap import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.utbot.jcdb.impl.fs.asByteCodeLocation -import org.utbot.jcdb.impl.index.Hierarchy -import org.utbot.jcdb.impl.index.HierarchyIndex -import org.utbot.jcdb.impl.index.ReversedUsageIndex -import org.utbot.jcdb.impl.index.ReversedUsages +import org.utbot.jcdb.impl.index.* import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream class IndexSerializationTest : LibrariesMixin { + val globalIds = GlobalIds() + @Test fun `hierarchy index serialization`() { val location = guavaLib.asByteCodeLocation() val index = HierarchyIndex( - location, mapOf( + globalIds, location, mapOf( 1 to persistentSetOf(1, 3, 4), 2 to persistentSetOf(7, 8), ) @@ -27,7 +26,7 @@ class IndexSerializationTest : LibrariesMixin { Hierarchy.serialize(index, out) val input = ByteArrayInputStream(out.toByteArray()) - val result = Hierarchy.deserialize(location, input) + val result = Hierarchy.deserialize(globalIds, location, input) with(result.parentToSubClasses) { assertEquals(2, size) assertEquals(sortedSetOf(1, 3, 4), get(1)?.toSortedSet()) @@ -40,6 +39,7 @@ class IndexSerializationTest : LibrariesMixin { val location = guavaLib.asByteCodeLocation() val index = ReversedUsageIndex( location, + globalIds, fieldsUsages = mapOf( 1 to persistentSetOf(1, 3, 4), 2 to persistentSetOf(7, 8), @@ -54,7 +54,7 @@ class IndexSerializationTest : LibrariesMixin { ReversedUsages.serialize(index, out) val input = ByteArrayInputStream(out.toByteArray()) - val result = ReversedUsages.deserialize(location, input) + val result = ReversedUsages.deserialize(globalIds, location, input) with(result.methodsUsages) { assertEquals(2, size) assertEquals(sortedSetOf(11, 13, 14), get(11)?.toSortedSet()) diff --git a/design.md b/design.md index c20e19a52..c623d47c8 100644 --- a/design.md +++ b/design.md @@ -65,6 +65,12 @@ About 80% of initialization time consumed by IO from file system in reading jar- Splitting bytecode processing into sync/async step gives performance improvement for startup. For example Java 1.8 rt.jar contains about 60mb of compressed bytecode. Main idea is to scan runtime locations depending on basic usage rules. Not so many applications using classes from `org.w3c.dom.*` or `jdk.*` packages. The idea is to move processing bytecode of rarely used classes to async step and release database instance faster. +# Hooks + +Compilation database can be extended with hooks. Hook is an environment extension with brings ability to implement remote api or call specific code during database lifecycle. + +Hook is called twice: after called twice when database is created and initialized properly and when it is closed. That means that + ## Performance `ClassTree` uses immutable lock-free structures from [Xodus database](https://github.com/JetBrains/xodus). This brings 10% lower memory footprint of application (`CompilationDatabaseImpl` created only with Java 8 runtime consumes about ~300Mb of heap memory). `ClassTree` implementation based on java concurrent collections consumes ~330Mb of memory. diff --git a/remote-rd/README.md b/remote-rd/README.md new file mode 100644 index 000000000..25a80754f --- /dev/null +++ b/remote-rd/README.md @@ -0,0 +1,28 @@ +## Rd extension for Java Compilation Database + +Extension provides proxy implementation to java compilation database running in a separate system process. Proxy implementation uses [rd protocol](https://github.com/JetBrains/rd) under the hood. + +### Limitations + +Rd protocol implies 1-1 communication between server and client. That means that stopping client database implies stopping server as well. + +### Usage example + +Create server: + +```kotlin + val db = compilationDatabase { + useProcessJavaRuntime() + exposeRd(port = 9090) // this will expose rd based server api + } +``` + +creating client: + +```kotlin + val clientDB: CompilationDatabase = remoteRdClient(port = 9090) // this will create client database + val classId = clientDB.classpathSet(emptyList()).findClassOrNull("java.util.HashMap") + println(classId.methods().size) + clientDB.close() // will close server side extension. +``` + diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/RDServer.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/RemoteRdServer.kt similarity index 70% rename from remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/RDServer.kt rename to remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/RemoteRdServer.kt index f5568ff00..8a5188eb1 100644 --- a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/RDServer.kt +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/RemoteRdServer.kt @@ -6,16 +6,13 @@ import com.jetbrains.rd.framework.Protocol import com.jetbrains.rd.framework.SocketWire import com.jetbrains.rd.util.lifetime.Lifetime import com.jetbrains.rd.util.threading.SingleThreadScheduler -import kotlinx.coroutines.runBlocking import org.utbot.jcdb.api.ClasspathSet import org.utbot.jcdb.api.Hook -import org.utbot.jcdb.compilationDatabase import org.utbot.jcdb.impl.CompilationDatabaseImpl -import org.utbot.jcdb.remote.rd.client.RemoteCompilationDatabase import java.util.concurrent.ConcurrentHashMap -class RDServer(port: Int, val db: CompilationDatabaseImpl) : Hook { +class RemoteRdServer(port: Int, val db: CompilationDatabaseImpl) : Hook { private val lifetimeDef = Lifetime.Eternal.createNested() @@ -47,19 +44,4 @@ class RDServer(port: Int, val db: CompilationDatabaseImpl) : Hook { lifetimeDef.terminate(true) } -} - -fun main() { - val db = runBlocking { - compilationDatabase { - useProcessJavaRuntime() - } as CompilationDatabaseImpl - } - RDServer(8080, db).afterStart() - - val client = RemoteCompilationDatabase(8080) - runBlocking { - val classpathSet = client.classpathSet(emptyList()) - println(classpathSet) - } } \ No newline at end of file diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/Resources.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/Resources.kt index 47d8c2780..ba43f402a 100644 --- a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/Resources.kt +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/Resources.kt @@ -6,7 +6,7 @@ import com.jetbrains.rd.framework.Protocol import com.jetbrains.rd.framework.base.IRdBindable import com.jetbrains.rd.framework.base.static import com.jetbrains.rd.framework.impl.RdCall -import kotlinx.coroutines.runBlocking +import com.jetbrains.rd.framework.util.setSuspend import kotlinx.serialization.cbor.Cbor import kotlinx.serialization.encodeToByteArray import org.utbot.jcdb.api.ClassId @@ -21,12 +21,14 @@ abstract class CallResource(val id: Int, val name: String) { private val clientProperty: RdCall get() = RdCall().static(id).makeAsync() private fun serverProperty(db: CompilationDatabase): RdCall { - return RdCall(null, null) { req -> - db.handler(req) - }.makeAsync().static(id) + val call = RdCall(null, null) { _ -> throw Error() } + call.setSuspend { _, request -> + db.handler(request) + } + return call.makeAsync().static(id) } - abstract fun CompilationDatabase.handler(req: REQUEST): RESPONSE + abstract suspend fun CompilationDatabase.handler(req: REQUEST): RESPONSE open val serializers: List> = emptyList() fun clientCall(protocol: Protocol): RdCall { @@ -66,11 +68,9 @@ class GetClasspathResource(private val classpaths: ConcurrentHashMap = ConcurrentHashMap()) : CallResource(2, "close-classpath") { - override fun CompilationDatabase.handler(req: String) { + override suspend fun CompilationDatabase.handler(req: String) { classpaths[req]?.close() classpaths.remove(req) } @@ -89,12 +89,10 @@ abstract class AbstractClasspathResource( id: Int, name: String, private val classpaths: ConcurrentHashMap = ConcurrentHashMap() ) : CallResource(id, name) { - override fun CompilationDatabase.handler(req: REQUEST): RESPONSE { + override suspend fun CompilationDatabase.handler(req: REQUEST): RESPONSE { val key = req.cpKey val cp = classpaths[key] ?: throw IllegalStateException("No classpath found by key $key. \n Create it first") - return runBlocking { - cp.handler(req) - } + return cp.handler(req) } protected abstract suspend fun ClasspathSet.handler(req: REQUEST): RESPONSE @@ -111,6 +109,8 @@ abstract class AbstractClasspathResource( class GetClassResource(classpaths: ConcurrentHashMap = ConcurrentHashMap()) : AbstractClasspathResource(3, "get-class", classpaths) { + override val serializers = listOf(GetClassReq, GetClassRes) + override suspend fun ClasspathSet.handler(req: GetClassReq): GetClassRes? { return findClassOrNull(req.className)?.toGetClass() } @@ -128,12 +128,26 @@ class GetSubClassesResource(classpaths: ConcurrentHashMap } +class GetGlobalId : CallResource(5, "get-global-id") { + + override suspend fun CompilationDatabase.handler(req: String): Int { + return globalIdStore.getId(req) + } +} + +class GetGlobalName : CallResource(6, "get-global-name") { + + override suspend fun CompilationDatabase.handler(req: Int): String? { + return globalIdStore.getName(req) + } +} + private suspend fun ClassId.convertToContainer(): ClassInfoContainer { return when (this) { is ArrayClassIdImpl -> ArrayClassInfo(elementClass.convertToContainer()) is ClassIdImpl -> info() is PredefinedPrimitive -> info - else -> throw IllegalStateException("Can't convert class ${name} to serializable class info") + else -> throw IllegalStateException("Can't convert class $name to serializable class info") } } diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteCompilationDatabase.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteCompilationDatabase.kt index 486dba63b..b241e64a8 100644 --- a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteCompilationDatabase.kt +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteCompilationDatabase.kt @@ -31,10 +31,15 @@ class RemoteCompilationDatabase(port: Int) : CompilationDatabase { private val closeClasspath = CloseClasspathResource().clientCall(clientProtocol) private val getSubClasses = GetSubClassesResource().clientCall(clientProtocol) + private val getId = GetGlobalId().clientCall(clientProtocol) + private val getName = GetGlobalName().clientCall(clientProtocol) + init { scheduler.flush() } + override val globalIdStore = RemoteGlobalIds(getName, getId) + override suspend fun classpathSet(dirOrJars: List): ClasspathSet { val id = getClasspath.startSuspending(GetClasspathReq(dirOrJars.map { it.absolutePath }.sorted())) return RemoteClasspathSet( diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteGlobalIds.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteGlobalIds.kt new file mode 100644 index 000000000..4070c20fe --- /dev/null +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteGlobalIds.kt @@ -0,0 +1,18 @@ +package org.utbot.jcdb.remote.rd.client + +import com.jetbrains.rd.framework.impl.RdCall +import org.utbot.jcdb.api.GlobalIdsStore + +class RemoteGlobalIds( + private val getName: RdCall, + private val getId: RdCall +) : GlobalIdsStore { + + override suspend fun getId(name: String): Int { + return getId.startSuspending(name) + } + + override suspend fun getName(id: Int): String? { + return getName.startSuspending(id) + } +} \ No newline at end of file diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteClassId.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteImpl.kt similarity index 84% rename from remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteClassId.kt rename to remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteImpl.kt index dd8b07750..7b7c2c182 100644 --- a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteClassId.kt +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/client/RemoteImpl.kt @@ -1,10 +1,17 @@ package org.utbot.jcdb.remote.rd.client +import mu.KLogging +import org.objectweb.asm.ClassReader +import org.objectweb.asm.Opcodes +import org.objectweb.asm.tree.ClassNode import org.objectweb.asm.tree.MethodNode import org.utbot.jcdb.api.* import org.utbot.jcdb.impl.fs.ByteCodeConverter import org.utbot.jcdb.impl.fs.asByteCodeLocation -import org.utbot.jcdb.impl.signature.* +import org.utbot.jcdb.impl.signature.FieldResolution +import org.utbot.jcdb.impl.signature.FieldSignature +import org.utbot.jcdb.impl.signature.MethodSignature +import org.utbot.jcdb.impl.signature.TypeSignature import org.utbot.jcdb.impl.suspendableLazy import org.utbot.jcdb.impl.types.ClassInfo import org.utbot.jcdb.impl.types.FieldInfo @@ -88,9 +95,7 @@ class RemoteClassId( return outerClass != null && outerClass.name == null } - override suspend fun signature(): TypeResolution { - return TypeSignature.of(classInfo.signature, classpath) - } + override suspend fun signature() = TypeSignature.of(classInfo.signature, classpath) override suspend fun outerMethod(): MethodId? { val outerMethod = classInfo.outerMethod @@ -125,8 +130,13 @@ class RemoteClassId( } -class RemoteMethodId(override val classId: ClassId, private val methodInfo: MethodInfo, val classpath: ClasspathSet) : - MethodId { +class RemoteMethodId( + override val classId: ClassId, + private val methodInfo: MethodInfo, + private val classpath: ClasspathSet +) : MethodId { + + companion object : KLogging() override val name: String get() = methodInfo.name @@ -150,9 +160,7 @@ class RemoteMethodId(override val classId: ClassId, private val methodInfo: Meth } } - override suspend fun signature(): MethodResolution { - return MethodSignature.of(methodInfo.signature, classId.classpath) - } + override suspend fun signature() = MethodSignature.of(methodInfo.signature, classId.classpath) override suspend fun returnType() = lazyReturnType() @@ -165,7 +173,11 @@ class RemoteMethodId(override val classId: ClassId, private val methodInfo: Meth } override suspend fun readBody(): MethodNode? { - TODO("Not yet implemented") + logger.error("GETTING BYTECODE IN REMOTE CLIENT") + val byteCode = classId.location?.resolve(classId.name) ?: return null + val classNode = ClassNode(Opcodes.ASM9) + ClassReader(byteCode).accept(classNode, ClassReader.EXPAND_FRAMES) + return classNode.methods.firstOrNull { it.name == methodInfo.name && it.desc == methodInfo.desc } } override suspend fun access() = methodInfo.access diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/rd.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/rd.kt index 0339c0fbd..aa487689c 100644 --- a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/rd.kt +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/rd.kt @@ -6,10 +6,10 @@ import org.utbot.jcdb.remote.rd.client.RemoteCompilationDatabase fun CompilationDatabaseSettings.exposeRd(port: Int) { withHook { - RDServer(port, it) + RemoteRdServer(port, it) } } -fun rdDatabase(port: Int): CompilationDatabase { +fun remoteRdClient(port: Int): CompilationDatabase { return RemoteCompilationDatabase(port) } \ No newline at end of file diff --git a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/InternalApi.kt b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/remoteApi.kt similarity index 68% rename from remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/InternalApi.kt rename to remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/remoteApi.kt index 2c4f5a307..f189e735a 100644 --- a/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/InternalApi.kt +++ b/remote-rd/src/main/kotlin/org/utbot/jcdb/remote/rd/remoteApi.kt @@ -3,12 +3,7 @@ package org.utbot.jcdb.remote.rd import com.jetbrains.rd.framework.* import kotlin.reflect.KClass -val serializers = Serializers().also { - it.register(GetClasspathReq) - it.register(GetClassReq) - it.register(GetClassRes) -} - +val serializers = Serializers() class GetClasspathReq(val locations: List) { @@ -17,11 +12,9 @@ class GetClasspathReq(val locations: List) { override val _type: KClass = GetClasspathReq::class override fun read(ctx: SerializationCtx, buffer: AbstractBuffer): GetClasspathReq { - return GetClasspathReq( - buffer.readArray { - buffer.readString() - }.toList() - ) + return GetClasspathReq(buffer.readArray { + buffer.readString() + }.toList()) } override fun write(ctx: SerializationCtx, buffer: AbstractBuffer, value: GetClasspathReq) { @@ -117,4 +110,46 @@ class GetSubClassesRes(val classes: List) { } } } -} \ No newline at end of file +} + +class CallIndexReq(cpKey: String, val indexKey: String, val location: String, val term: String): ClasspathBasedReq(cpKey) { + + companion object : IMarshaller { + + override val _type: KClass = CallIndexReq::class + + override fun read(ctx: SerializationCtx, buffer: AbstractBuffer): CallIndexReq { + return CallIndexReq( + buffer.readString(), + buffer.readString(), + buffer.readString(), + buffer.readString() + ) + } + + override fun write(ctx: SerializationCtx, buffer: AbstractBuffer, value: CallIndexReq) { + buffer.writeString(value.cpKey) + buffer.writeString(value.indexKey) + buffer.writeString(value.location) + buffer.writeString(value.term) + } + } +} +// +//class CallIndexRes(val type: String, val result: List<*>) { +// +// companion object : IMarshaller { +// +// override val _type: KClass = CallIndexRes::class +// +// override fun read(ctx: SerializationCtx, buffer: AbstractBuffer): CallIndexRes { +// return CallIndexRes(emptyList()) +// } +// +// override fun write(ctx: SerializationCtx, buffer: AbstractBuffer, value: CallIndexRes) { +// buffer.writeString(value.type) +// buffer.writeString(value.indexKey) +// buffer.writeString(value.term) +// } +// } +//} \ No newline at end of file diff --git a/remote-rd/src/test/kotlin/org/utbot/jcdb/remote/rd/RemoteClientTest.kt b/remote-rd/src/test/kotlin/org/utbot/jcdb/remote/rd/RemoteClientTest.kt index 5b51f8d81..006cc9e2e 100644 --- a/remote-rd/src/test/kotlin/org/utbot/jcdb/remote/rd/RemoteClientTest.kt +++ b/remote-rd/src/test/kotlin/org/utbot/jcdb/remote/rd/RemoteClientTest.kt @@ -36,7 +36,7 @@ class RemoteClientTest { } } - private val db = rdDatabase(8080) + private val db = remoteRdClient(8080) private val cp = runBlocking { db.classpathSet(allClasspath) }