Feature/graal support (#256)

* Add graalVM native image support for desktop Platforms. * Fix bunnymark project Co-authored-by: Cedric Hippmann <[email protected]>
utopia-rise · Aug 13, 2021 · 2f431c0 · 2f431c0
1 parent 944cad3
commit 2f431c0
Show file tree

Hide file tree

Showing 32 changed files with 1,190 additions and 134 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,5 +27,5 @@ add_compile_definitions(DEBUG_ENABLED)
 # platforms
 #add_compile_definitions(WINDOWS_ENABLED)
 #add_compile_definitions(X11_ENABLED)
-#add_compile_definitions(OSX_ENABLED)
-add_compile_definitions(__ANDROID__)
+add_compile_definitions(OSX_ENABLED)
+#add_compile_definitions(__ANDROID__)
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -64,6 +64,7 @@ nav:
       - Custom gradle wrapper path: user-guide/advanced/custom_gradle_wrapper_path.md
       - Commandline args: user-guide/advanced/commandline-args.md
       - Kotlin singletons: user-guide/advanced/kotlin-singleton.md
+      - GraalVM native-image: user-guide/advanced/graal-vm-native-image.md
   - Contributing:
     - General: contribution/general.md
     - Guidelines: contribution/guidelines.md

diff --git a/docs/src/doc/user-guide/advanced/commandline-args.md b/docs/src/doc/user-guide/advanced/commandline-args.md
@@ -2,6 +2,7 @@ The following command line args can be supplied to customize the behaviour or th
 
 | Argument | Default value | Description |
 | --- | --- | ---|
+| --java-vm-type | jvm | Defines the VM to run on. Possible values are `jvm` and `graal_native_image`. When set to `graal_native_image` it uses Graal native image. This has no effect on android platform. |
 | --jvm-debug-port | | Defines the port to which you can attach a remote debugger. **Note:** the module `jdk.jdwp.agent` is needed in the embedded JRE if you want to debug your application. If you need `jmx`, also the module `jdk.management.agent` is needed |
 | --jvm-debug-address | | Defines which adresses are allowed for debugging |
 | --jvm-jmx-port | | Defines the jmx port. **Note:** the module `jdk.management.agent` is needed in the embedded JRE to be able to use jmx |

diff --git a/docs/src/doc/user-guide/advanced/graal-vm-native-image.md b/docs/src/doc/user-guide/advanced/graal-vm-native-image.md
@@ -0,0 +1,37 @@
+## Default GraalVM native image configuration
+
+!!! warning  
+    Reloading code changes in the editor is not possible with native-image, as it would require to reload the JVM.
+
+On desktop platform, you can choose to build a [GraalVM native image](https://www.graalvm.org/reference-manual/native-image/). You first need to install graal-vm and its tool native image. Then, you should set `GRAALVM_HOME` environment variable to point to GraalVM's home folder.  
+
+On windows, you should add `VC_VARS_PATH` environment variable to point to vcvars bat file. This is mandatory so that we can initialize visual studio tools. (Example: `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat`)
+
+In order to build a native image, you should add configuration to gradle plugin to enable building of native-image this way:
+```kotlin
+godot {
+    isGraalExportEnabled.set(true)
+    nativeImageToolPath.set("${System.getenv("GRAALVM_HOME")}/bin/native-image")
+
+    windowsDeveloperVCVarsPath.set("System.getenv("VC_VARS_PATH")")
+}
+```
+
+In order to use the generated native-image, you can pass `--java-vm-type=graal` argument to engine, or simply change `godot_kotlin_configuration.json` to set vm_type to `graal`.
+
+## Reflection, libraries and JNI with native image
+
+GraalVM native image performs AOT compilation. In order to be able to use reflection and jni, you need to provide an additional configuration file.  
+This applies also for third party library you use that would do reflection. You can find documentation on how to easily generate them [here](https://www.graalvm.org/reference-manual/native-image/Agent/).
+
+In order to append those configurations add the json in `graal` folder of your project (it should be generated on first graal native image use). Then you can add `additionalGraalJniConfigurationFiles` parameter, this way:
+
+```kotlin
+godot {
+    isGraalExportEnabled.set(true)
+    nativeImageToolPath.set("${System.getenv("GRAALVM_HOME")}/bin/native-image")
+
+    windowsDeveloperVCVarsPath.set(System.getenv("VC_VARS_PATH"))
+    additionalGraalJniConfigurationFiles.set(arrayOf("my-jni-configuration-file.json", "another-conf.json"))
+}
+```
diff --git a/docs/src/doc/user-guide/exporting.md b/docs/src/doc/user-guide/exporting.md
@@ -65,3 +65,16 @@ On android, we do not embed a JVM, we use the existing ART provided by the OS. I
 
 !!! warning
     Similar to the desktop targets, the game copies the needed jar files to the `user://` directory upon first execution or if the files have changed. On android this is the applications `files` folder. If you do IO operations on Android, never empty the whole `files` folder! Only delete what you have added or exclude the following two files when clearing the `files` folder: `godot-bootstrap-dex.jar` and `main-dex.jar`.
+
+## GraalVM Native Image
+
+!!! warning
+    GraalVM native image is a advanced feature and requires a lot of work to support. Especially if you rely on many third party libraries.
+
+In order to build for graalvm, follow `GraalVM native-image` section in advanced user guide.
+
+As `main.jar` and `godot-bootstrap.jar`, `usercode` shared library is set in `pck` during the export process and is copied to user directory. Don't forget to destroy them when creating an uninstaller.
+
+- GraalVM native image is not available for android platform.
+
+On desktop platform default export is inferred by the `godot_kotin_configuration.json` file. You still can export for `jvm` and `native-image`, by adding feature `export-all-jvm`. In this case, the default JVM started by engine is the one from `godot_kotin_configuration.json` and can be overridden by command line. 
diff --git a/harness/bunnymark/.gitignore b/harness/bunnymark/.gitignore
@@ -490,3 +490,5 @@ hs_err_pid*
 build/
 
 /.gradle/
+
+graal/godot-kotlin-graal-jni-config.json
diff --git a/harness/bunnymark/build.gradle.kts b/harness/bunnymark/build.gradle.kts
@@ -1,9 +1,15 @@
-
 plugins {
-    kotlin("jvm") version "1.4.32"
+    kotlin("jvm") version "1.5.21"
     id("com.utopia-rise.godot-kotlin-jvm")
 }
 
 repositories {
     mavenCentral()
 }
+
+godot {
+    //uncomment to test graal vm native image
+//    isGraalNativeImageExportEnabled.set(true)
+//    nativeImageToolPath.set("${System.getenv("GRAALVM_HOME")}/bin/native-image")
+//    windowsDeveloperVCVarsPath.set(System.getenv("VC_VARS_PATH"))
+}
diff --git a/harness/bunnymark/project.godot b/harness/bunnymark/project.godot
@@ -13,9 +13,39 @@ _global_script_classes=[ {
 "class": "Bunny",
 "language": "GDScript",
 "path": "res://benchmarks/BunnymarkV3/gd/Bunny.gd"
+}, {
+"base": "Node2D",
+"class": "BunnymarkV1DrawTexture",
+"language": "Kotlin",
+"path": "res://src/main/kotlin/godot/benchmark/bunnymark/BunnymarkV1DrawTexture.kt"
+}, {
+"base": "Node2D",
+"class": "BunnymarkV1Sprites",
+"language": "Kotlin",
+"path": "res://src/main/kotlin/godot/benchmark/bunnymark/BunnymarkV1Sprites.kt"
+}, {
+"base": "Node2D",
+"class": "BunnymarkV2",
+"language": "Kotlin",
+"path": "res://src/main/kotlin/godot/benchmark/bunnymark/BunnymarkV2.kt"
+}, {
+"base": "Node2D",
+"class": "BunnymarkV3",
+"language": "Kotlin",
+"path": "res://src/main/kotlin/godot/benchmark/bunnymark/BunnymarkV3.kt"
+}, {
+"base": "Sprite",
+"class": "godot_benchmark_bunnymark_v3_Bunny",
+"language": "Kotlin",
+"path": "res://src/main/kotlin/godot/benchmark/bunnymark/v3/Bunny.kt"
 } ]
 _global_script_class_icons={
-"Bunny": ""
+"Bunny": "",
+"BunnymarkV1DrawTexture": "",
+"BunnymarkV1Sprites": "",
+"BunnymarkV2": "",
+"BunnymarkV3": "",
+"godot_benchmark_bunnymark_v3_Bunny": ""
 }
 
 [application]

diff --git a/harness/bunnymark/settings.gradle.kts b/harness/bunnymark/settings.gradle.kts
@@ -8,20 +8,19 @@ includeBuild("../../kt/api-generator") {
 includeBuild("../../kt") {
     dependencySubstitution {
         substitute(module("com.utopia-rise:godot-gradle-plugin")).with(project(":godot-gradle-plugin"))
-        substitute(module("com.utopia-rise:godot-annotation-processor")).with(project(":godot-annotation-processor"))
         substitute(module("com.utopia-rise:godot-runtime")).with(project(":godot-runtime"))
         substitute(module("com.utopia-rise:godot-library")).with(project(":godot-library"))
         substitute(module("com.utopia-rise:godot-bootstrap")).with(project(":godot-bootstrap"))
-        substitute(module("com.utopia-rise:godot-kotlin-compiler-plugin-common")).with(project(":godot-kotlin-compiler-plugin-common"))
-        substitute(module("com.utopia-rise:godot-kotlin-compiler-plugin")).with(project(":godot-kotlin-compiler-plugin"))
-        substitute(module("com.utopia-rise:godot-kotlin-entry-generator")).with(project(":godot-kotlin-entry-generator"))
+        substitute(module("com.utopia-rise:godot-kotlin-symbol-processor")).with(project(":godot-kotlin-symbol-processor"))
+        substitute(module("com.utopia-rise:godot-entry-generator")).with(project(":godot-entry-generator"))
     }
 }
 
 pluginManagement {
     repositories {
         mavenCentral()
         gradlePluginPortal()
+        google()
     }
     resolutionStrategy.eachPlugin {
         if (requested.id.id == "com.utopia-rise.godot-kotlin-jvm") {

diff --git a/harness/tests/.gitignore b/harness/tests/.gitignore
@@ -3,3 +3,5 @@
 
 # Ignore Gradle build output directory
 build
+
+graal/godot-kotlin-graal-jni-config.json
diff --git a/harness/tests/build.gradle.kts b/harness/tests/build.gradle.kts
@@ -1,4 +1,3 @@
-
 plugins {
     kotlin("jvm") version "1.5.21"
     id("com.utopia-rise.godot-kotlin-jvm")
@@ -13,6 +12,11 @@ godot {
 //    isAndroidExportEnabled.set(true)
 //    d8ToolPath = File("${System.getenv("ANDROID_SDK_ROOT")}/build-tools/30.0.3/d8")
 //    androidCompileSdkDir = File("${System.getenv("ANDROID_SDK_ROOT")}/platforms/android-30")
+
+    //uncomment to test graal vm native image
+//    isGraalNativeImageExportEnabled.set(true)
+//    nativeImageToolPath.set("${System.getenv("GRAALVM_HOME")}/bin/native-image")
+//    windowsDeveloperVCVarsPath.set(System.getenv("VC_VARS_PATH"))
 }
 
 dependencies {

diff --git a/harness/tests/export_presets.cfg b/harness/tests/export_presets.cfg
@@ -8,7 +8,7 @@ export_filter="all_resources"
 include_filter="*.txt, *.cfg"
 exclude_filter=""
 export_path="../tests-win64-export/Godot Kotlin Tests.exe"
-script_export_mode=1
+script_export_mode=0
 script_encryption_key=""
 
 [preset.0.options]

diff --git a/harness/tests/godot_kotlin_configuration.json b/harness/tests/godot_kotlin_configuration.json
@@ -0,0 +1 @@
+{"vm_type":"jvm"}
diff --git a/kt/godot-runtime/src/main/kotlin/godot/runtime/Bootstrap.kt b/kt/godot-runtime/src/main/kotlin/godot/runtime/Bootstrap.kt
@@ -28,51 +28,55 @@ class Bootstrap {
     private var engineTypesRegistered: Boolean = false
 
     fun init(isEditor: Boolean, projectRootDir: String, jarRootDir: String, jarFile: String, loader: ClassLoader?) {
-        val libsDir = Paths.get(jarRootDir)
-        val mainJarPath = libsDir.resolve(jarFile)
-
-        if (File(mainJarPath.toString()).exists()) {
-            doInit(mainJarPath.toUri().toURL(), loader)
+        if (jarFile == "graal_usercode") {
+            doInitGraal()
         } else {
-            if (isEditor) {
-                ::warning
-            } else {
-                ::err
-            }.invoke("No main.jar detected. No classes will be loaded. Build the gradle project to load classes")
-        }
+            val libsDir = Paths.get(jarRootDir)
+            val mainJarPath = libsDir.resolve(jarFile)
 
-        if (isEditor) {
-            watchService = FileSystems.getDefault().newWatchService()
-            val watchKey = getBuildLockDir(projectRootDir).toPath().register(
-                watchService,
-                StandardWatchEventKinds.ENTRY_CREATE,
-                StandardWatchEventKinds.ENTRY_DELETE,
-                StandardWatchEventKinds.ENTRY_MODIFY,
-            )
-
-            executor = Executors.newSingleThreadScheduledExecutor { runnable ->
-                val thread = Thread(runnable)
-                thread.isDaemon = true
-                thread
+            if (File(mainJarPath.toString()).exists()) {
+                doInit(mainJarPath.toUri().toURL(), loader)
+            } else {
+                if (isEditor) {
+                    ::warning
+                } else {
+                    ::err
+                }.invoke("No main.jar detected. No classes will be loaded. Build the gradle project to load classes")
             }
 
-            executor!!.scheduleAtFixedRate({
-                val events = watchKey.pollEvents()
-                if (events.isNotEmpty()) {
-                    if (File(getBuildLockDir(projectRootDir), "buildLock.lock").exists()) {
-                        info("Build lock present. Not reloading...")
-                        return@scheduleAtFixedRate
-                    }
-                    info("Changes detected, reloading classes ...")
-                    clearClassesCache()
+            if (isEditor) {
+                watchService = FileSystems.getDefault().newWatchService()
+                val watchKey = getBuildLockDir(projectRootDir).toPath().register(
+                    watchService,
+                    StandardWatchEventKinds.ENTRY_CREATE,
+                    StandardWatchEventKinds.ENTRY_DELETE,
+                    StandardWatchEventKinds.ENTRY_MODIFY,
+                )
+
+                executor = Executors.newSingleThreadScheduledExecutor { runnable ->
+                    val thread = Thread(runnable)
+                    thread.isDaemon = true
+                    thread
+                }
 
-                    if (File(mainJarPath.toString()).exists()) {
-                        doInit(mainJarPath.toUri().toURL(), null) //no classloader so new main jar get's loaded
-                    } else {
-                        warning("No main.jar detected. No classes will be loaded. Build the project to load classes")
+                executor!!.scheduleAtFixedRate({
+                    val events = watchKey.pollEvents()
+                    if (events.isNotEmpty()) {
+                        if (File(getBuildLockDir(projectRootDir), "buildLock.lock").exists()) {
+                            info("Build lock present. Not reloading...")
+                            return@scheduleAtFixedRate
+                        }
+                        info("Changes detected, reloading classes ...")
+                        clearClassesCache()
+
+                        if (File(mainJarPath.toString()).exists()) {
+                            doInit(mainJarPath.toUri().toURL(), null) //no classloader so new main jar get's loaded
+                        } else {
+                            warning("No main.jar detected. No classes will be loaded. Build the project to load classes")
+                        }
                     }
-                }
-            }, 3, 3, TimeUnit.SECONDS)
+                }, 3, 3, TimeUnit.SECONDS)
+            }
         }
     }
 
@@ -94,6 +98,16 @@ class Bootstrap {
         classloader = classLoader ?: URLClassLoader(arrayOf(mainJar), this::class.java.classLoader)
         Thread.currentThread().contextClassLoader = classloader
         serviceLoader = ServiceLoader.load(Entry::class.java, classloader)
+        initializeUsingEntry()
+    }
+
+    private fun doInitGraal() {
+        registry = ClassRegistry()
+        serviceLoader = ServiceLoader.load(Entry::class.java)
+        initializeUsingEntry()
+    }
+
+    private fun initializeUsingEntry() {
         val entryIterator = serviceLoader.iterator()
         if (entryIterator.hasNext()) {
             with(entryIterator.next()) {

diff --git a/kt/plugins/godot-gradle-plugin/src/main/kotlin/godot/gradle/GodotExtension.kt b/kt/plugins/godot-gradle-plugin/src/main/kotlin/godot/gradle/GodotExtension.kt
@@ -26,4 +26,40 @@ open class GodotExtension(objects: ObjectFactory) {
      * example: "${System.getenv("ANDROID_SDK_ROOT")}/platforms/android-30"
      */
     var androidCompileSdkDir: File? = null
+
+    /**
+     * enable Graal Native Image Export
+     *
+     * if is set to true, native-image tool and graalvm home has to be resolvable
+     */
+    val isGraalNativeImageExportEnabled = objects.property(Boolean::class.java)
+
+    /**
+     * path to the native-image tool used to convert jar to native.
+     *
+     * example: "${System.getenv("GRAALVM_HOME")}/bin/native-image"
+     */
+    val nativeImageToolPath = objects.property(String::class.java)
+
+    /**
+     * Windows specific.
+     * Path to Visual Studio VCVARS to initialize native developer tools.
+     *
+     * example: System.getenv("VC_VARS_PATH")
+     */
+    val windowsDeveloperVCVarsPath = objects.property(String::class.java)
+
+    /**
+     * Additional Graal JNI/reflection configurations.
+     *
+     * example: arrayOf("my-jni-configuration-file.json", "another-conf.json")
+     */
+    val additionalGraalJniConfigurationFiles = objects.property(Array<String>::class.java)
+
+    /**
+     * enable verbose mode on native image generation.
+     *
+     * if set to true, native-image tool will be in verbose mode.
+     */
+    val isGraalVmNativeImageGenerationVerbose = objects.property(Boolean::class.java)
 }
diff --git a/kt/plugins/godot-gradle-plugin/src/main/kotlin/godot/gradle/GodotPlugin.kt b/kt/plugins/godot-gradle-plugin/src/main/kotlin/godot/gradle/GodotPlugin.kt
@@ -107,6 +107,12 @@ class GodotPlugin : Plugin<Project> {
             if (androidCompileSdkDirFile != null) {
                 androidCompileSdkDir = d8Tool
             }
+
+            isGraalNativeImageExportEnabled.set(false)
+            nativeImageToolPath.set("native-image")
+            additionalGraalJniConfigurationFiles.set(arrayOf())
+            isGraalVmNativeImageGenerationVerbose.set(false)
+            windowsDeveloperVCVarsPath.set("\"%VC_VARS_PATH%\"")
         }
     }
 }
diff --git a/...le-plugin/src/main/kotlin/godot/gradle/exception/GraalNativeImageToolNotFountException.kt b/...le-plugin/src/main/kotlin/godot/gradle/exception/GraalNativeImageToolNotFountException.kt
@@ -0,0 +1,5 @@
+package godot.gradle.exception
+
+class GraalNativeImageToolNotFountException: IllegalArgumentException(
+    "native-image tool not set! Make sure you've either set the GRAALVM_HOME environment variable or set the nativeImageToolPath. For more information, visit: https://godot-kotl.in/en/stable/user-guide/advanced/graal-vm-native-image"
+)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -490,3 +490,5 @@ hs_err_pid*
		build/

		/.gradle/

		graal/godot-kotlin-graal-jni-config.json
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,3 +3,5 @@

		# Ignore Gradle build output directory
		build

		graal/godot-kotlin-graal-jni-config.json