From 8dec7ae90bc02de45c5273d3f00271e7eb717cae Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 13 May 2021 09:04:33 -0500 Subject: [PATCH 1/3] Support for jpy integration junit tests Partial #430 --- py/jpy-integration/build.gradle | 44 +++++++++++++++++++ .../src/javaToPython/build.gradle.template | 18 ++++++++ 2 files changed, 62 insertions(+) create mode 100644 py/jpy-integration/src/javaToPython/build.gradle.template diff --git a/py/jpy-integration/build.gradle b/py/jpy-integration/build.gradle index 97832b65589..885958f22f5 100644 --- a/py/jpy-integration/build.gradle +++ b/py/jpy-integration/build.gradle @@ -110,6 +110,50 @@ Task pythonToJava = tasks.create 'unittestPythonToJava', { } if (!PyEnv.pythonEnabled(project)) { + Closure> gradleTestInDocker = { String taskName, SourceSet sourceSet -> + + def gradleWrapper = tasks.register("${taskName}GradleInit", Wrapper.class) { wrapper -> + wrapper.scriptFile "${buildDir}/template-project/gradlew" + wrapper.jarFile "${buildDir}/template-project/gradle/wrapper/gradle-wrapper.jar" + } + return Docker.registerDockerTask(project, taskName) { + copyIn { + dependsOn gradleWrapper + from ("${buildDir}/template-project") { + into 'project' + } + from(project.file('src/javaToPython/build.gradle.template')) { + into 'project' + rename { file -> 'build.gradle' } + } + from (sourceSet.runtimeClasspath) { + into 'classpath' + } + from (sourceSet.output.getClassesDirs()) { + into 'classes' + } + } + dockerfile { + // base image with default java, python, wheels + from 'deephaven/runtime-base' + + // set up the project + copyFile 'project', '/project' + workingDir '/project' + // run once with no actual classes, so that gradle is preinstalled and cached + runCommand '/project/gradlew' + copyFile 'classpath', '/classpath' + copyFile 'classes', '/classes' + } + entrypoint = ['/project/gradlew', 'test', '--info'] + containerOutPath = '/project/build/test-results/test/' + copyOut { + into "${buildDir}/test-results/${taskName}" + } + } + } + testJavaToPython.dependsOn gradleTestInDocker('java-to-python-test', sourceSets.javaToPython) + // Task testTask = venvTest.javaTest(project, "java-to-python-test-${pv.name}", installWheels, sourceSets.javaToPython) // testJavaToPython.dependsOn testTask diff --git a/py/jpy-integration/src/javaToPython/build.gradle.template b/py/jpy-integration/src/javaToPython/build.gradle.template new file mode 100644 index 00000000000..52bc2960906 --- /dev/null +++ b/py/jpy-integration/src/javaToPython/build.gradle.template @@ -0,0 +1,18 @@ +plugins { + id 'java-library' +} + +// Classpath is already built, just need to pass to the test task +test.classpath = fileTree('/classpath/').plus(files('/classpath')) +test.testClassesDirs = files('/classes') + +test.systemProperties([ + 'jpy.jpyLib':'/usr/local/lib/python3.7/dist-packages/jpy.cpython-37m-x86_64-linux-gnu.so', + 'jpy.jdlLib':'/usr/local/lib/python3.7/dist-packages/jdl.cpython-37m-x86_64-linux-gnu.so', + 'jpy.pythonLib':'/usr/lib/x86_64-linux-gnu/libpython3.7m.so.1.0', + // Cleaning up on a dedicated thread has some issues when there is frequent starting + // and stopping of the python virtual environment. We'll rely on cleaning up inline + // when necessary. + 'PyObject.cleanup_on_thread':'false', +// 'jpy.debug':'true' +]) From 6e31778506279b93cd6c1f5bb88c72f9c8fa3e9e Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Tue, 18 May 2021 15:33:06 -0500 Subject: [PATCH 2/3] explicitly depend on Integrations for the runtime-base image --- py/jpy-integration/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py/jpy-integration/build.gradle b/py/jpy-integration/build.gradle index 885958f22f5..6269880d395 100644 --- a/py/jpy-integration/build.gradle +++ b/py/jpy-integration/build.gradle @@ -7,6 +7,7 @@ plugins { } evaluationDependsOn ':deephaven-jpy' +evaluationDependsOn ':Integrations' sourceSets { // All sourcesets here are used for testing; we'll inform the idea{} plugin, below @@ -133,6 +134,7 @@ if (!PyEnv.pythonEnabled(project)) { into 'classes' } } + parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base dockerfile { // base image with default java, python, wheels from 'deephaven/runtime-base' From a80009d5b7387215ba9c401db1a9284ae41d549d Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 21 May 2021 13:55:10 -0500 Subject: [PATCH 3/3] Add reference to #651 --- py/jpy-integration/src/javaToPython/build.gradle.template | 1 + py/jpy-integration/src/pythonToJava/jpyconfig.py.template | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/py/jpy-integration/src/javaToPython/build.gradle.template b/py/jpy-integration/src/javaToPython/build.gradle.template index 52bc2960906..3025e3821e5 100644 --- a/py/jpy-integration/src/javaToPython/build.gradle.template +++ b/py/jpy-integration/src/javaToPython/build.gradle.template @@ -13,6 +13,7 @@ test.systemProperties([ // Cleaning up on a dedicated thread has some issues when there is frequent starting // and stopping of the python virtual environment. We'll rely on cleaning up inline // when necessary. + // TODO issue #651 to see if we can remove this 'PyObject.cleanup_on_thread':'false', // 'jpy.debug':'true' ]) diff --git a/py/jpy-integration/src/pythonToJava/jpyconfig.py.template b/py/jpy-integration/src/pythonToJava/jpyconfig.py.template index 3db44b50727..3124f2e1793 100644 --- a/py/jpy-integration/src/pythonToJava/jpyconfig.py.template +++ b/py/jpy-integration/src/pythonToJava/jpyconfig.py.template @@ -2,6 +2,6 @@ java_home = '${javaHome}' jvm_dll = None jvm_maxmem = '${jvmMaxMem}' jvm_classpath = ['${jvmClasspath}'] -jvm_properties = {'PyObject.cleanup_on_thread': 'false'} +jvm_properties = {'PyObject.cleanup_on_thread': 'false'} # TODO issue #651 to see if we can remove this #jvm_options = ['-Djpy.debug=true'] jvm_options = []