Use wheels directly in the build process

Integrations/build.gradle

@@ -65,34 +65,9 @@ idea {
         }
 }
 
-/**
- * Produce a docker image with the prereqs to run and use deephaven python
- */
-def deephavenPythonDockerfile = tasks.register('deephavenPythonDockerfile', Dockerfile) {
-    destFile.set layout.buildDirectory.file('deephaven-python-docker/Dockerfile')
-    from new Dockerfile.From('deephaven/deephaven-jpy-wheel:local-build').withStage('deephaven-jpy-wheel')
-    from new Dockerfile.From('deephaven/deephaven-wheel:local-build').withStage('deephaven-wheel')
-    from new Dockerfile.From('deephaven/deephaven2-wheel:local-build').withStage('deephaven2-wheel')
-    from new Dockerfile.From('deephaven/java-and-python:local-build')
-
-    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven-jpy-wheel'))
-    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven-wheel'))
-    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven2-wheel'))
-
-    // liblzo2-2 below is needed for supporting LZO parquet encoding.
-    runCommand('''set -eux; \\
-                  apt update; \\
-                  apt -y install liblzo2-2; \\
-                  apt -y install curl; \\
-                  pip3 install setuptools wheel; \\
-                  pip3 install *.whl; \\
-                  rm *.whl
-                  ''')
-}
-Docker.registerDockerImage(project, 'buildDeephavenPython') {
-    // deephaven/java-and-python:local-build
-    def javaAndPython = project(':deephaven-jpy').tasks.getByName('buildDockerForRuntime')
+def dockerContext = project.layout.buildDirectory.dir('context')
 
+def prepareDocker = project.tasks.register('prepareDocker', Sync) {
     // deephaven-jpy.whl
     def deephavenJpyWheel = project(':deephaven-jpy').tasks.getByName('buildDockerForWheel')
 
@@ -102,16 +77,29 @@ Docker.registerDockerImage(project, 'buildDeephavenPython') {
     // deephaven2.whl
     def deephaven2Wheel = project(':deephaven2-wheel').tasks.getByName('buildWheel')
 
-    // Dockerfile
-    def dockerfile = deephavenPythonDockerfile.get()
+    from (deephavenJpyWheel.outputs.files) {
+        into 'deephaven-jpy-wheel'
+    }
+    from (deephavenWheel.outputs.files) {
+        into 'deephaven-wheel'
+    }
+    from (deephaven2Wheel.outputs.files) {
+        into 'deephaven2-wheel'
+    }
+
+    from 'src/main/docker'
+    into dockerContext
+}
+
+Docker.registerDockerImage(project, 'buildDeephavenPython') {
+    // deephaven/java-and-python:local-build
+    def javaAndPython = project(':deephaven-jpy').tasks.getByName('buildDockerForRuntime')
+
+    dependsOn prepareDocker
 
-    inputs.files javaAndPython.outputs.files,
-            deephavenJpyWheel.outputs.files,
-            deephavenWheel.outputs.files,
-            deephaven2Wheel.outputs.files,
-            dockerfile.outputs.files
+    inputs.files javaAndPython.outputs.files
 
-    inputDir.set layout.buildDirectory.dir('deephaven-python-docker')
+    inputDir.set dockerContext
 
     images.add('deephaven/runtime-base:local-build')
 }

Integrations/src/main/docker/Dockerfile

@@ -0,0 +1,24 @@
+FROM deephaven/java-and-python:local-build as install-wheels
+
+RUN set -eux; \
+    apt-get -qq update; \
+    apt-get -qq -y --no-install-recommends install liblzo2-2 curl; \
+    rm -rf /var/lib/apt/lists/*
+
+COPY deephaven-jpy-wheel/ /deephaven-jpy-wheel
+RUN set -eux; \
+    python3 -m pip install -q --no-cache-dir /deephaven-jpy-wheel/*.whl; \
+    rm -r /deephaven-jpy-wheel
+
+COPY deephaven-wheel/ /deephaven-wheel
+RUN set -eux; \
+    python3 -m pip install -q --no-cache-dir /deephaven-wheel/*.whl; \
+    rm -r /deephaven-wheel
+
+COPY deephaven2-wheel/ /deephaven2-wheel
+RUN set -eux; \
+    python3 -m pip install -q --no-cache-dir /deephaven2-wheel/*.whl; \
+    rm -r /deephaven2-wheel
+
+FROM deephaven/java-and-python:local-build
+COPY --from=install-wheels / /

buildSrc/src/main/groovy/Docker.groovy

@@ -149,6 +149,11 @@ class Docker {
          */
         List<String> entrypoint;
 
+        /**
+         * Optional build arguments
+         */
+        Map<String, String> buildArgs;
+
         /**
          * Logs are always printed from the build task when it runs, but entrypoint logs are only printed
          * when it fails. Set this flag to always show logs, even when entrypoint is successful.
@@ -256,6 +261,11 @@ class Docker {
                 if (cfg.imageName) {
                     images.add(cfg.imageName)
                 }
+
+                // add build arguments, if provided
+                if (cfg.buildArgs) {
+                    buildArgs.putAll(cfg.buildArgs)
+                }
             }
         }
 

gradle/deephaven-jpy.gradle

@@ -24,24 +24,31 @@ project(':deephaven-jpy') {
   // For OSS, we need to currently move fast, and will prefer to make edits directly in py/jpy.
   // These can be backported to jpy proper in the future if necessary.
 
-  task prepareDocker(type: Sync) {
+  def prepareDocker = project.tasks.register('prepareDocker', Sync) {
     from(project.projectDir) {
       exclude 'build'
     }
     into 'build/docker'
   }
+
   Docker.registerDockerImage(project, 'buildDockerForRuntime') {
     dependsOn prepareDocker
     buildArgs.put('DEEPHAVEN_VERSION', "${project.version}")
     images.add('deephaven/java-and-python:local-build')
     target.set('runtime_reqs')
   }
 
-  Docker.registerDockerImage(project, 'buildDockerForWheel') {
-    dependsOn prepareDocker
-    buildArgs.put('DEEPHAVEN_VERSION', "${project.version}")
-    images.add('deephaven/deephaven-jpy-wheel:local-build')
-
-    target.set('build')
+  Docker.registerDockerTask(project, 'buildDockerForWheel') {
+    copyIn {
+      from(project.projectDir) {
+        exclude 'build'
+      }
+    }
+    buildArgs = [ DEEPHAVEN_VERSION: project.version.toString() ]
+    imageName = 'deephaven/deephaven-jpy-wheel:local-build'
+    containerOutPath = '/usr/src/app/dist'
+    copyOut {
+      into 'build/deephaven-jpy-wheel'
+    }
   }
 }

py/jpy/Dockerfile

@@ -51,14 +51,8 @@ FROM build_reqs as sources
 WORKDIR /usr/src/app
 COPY . .
 
-FROM sources as build
+FROM sources
 ARG DEEPHAVEN_VERSION
 RUN set -eux; \
     test -n "${DEEPHAVEN_VERSION}"; \
     python3.7 setup.py bdist_wheel
-
-FROM runtime_reqs
-COPY --from=build /usr/src/app/dist/ .
-RUN set -eux; \
-    pip3 install *.whl; \
-    rm *.whl

sphinx/sphinx.gradle

@@ -89,7 +89,9 @@ def cppClientDoxygenTask = Docker.registerDockerTask(project, 'cppClientDoxygen'
         from 'deephaven/runtime-base:local-build'
 
         runCommand('''set -eux; \\
-                      apt-get -y install doxygen graphviz
+                      apt-get update; \\
+                      apt-get -y install doxygen graphviz; \\
+                      rm -rf /var/lib/apt/lists/*
                       ''')
 
         copyFile('.', '/project')