diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 567edda161187..80e4d33325acb 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -36,7 +36,7 @@ bintray { name = rootProject.name userOrg = 'palantir' licenses = ['Apache-2.0'] - publications = ['nebula'] + publications = ['nebula', 'dockerBundle'] } } diff --git a/settings.gradle b/settings.gradle index ddb71100a5d95..31b017a35e50f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,3 +19,4 @@ rootProject.name = 'spark' include ':spark-docker-image-generator' +enableFeaturePreview('STABLE_PUBLISHING') diff --git a/spark-docker-image-generator/build.gradle b/spark-docker-image-generator/build.gradle index 623cf8fcdcffa..38df4f4e252b3 100644 --- a/spark-docker-image-generator/build.gradle +++ b/spark-docker-image-generator/build.gradle @@ -41,7 +41,7 @@ test { maxHeapSize = "512m" } -task prepareDockerBundleDir(type: Sync) { +task prepareDockerBundleDir(type: Zip) { from("${rootDir}/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile") { into 'kubernetes/dockerfiles/spark' rename 'Dockerfile', 'Dockerfile.original' @@ -57,9 +57,33 @@ task prepareDockerBundleDir(type: Sync) { from(fileTree("${rootDir}/sbin")) { into 'sbin' } - into file("src/main/resources/docker-resources") + destinationDir file("${buildDir}/docker-resources") + archiveName "docker-resources.zip" includeEmptyDirs = false } tasks.compileJava.dependsOn tasks.prepareDockerBundleDir tasks.idea.dependsOn tasks.prepareDockerBundleDir + +publishing { + publications { + dockerBundle(MavenPublication) { + artifactId 'spark-docker-resources' + artifact tasks.prepareDockerBundleDir + } + } + + repositories { + maven { + name = "mavenTest" + url = "${buildDir}/mavenTest" + } + } +} + +test { + systemProperty 'docker-resources-zip-path', "${buildDir}/docker-resources/docker-resources.zip" +} + +tasks.test.dependsOn tasks.publishDockerBundlePublicationToMavenTestRepository +tasks.test.environment 'MAVEN_REPO', "${buildDir.getAbsolutePath()}/mavenTest" diff --git a/spark-docker-image-generator/plugin-test-project/build.gradle b/spark-docker-image-generator/plugin-test-project/build.gradle index 94a0058431498..15406a859940c 100644 --- a/spark-docker-image-generator/plugin-test-project/build.gradle +++ b/spark-docker-image-generator/plugin-test-project/build.gradle @@ -29,6 +29,7 @@ plugins { repositories { jcenter() maven { url "http://palantir.bintray.com/releases" } + maven { url System.env.MAVEN_REPO } } apply plugin: 'java' diff --git a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java index 640101e9d4e0d..7523097df83b5 100644 --- a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java +++ b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java @@ -17,11 +17,11 @@ package org.apache.spark.deploy.kubernetes.docker.gradle; import java.io.File; -import java.net.URL; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -77,10 +77,13 @@ public void apply(Project project) { sparkAppJar) .into(jarsDirProvider)); copySparkAppLibTask.dependsOn(jarTask); - URL dockerResourcesUrl = getClass().getResource("/docker-resources"); + String version = Optional.ofNullable(getClass().getPackage().getImplementationVersion()) + .orElse("latest.release"); + Configuration dockerResourcesConf = project.getConfigurations().detachedConfiguration( + project.getDependencies().create("org.apache.spark:spark-docker-resources:" + version)); Sync deployScriptsTask = project.getTasks().create( "sparkDockerDeployScripts", Sync.class, task -> { - task.from(project.fileTree(dockerResourcesUrl)); + task.from(project.zipTree(dockerResourcesConf.getSingleFile())); task.setIncludeEmptyDirs(false); task.into(dockerBuildDirectory); }); diff --git a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java index bfb0281c67714..6b94fcefdafba 100644 --- a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java +++ b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java @@ -18,6 +18,7 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -26,6 +27,8 @@ import java.nio.file.Files; import java.util.List; import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.io.IOUtils; import org.assertj.core.api.Assertions; @@ -37,7 +40,6 @@ import org.mockito.Mockito; public final class GenerateDockerFileTaskSuite { - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -49,10 +51,24 @@ public void before() throws IOException { File dockerFileDir = tempFolder.newFolder("docker"); destDockerFile = new File(dockerFileDir, "Dockerfile"); srcDockerFile = tempFolder.newFile("Dockerfile.original"); - try (InputStream originalDockerFileResource = getClass().getResourceAsStream( - "/docker-resources/kubernetes/dockerfiles/spark/Dockerfile.original"); - FileOutputStream srcDockerFileStream = new FileOutputStream(srcDockerFile)) { - IOUtils.copy(originalDockerFileResource, srcDockerFileStream); + + try (InputStream originalDockerBundleZipped = new FileInputStream( + new File(System.getProperty("docker-resources-zip-path"))); + ZipInputStream unzipped = new ZipInputStream(originalDockerBundleZipped); + FileOutputStream srcDockerFileStream = new FileOutputStream(srcDockerFile)) { + ZipEntry currentEntry = unzipped.getNextEntry(); + boolean foundDockerFile = false; + while (currentEntry != null && !foundDockerFile) { + if (currentEntry.getName().equals("kubernetes/dockerfiles/spark/Dockerfile.original")) { + IOUtils.copy(unzipped, srcDockerFileStream); + foundDockerFile = true; + } else { + currentEntry = unzipped.getNextEntry(); + } + } + if (!foundDockerFile) { + throw new IllegalStateException("Dockerfile not found."); + } } } @@ -73,7 +89,7 @@ public void testGenerateDockerFile() throws IOException { getClass().getResourceAsStream("/ExpectedDockerfile"); InputStreamReader expectedDockerFileReader = new InputStreamReader(expectedDockerFileInput, StandardCharsets.UTF_8); - BufferedReader expectedDockerFileBuffered = + BufferedReader expectedDockerFileBuffered = new BufferedReader(expectedDockerFileReader)) { List expectedFileLines = expectedDockerFileBuffered .lines()