From 7bc2bf50414f8daedcaac161b9f71e7f258ce16b Mon Sep 17 00:00:00 2001 From: Zacharya Haitin Date: Wed, 14 Oct 2020 11:49:17 +0300 Subject: [PATCH] Add google cloud storage support to java_export --- .gitignore | 3 +- README.md | 12 ++++++ WORKSPACE | 4 +- defs.bzl | 17 ++++++++ examples/java-export/BUILD.bazel | 1 - examples/java-export/WORKSPACE | 5 ++- .../java/rules/jvm/external/maven/BUILD.bazel | 12 +++++- .../jvm/external/maven/MavenPublisher.java | 43 +++++++++++++++++-- 8 files changed, 88 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index bd6dbb2d4..be804af5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /*.class bazel-* .ijwb -.DS_Store \ No newline at end of file +.DS_Store +.idea diff --git a/README.md b/README.md index a5c6095f8..564993a20 100644 --- a/README.md +++ b/README.md @@ -916,6 +916,14 @@ Any version conflicts or duplicate artifacts will resolved automatically. ## Publishing to External Repositories +In order to use this feature you must import the java_export Maven dependencies in your WORKSPACE file: + +```python +load("@rules_jvm_external//:defs.bzl", "java_export_deps") + +java_export_deps() +``` + In order to publish an artifact from your repo to a maven repository, you must first create a `java_export` target. This is similar to a regular `java_library`, but allows two additional parameters: the maven coordinates @@ -952,6 +960,10 @@ bazel run --stamp \ //user_project:exported_lib.publish` ``` +It's also possible to publish to a Google Cloud Storage bucket: + +`bazel run --define "maven_repo=gs://example-bucket/repository" //user_project:exported_lib.publish` + When using the `gpg_sign` option, the current default key will be used for signing, and the `gpg` binary needs to be installed on the machine. diff --git a/WORKSPACE b/WORKSPACE index 9fce81bb2..5de3ae24b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -53,7 +53,7 @@ http_archive( # Begin test dependencies -load("//:defs.bzl", "maven_install") +load("//:defs.bzl", "java_export_deps", "maven_install") load("//:specs.bzl", "maven") maven_install( @@ -67,6 +67,8 @@ maven_install( ], ) +java_export_deps() + load("@maven//:defs.bzl", "pinned_maven_install") pinned_maven_install() diff --git a/defs.bzl b/defs.bzl index e016ed977..3949e60c2 100644 --- a/defs.bzl +++ b/defs.bzl @@ -154,6 +154,23 @@ def _parse_artifact_str(artifact_str): else: return parse.parse_maven_coordinate(artifact_str) +INTERNAL_JAVA_EXPORT_REPO_NAME = "internal_java_exports_maven" + +def java_export_deps(maven_repositories = ["https://jcenter.bintray.com/"]): + """ + Adds the maven dependencies that are needed to run java_export rule. + + Args: + maven_repositories: A list of Maven repository URLs, specified in lookup order. + """ + maven_install( + name = INTERNAL_JAVA_EXPORT_REPO_NAME, + artifacts = [ + "com.google.cloud:google-cloud-storage:1.111.2", + ], + repositories = maven_repositories, + ) + java_export = _java_export javadoc = _javadoc pom_file = _pom_file diff --git a/examples/java-export/BUILD.bazel b/examples/java-export/BUILD.bazel index 2fcaecbcf..f87dea410 100644 --- a/examples/java-export/BUILD.bazel +++ b/examples/java-export/BUILD.bazel @@ -14,4 +14,3 @@ java_export( "//src/main/java/com/github/bazelbuild/rulesjvmexternal/example/export", ], ) - diff --git a/examples/java-export/WORKSPACE b/examples/java-export/WORKSPACE index 182f30661..b44922fd1 100644 --- a/examples/java-export/WORKSPACE +++ b/examples/java-export/WORKSPACE @@ -5,7 +5,7 @@ local_repository( path = "../../", ) -load("@rules_jvm_external//:defs.bzl", "maven_install") +load("@rules_jvm_external//:defs.bzl", "java_export_deps", "maven_install") protobuf_version = "3.11.3" @@ -21,6 +21,8 @@ maven_install( ], ) +java_export_deps() + load("@maven//:defs.bzl", "pinned_maven_install") pinned_maven_install() @@ -56,4 +58,3 @@ load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_ rules_proto_dependencies() rules_proto_toolchains() - diff --git a/private/tools/java/rules/jvm/external/maven/BUILD.bazel b/private/tools/java/rules/jvm/external/maven/BUILD.bazel index 37e9f5579..8a62debdf 100644 --- a/private/tools/java/rules/jvm/external/maven/BUILD.bazel +++ b/private/tools/java/rules/jvm/external/maven/BUILD.bazel @@ -1,3 +1,5 @@ +load("@rules_jvm_external//:defs.bzl", "INTERNAL_JAVA_EXPORT_REPO_NAME", "artifact") + java_binary( name = "MavenPublisher", srcs = ["MavenPublisher.java"], @@ -8,8 +10,16 @@ java_binary( "8", ], main_class = "rules.jvm.external.maven.MavenPublisher", + visibility = ["//visibility:public"], deps = [ "//private/tools/java/rules/jvm/external:byte-streams", + artifact( + "com.google.cloud:google-cloud-storage", + repository_name = INTERNAL_JAVA_EXPORT_REPO_NAME, + ), + artifact( + "com.google.cloud:google-cloud-core", + repository_name = INTERNAL_JAVA_EXPORT_REPO_NAME, + ), ], - visibility = ["//visibility:public"], ) diff --git a/private/tools/java/rules/jvm/external/maven/MavenPublisher.java b/private/tools/java/rules/jvm/external/maven/MavenPublisher.java index 08f37c8a7..0bbd87cdb 100644 --- a/private/tools/java/rules/jvm/external/maven/MavenPublisher.java +++ b/private/tools/java/rules/jvm/external/maven/MavenPublisher.java @@ -17,6 +17,10 @@ package rules.jvm.external.maven; +import com.google.cloud.WriteChannel; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; import rules.jvm.external.ByteStreams; import java.io.IOException; @@ -24,7 +28,9 @@ import java.io.OutputStream; import java.math.BigInteger; import java.net.HttpURLConnection; +import java.net.URI; import java.net.URL; +import java.nio.channels.Channels; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -50,11 +56,13 @@ public class MavenPublisher { private static final Logger LOG = Logger.getLogger(MavenPublisher.class.getName()); private static final ExecutorService EXECUTOR = Executors.newFixedThreadPool(1); + private static final String[] SUPPORTED_SCHEMES = {"file:/", "https://", "gs://"}; public static void main(String[] args) throws IOException, InterruptedException, ExecutionException, TimeoutException { String repo = args[0]; - if (!(repo.startsWith("file:/") || repo.startsWith("https://"))) { - throw new IllegalArgumentException("Repository must be accessed via file or https: " + repo); + if (!isSchemeSupported(repo)) { + throw new IllegalArgumentException("Repository must be accessed via the supported schemes: " + + Arrays.toString(SUPPORTED_SCHEMES)); } Credentials credentials = new Credentials(args[2], args[3], Boolean.parseBoolean(args[1])); @@ -85,6 +93,15 @@ public static void main(String[] args) throws IOException, InterruptedException, } } + private static boolean isSchemeSupported(String repo) { + for (String scheme : SUPPORTED_SCHEMES) { + if (repo.startsWith(scheme)) { + return true; + } + } + return false; + } + private static CompletableFuture upload( String repo, Credentials credentials, @@ -94,7 +111,7 @@ private static CompletableFuture upload( String base = String.format( "%s/%s/%s/%s/%s-%s", - repo, + repo.replaceAll("/$", ""), coords.groupId.replace('.', '/'), coords.artifactId, coords.version, @@ -144,6 +161,8 @@ private static CompletableFuture upload(String targetUrl, Credentials cred Callable callable; if (targetUrl.startsWith("http://") || targetUrl.startsWith("https://")) { callable = httpUpload(targetUrl, credentials, toUpload); + } else if (targetUrl.startsWith("gs://")) { + callable = gcsUpload(targetUrl, toUpload); } else { callable = writeFile(targetUrl, toUpload); } @@ -210,6 +229,24 @@ private static Callable writeFile(String targetUrl, Path toUpload) { }; } + private static Callable gcsUpload(String targetUrl, Path toUpload) { + return () -> { + Storage storage = StorageOptions.getDefaultInstance().getService(); + URI gsUri = new URI(targetUrl); + String bucketName = gsUri.getHost(); + String path = gsUri.getPath().substring(1); + + LOG.info(String.format("Copying %s to gs://%s/%s", toUpload, bucketName, path)); + BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, path).build(); + try (WriteChannel writer = storage.writer(blobInfo); + InputStream is = Files.newInputStream(toUpload)) { + ByteStreams.copy(is, Channels.newOutputStream(writer)); + } + + return null; + }; + } + private static Path sign(Path toSign) throws IOException, InterruptedException { LOG.info("Signing " + toSign);