diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e1f26932..d296f22a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,7 +192,7 @@ jobs: python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" python builder.pyz build -p ${{ env.PACKAGE_NAME }} downstream - macos: + macos: runs-on: macos-14 #latest steps: - name: Checkout Sources @@ -204,7 +204,7 @@ jobs: python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" chmod a+x builder ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream - python3 codebuild/macos_compatibility_check.py + python3 codebuild/macos_compatibility_check.py macos-x64: runs-on: macos-14-large #latest @@ -274,7 +274,7 @@ jobs: # check that docs can still build check-docs: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-22.04 # use same version as docs.yml steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 232062481..fc99ab2c7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,7 +9,7 @@ on: jobs: update-docs-branch: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # use same version as ci.yml's check-docs permissions: contents: write # allow push steps: @@ -17,7 +17,6 @@ jobs: uses: actions/checkout@v3 with: submodules: true - - name: Update docs branch run: | ./make-docs.sh diff --git a/README.md b/README.md index 8394455a0..ec56bfae1 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,13 @@ Platforms without FIPS compliance are also included in this jar, for compatibili > [!WARNING] > The classifier, and platforms with FIPS compliance are subject to change in the future. +## GraalVM support + +Since version v0.29.20, GraalVM native image was supported. You can compile your application with AWS CRT in a GraalVM native image project without any additional configuration. + +Since version v0.30.12, GraalVM support was updated. Instead of packaging the JNI shared lib with native image as resource, the corresponding shared lib will be written to the same directory as the native image. +In this way, it reduces the native image size around 30%, and avoids the extra loading time needed for extracting the JNI lib to the temporary path for load. No additional configuration needed. +**Note**: the JNI shared lib must be in the same directory as the GraalVM native image. If you move the native image, you must move this file too. It is `aws-crt-jni.dll` on Windows, `libaws-crt-jni.dylib` on macOS, and `libaws-crt-jni.so` on Unix. ## System Properties diff --git a/android/crt/build.gradle b/android/crt/build.gradle index 2c2f4c46c..837c93082 100644 --- a/android/crt/build.gradle +++ b/android/crt/build.gradle @@ -76,6 +76,7 @@ android { main { java.srcDir '../../src/main/java' java.srcDir 'src/main/java' + java.exclude '**/GraalVMNativeFeature.java' } androidTest { setRoot '../../src/test' diff --git a/javadoc.options b/javadoc.options deleted file mode 100644 index 5c60f19ec..000000000 --- a/javadoc.options +++ /dev/null @@ -1,12 +0,0 @@ --d docs --public --windowtitle 'AWS Common Runtime for Java/JVM' --doctitle 'AWS Common Runtime for Java/JVM' --header 'AWS Common Runtime for Java/JVM' --bottom 'Copyright © Amazon.com, Inc. or its affiliates. All Rights Reserved.' --sourcepath src/main/java --notimestamp --subpackages software.amazon.awssdk.crt --quiet --Xdoclint:all --Xwerror diff --git a/make-docs.sh b/make-docs.sh index 721dc12e6..2da708239 100755 --- a/make-docs.sh +++ b/make-docs.sh @@ -8,6 +8,9 @@ pushd $(dirname $0) > /dev/null rm -rf docs/ # build -javadoc @javadoc.options +mvn javadoc:javadoc -Dmaven.javadoc.failOnWarnings=true + +# copy to docs/ +cp -r target/site/apidocs/ docs/ popd > /dev/null diff --git a/pom.xml b/pom.xml index d059d4486..572d076ad 100644 --- a/pom.xml +++ b/pom.xml @@ -162,7 +162,7 @@ cmake-configure - generate-sources + generate-resources exec @@ -292,7 +292,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.4.0 attach-javadocs @@ -351,6 +351,13 @@ 1.4 test + + org.graalvm.sdk + graal-sdk + 21.3.11 + provided + true + @@ -484,6 +491,23 @@ 0 + + org.apache.maven.plugins + maven-javadoc-plugin + 3.4.0 + + AWS Common Runtime for Java/JVM + AWS Common Runtime for Java/JVM +
AWS Common Runtime for Java/JVM
+ Copyright © Amazon.com, Inc. or its affiliates. All Rights Reserved. + public + src/main/java + true + true + all + software.amazon.awssdk.crt.internal + + diff --git a/src/main/java/software/amazon/awssdk/crt/CRT.java b/src/main/java/software/amazon/awssdk/crt/CRT.java index eb28886a6..10f4b3e0d 100644 --- a/src/main/java/software/amazon/awssdk/crt/CRT.java +++ b/src/main/java/software/amazon/awssdk/crt/CRT.java @@ -4,16 +4,15 @@ */ package software.amazon.awssdk.crt; +import software.amazon.awssdk.crt.internal.ExtractLib; import software.amazon.awssdk.crt.io.ClientBootstrap; import software.amazon.awssdk.crt.io.EventLoopGroup; import software.amazon.awssdk.crt.io.HostResolver; import java.io.BufferedReader; import java.io.File; -import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.lang.reflect.InvocationTargetException; import java.util.*; @@ -29,7 +28,7 @@ public final class CRT { private static final String CRT_ARCH_OVERRIDE_SYSTEM_PROPERTY = "aws.crt.arch"; private static final String CRT_ARCH_OVERRIDE_ENVIRONMENT_VARIABLE = "AWS_CRT_ARCH"; - private static final String CRT_LIB_NAME = "aws-crt-jni"; + public static final String CRT_LIB_NAME = "aws-crt-jni"; public static final int AWS_CRT_SUCCESS = 0; private static final CrtPlatform s_platform; @@ -41,8 +40,15 @@ public final class CRT { // If the lib is already present/loaded or is in java.library.path, just use it System.loadLibrary(CRT_LIB_NAME); } catch (UnsatisfiedLinkError e) { - // otherwise, load from the jar this class is in - loadLibraryFromJar(); + String graalVMImageCode = System.getProperty("org.graalvm.nativeimage.imagecode"); + if (graalVMImageCode != null && graalVMImageCode == "runtime") { + throw new CrtRuntimeException( + "Failed to load '" + System.mapLibraryName(CRT_LIB_NAME) + + "'. Make sure this file is in the same directory as the GraalVM native image. "); + } else { + // otherwise, load from the jar this class is in + loadLibraryFromJar(); + } } // Initialize the CRT @@ -246,112 +252,65 @@ private static List runProcess(String[] cmdArray) throws IOException { } private static void extractAndLoadLibrary(String path) { + // Check java.io.tmpdir + String tmpdirPath; + File tmpdirFile; try { - // Check java.io.tmpdir - String tmpdirPath; - File tmpdirFile; - try { - tmpdirFile = new File(path).getAbsoluteFile(); - tmpdirPath = tmpdirFile.getAbsolutePath(); - if (tmpdirFile.exists()) { - if (!tmpdirFile.isDirectory()) { - throw new IOException("not a directory: " + tmpdirPath); - } - } else { - tmpdirFile.mkdirs(); - } - - if (!tmpdirFile.canRead() || !tmpdirFile.canWrite()) { - throw new IOException("access denied: " + tmpdirPath); - } - } catch (Exception ex) { - String msg = "Invalid directory: " + path; - throw new IOException(msg, ex); - } - - String libraryName = System.mapLibraryName(CRT_LIB_NAME); - - // Prefix the lib we'll extract to disk - String tempSharedLibPrefix = "AWSCRT_"; - - File tempSharedLib = File.createTempFile(tempSharedLibPrefix, libraryName, tmpdirFile); - if (!tempSharedLib.setExecutable(true, true)) { - throw new CrtRuntimeException("Unable to make shared library executable by owner only"); - } - if (!tempSharedLib.setWritable(true, true)) { - throw new CrtRuntimeException("Unable to make shared library writeable by owner only"); - } - if (!tempSharedLib.setReadable(true, true)) { - throw new CrtRuntimeException("Unable to make shared library readable by owner only"); - } - - // The temp lib file should be deleted when we're done with it. - // Ask Java to try and delete it on exit. We call this immediately - // so that if anything goes wrong writing the file to disk, or - // loading it as a shared lib, it will still get cleaned up. - tempSharedLib.deleteOnExit(); - - // Unfortunately File.deleteOnExit() won't work on Windows, where - // files cannot be deleted while they're in use. On Windows, once - // our .dll is loaded, it can't be deleted by this process. - // - // The Windows-only solution to this problem is to scan on startup - // for old instances of the .dll and try to delete them. If another - // process is still using the .dll, the delete will fail, which is fine. - String os = getOSIdentifier(); - if (os.equals("windows")) { - tryDeleteOldLibrariesFromTempDir(tmpdirFile, tempSharedLibPrefix, libraryName); - } - - // open a stream to read the shared lib contents from this JAR - String libResourcePath = "/" + os + "/" + getArchIdentifier() + "/" + getCRuntime(os) + "/" + libraryName; - // Check whether there is a platform specific resource path to use - CrtPlatform platform = getPlatformImpl(); - if (platform != null){ - String platformLibResourcePath = platform.getResourcePath(getCRuntime(os), libraryName); - if (platformLibResourcePath != null){ - libResourcePath = platformLibResourcePath; + tmpdirFile = new File(path).getAbsoluteFile(); + tmpdirPath = tmpdirFile.getAbsolutePath(); + if (tmpdirFile.exists()) { + if (!tmpdirFile.isDirectory()) { + throw new IOException("not a directory: " + tmpdirPath); } + } else { + tmpdirFile.mkdirs(); } - try (InputStream in = CRT.class.getResourceAsStream(libResourcePath)) { - if (in == null) { - throw new IOException("Unable to open library in jar for AWS CRT: " + libResourcePath); - } - - // Copy from jar stream to temp file - try (FileOutputStream out = new FileOutputStream(tempSharedLib)) { - int read; - byte [] bytes = new byte[1024]; - while ((read = in.read(bytes)) != -1){ - out.write(bytes, 0, read); - } - } + if (!tmpdirFile.canRead() || !tmpdirFile.canWrite()) { + throw new IOException("access denied: " + tmpdirPath); } + } catch (IOException ex) { + CrtRuntimeException rex = new CrtRuntimeException("Invalid directory: " + path); + rex.initCause(ex); + throw rex; + } - if (!tempSharedLib.setWritable(false)) { - throw new CrtRuntimeException("Unable to make shared library read-only"); - } + String libraryName = System.mapLibraryName(CRT_LIB_NAME); - // load the shared lib from the temp path - System.load(tempSharedLib.getAbsolutePath()); - } catch (CrtRuntimeException crtex) { - System.err.println("Unable to initialize AWS CRT: " + crtex); - crtex.printStackTrace(); - throw crtex; - } catch (UnknownPlatformException upe) { - System.err.println("Unable to determine platform for AWS CRT: " + upe); - upe.printStackTrace(); - CrtRuntimeException rex = new CrtRuntimeException("Unable to determine platform for AWS CRT"); - rex.initCause(upe); - throw rex; - } catch (Exception ex) { - System.err.println("Unable to unpack AWS CRT lib: " + ex); + // Prefix the lib we'll extract to disk + String tempSharedLibPrefix = "AWSCRT_"; + File tempSharedLib = null; + try{ + tempSharedLib = File.createTempFile(tempSharedLibPrefix, libraryName, tmpdirFile); + } + catch (IOException ex){ + System.err.println("Unable to create temp file to extract AWS CRT library: " + ex); ex.printStackTrace(); - CrtRuntimeException rex = new CrtRuntimeException("Unable to unpack AWS CRT library"); + CrtRuntimeException rex = new CrtRuntimeException("Unable to create temp file to extract AWS CRT library"); rex.initCause(ex); throw rex; } + // The temp lib file should be deleted when we're done with it. + // Ask Java to try and delete it on exit. We call this immediately + // so that if anything goes wrong writing the file to disk, or + // loading it as a shared lib, it will still get cleaned up. + tempSharedLib.deleteOnExit(); + + // Unfortunately File.deleteOnExit() won't work on Windows, where + // files cannot be deleted while they're in use. On Windows, once + // our .dll is loaded, it can't be deleted by this process. + // + // The Windows-only solution to this problem is to scan on startup + // for old instances of the .dll and try to delete them. If another + // process is still using the .dll, the delete will fail, which is fine. + String os = getOSIdentifier(); + if (os.equals("windows")) { + tryDeleteOldLibrariesFromTempDir(tmpdirFile, tempSharedLibPrefix, libraryName); + } + ExtractLib.extractLibrary(tempSharedLib); + // load the shared lib from the temp path + System.load(tempSharedLib.getAbsolutePath()); + } private static void loadLibraryFromJar() { diff --git a/src/main/java/software/amazon/awssdk/crt/internal/ExtractLib.java b/src/main/java/software/amazon/awssdk/crt/internal/ExtractLib.java new file mode 100644 index 000000000..69b623211 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/internal/ExtractLib.java @@ -0,0 +1,107 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package software.amazon.awssdk.crt.internal; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.CRT.UnknownPlatformException; +import software.amazon.awssdk.crt.CrtPlatform; +import software.amazon.awssdk.crt.CrtRuntimeException; + +/** + * Helper to extract JNI shared lib from Jar. + * Internal API, not for external usage. + */ +public class ExtractLib { + + /** + * Extract the CRT JNI library on current platform to a specific File + * + * @param extractFile the File extracting to + */ + public static void extractLibrary(File extractFile) { + + try { + if (!extractFile.setExecutable(true, true)) { + throw new CrtRuntimeException("Unable to make shared library executable by owner only"); + } + if (!extractFile.setWritable(true, true)) { + throw new CrtRuntimeException("Unable to make shared library writeable by owner only"); + } + if (!extractFile.setReadable(true, true)) { + throw new CrtRuntimeException("Unable to make shared library readable by owner only"); + } + String libraryName = System.mapLibraryName(CRT.CRT_LIB_NAME); + String os = CRT.getOSIdentifier(); + // open a stream to read the shared lib contents from this JAR + String libResourcePath = "/" + os + "/" + CRT.getArchIdentifier() + "/" + CRT.getCRuntime(os) + "/" + + libraryName; + // Check whether there is a platform specific resource path to use + CrtPlatform platform = CRT.getPlatformImpl(); + if (platform != null) { + String platformLibResourcePath = platform.getResourcePath(CRT.getCRuntime(os), libraryName); + if (platformLibResourcePath != null) { + libResourcePath = platformLibResourcePath; + } + } + try (InputStream in = CRT.class.getResourceAsStream(libResourcePath)) { + if (in == null) { + throw new IOException("Unable to open library in jar for AWS CRT: " + libResourcePath); + } + + // Copy from jar stream to temp file + try (FileOutputStream out = new FileOutputStream(extractFile)) { + int read; + byte[] bytes = new byte[1024]; + while ((read = in.read(bytes)) != -1) { + out.write(bytes, 0, read); + } + } + } + if (!extractFile.setWritable(false)) { + throw new CrtRuntimeException("Unable to make shared library read-only"); + } + } catch (CrtRuntimeException crtex) { + System.err.println("Unable to initialize AWS CRT: " + crtex); + crtex.printStackTrace(); + throw crtex; + } catch (UnknownPlatformException upe) { + System.err.println("Unable to determine platform for AWS CRT: " + upe); + upe.printStackTrace(); + CrtRuntimeException rex = new CrtRuntimeException("Unable to determine platform for AWS CRT"); + rex.initCause(upe); + throw rex; + } catch (Exception ex) { + System.err.println("Unable to unpack AWS CRT lib: " + ex); + ex.printStackTrace(); + CrtRuntimeException rex = new CrtRuntimeException("Unable to unpack AWS CRT library"); + rex.initCause(ex); + throw rex; + } + } + + /** + * Extract the CRT JNI library on current platform to a specific path. + * + * @param path the path extracting to + */ + public static void extractLibrary(String path) { + String libraryName = System.mapLibraryName(CRT.CRT_LIB_NAME); + File extractFile = new File(path, libraryName); + try { + extractFile.createNewFile(); + } catch (Exception ex) { + CrtRuntimeException rex = new CrtRuntimeException( + "Unable to create file on path:" + extractFile.getAbsolutePath()); + rex.initCause(ex); + throw rex; + } + extractLibrary(extractFile); + } +} diff --git a/src/main/java/software/amazon/awssdk/crt/internal/GraalVMNativeFeature.java b/src/main/java/software/amazon/awssdk/crt/internal/GraalVMNativeFeature.java new file mode 100644 index 000000000..b1e5c3b28 --- /dev/null +++ b/src/main/java/software/amazon/awssdk/crt/internal/GraalVMNativeFeature.java @@ -0,0 +1,26 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +package software.amazon.awssdk.crt.internal; + +import org.graalvm.nativeimage.hosted.Feature; + +import software.amazon.awssdk.crt.CRT; + +/** + * Implementation of GraalVM feature to extract the share lib to the image path. + * From GraalVM docs: + * > When loading native libraries using System.loadLibrary() (and related APIs), + * > the native image will search the directory containing the native image before searching the Java library path + * https://www.graalvm.org/latest/reference-manual/native-image/dynamic-features/JNI/#loading-native-libraries + * Internal API, not for external usage. + */ +public class GraalVMNativeFeature implements Feature { + + @Override + public void afterImageWrite(AfterImageWriteAccess access) { + new CRT(); + ExtractLib.extractLibrary(access.getImagePath().getParent().toString()); + } +} diff --git a/src/main/resources/META-INF/native-image/software.amazon.awssdk/crt/aws-crt/native-image.properties b/src/main/resources/META-INF/native-image/software.amazon.awssdk/crt/aws-crt/native-image.properties new file mode 100644 index 000000000..19e1a2989 --- /dev/null +++ b/src/main/resources/META-INF/native-image/software.amazon.awssdk/crt/aws-crt/native-image.properties @@ -0,0 +1 @@ +Args=--enable-url-protocols=jar --features=software.amazon.awssdk.crt.internal.GraalVMNativeFeature diff --git a/src/main/resources/META-INF/native-image/software.amazon.awssdk/crt/aws-crt/resource-config.json b/src/main/resources/META-INF/native-image/software.amazon.awssdk/crt/aws-crt/resource-config.json deleted file mode 100644 index 7fefd172c..000000000 --- a/src/main/resources/META-INF/native-image/software.amazon.awssdk/crt/aws-crt/resource-config.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "resources": { - "includes": [ - { - "pattern": ".*libaws-crt-jni.dylib" - }, - { - "pattern": ".*libaws-crt-jni.so" - }, - { - "pattern": ".*aws-crt-jni.dll" - } - ] - }, - "bundles": [] -} \ No newline at end of file