From 398e3c19550c85f53d30309cc36e171555094d60 Mon Sep 17 00:00:00 2001 From: Bernhard Urban-Forster Date: Wed, 5 Jan 2022 16:26:56 +0100 Subject: [PATCH 1/2] darwin-aarch64: reg dump formatting --- .../posix/aarch64/AArch64DarwinUContextRegisterDumper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/aarch64/AArch64DarwinUContextRegisterDumper.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/aarch64/AArch64DarwinUContextRegisterDumper.java index 7ad6e84bb096..0b520aebf8f6 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/aarch64/AArch64DarwinUContextRegisterDumper.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/aarch64/AArch64DarwinUContextRegisterDumper.java @@ -91,8 +91,8 @@ public void dumpRegisters(Log log, Signal.ucontext_t uContext, boolean printLoca dumpReg(log, "R28 ", regs.read(28), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); dumpReg(log, "R29 ", sigcontext.fp(), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); dumpReg(log, "R30 ", sigcontext.lr(), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); - dumpReg(log, "SP ", sigcontext.sp(), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); - dumpReg(log, "PC ", sigcontext.pc(), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); + dumpReg(log, "SP ", sigcontext.sp(), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); + dumpReg(log, "PC ", sigcontext.pc(), printLocationInfo, allowJavaHeapAccess, allowUnsafeOperations); } @Override From 04391068ec4921bbe73d3c373441e69af8a287cc Mon Sep 17 00:00:00 2001 From: Bernhard Urban-Forster Date: Tue, 15 Feb 2022 17:13:01 +0100 Subject: [PATCH 2/2] darwin-aarch64: add support for MAP_JIT/pthread_jit_write_protect_np --- .../oracle/svm/core/genscavenge/GCImpl.java | 51 +++++----- .../posix/PosixVirtualMemoryProvider.java | 22 ++++- .../oracle/svm/core/posix/headers/Mman.java | 6 ++ .../posix/headers/darwin/DarwinPthread.java | 6 ++ .../windows/WindowsVirtualMemoryProvider.java | 2 +- .../svm/core/code/RuntimeCodeInfoAccess.java | 31 ++++++ .../svm/core/code/RuntimeCodeInfoMemory.java | 25 +++-- .../os/AbstractCommittedMemoryProvider.java | 94 +++++++++++++++++++ .../core/os/OSCommittedMemoryProvider.java | 93 ------------------ .../svm/core/os/VirtualMemoryProvider.java | 23 ++++- .../IsolatedRuntimeCodeInstaller.java | 9 +- .../svm/graal/meta/RuntimeCodeInstaller.java | 91 +++++++++--------- 12 files changed, 279 insertions(+), 174 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index db2af5ae1539..911532d3e0cd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -788,33 +788,40 @@ private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list) @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) private void blackenStackRoots() { Timer blackenStackRootsTimer = timers.blackenStackRoots.open(); + RuntimeCodeInfoAccess.acquireThreadWriteAccess(); try { - Pointer sp = readCallerStackPointer(); - CodePointer ip = readReturnAddress(); + try { + Pointer sp = readCallerStackPointer(); + CodePointer ip = readReturnAddress(); - JavaStackWalk walk = StackValue.get(JavaStackWalk.class); - JavaStackWalker.initWalk(walk, sp, ip); - walkStack(walk); + JavaStackWalk walk = StackValue.get(JavaStackWalk.class); + JavaStackWalker.initWalk(walk, sp, ip); + walkStack(walk); - if (SubstrateOptions.MultiThreaded.getValue()) { - /* - * Scan the stacks of all the threads. Other threads will be blocked at a safepoint - * (or in native code) so they will each have a JavaFrameAnchor in their VMThread. - */ - for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { - if (vmThread == CurrentIsolate.getCurrentThread()) { - /* - * The current thread is already scanned by code above, so we do not have to - * do anything for it here. It might have a JavaFrameAnchor from earlier - * Java-to-C transitions, but certainly not at the top of the stack since it - * is running this code, so just this scan would be incomplete. - */ - continue; - } - if (JavaStackWalker.initWalk(walk, vmThread)) { - walkStack(walk); + if (SubstrateOptions.MultiThreaded.getValue()) { + /* + * Scan the stacks of all the threads. Other threads will be blocked at a + * safepoint (or in native code) so they will each have a JavaFrameAnchor in + * their VMThread. + */ + for (IsolateThread vmThread = VMThreads.firstThread(); vmThread.isNonNull(); vmThread = VMThreads.nextThread(vmThread)) { + if (vmThread == CurrentIsolate.getCurrentThread()) { + /* + * The current thread is already scanned by code above, so we do not + * have to do anything for it here. It might have a JavaFrameAnchor from + * earlier Java-to-C transitions, but certainly not at the top of the + * stack since it is running this code, so just this scan would be + * incomplete. + */ + continue; + } + if (JavaStackWalker.initWalk(walk, vmThread)) { + walkStack(walk); + } } } + } finally { + RuntimeCodeInfoAccess.releaseThreadWriteAccess(); } } finally { blackenStackRootsTimer.close(); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java index 572900b32ec1..09b93a8da6ea 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java @@ -27,6 +27,7 @@ import static com.oracle.svm.core.posix.headers.Mman.MAP_ANON; import static com.oracle.svm.core.posix.headers.Mman.MAP_FAILED; import static com.oracle.svm.core.posix.headers.Mman.MAP_FIXED; +import static com.oracle.svm.core.posix.headers.Mman.MAP_JIT; import static com.oracle.svm.core.posix.headers.Mman.MAP_NORESERVE; import static com.oracle.svm.core.posix.headers.Mman.MAP_PRIVATE; import static com.oracle.svm.core.posix.headers.Mman.PROT_EXEC; @@ -38,8 +39,10 @@ import static com.oracle.svm.core.posix.headers.Mman.NoTransitions.munmap; import static org.graalvm.word.WordFactory.nullPointer; +import com.oracle.svm.core.posix.headers.darwin.DarwinPthread; import org.graalvm.compiler.word.Word; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; @@ -107,7 +110,7 @@ public UnsignedWord getGranularity() { @Override @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) { + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } @@ -119,7 +122,11 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) { mappingSize = mappingSize.add(alignment); } mappingSize = UnsignedUtils.roundUp(mappingSize, granularity); - Pointer mappingBegin = mmap(nullPointer(), mappingSize, PROT_NONE(), MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET); + int flags = MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(); + if (Platform.includedIn(Platform.DARWIN_AARCH64.class) && executable) { + flags |= MAP_JIT(); + } + Pointer mappingBegin = mmap(nullPointer(), mappingSize, PROT_NONE(), flags, NO_FD, NO_FD_OFFSET); if (mappingBegin.equal(MAP_FAILED())) { return nullPointer(); } @@ -167,6 +174,11 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) { if (start.isNonNull()) { flags |= MAP_FIXED(); } + + boolean isWX = (access & Access.WRITE) != 0 && (access & Access.EXECUTE) != 0; + if (Platform.includedIn(Platform.DARWIN_AARCH64.class) && isWX) { + flags |= MAP_JIT(); + } /* The memory returned by mmap is guaranteed to be zeroed. */ final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET); return result.notEqual(MAP_FAILED()) ? result : nullPointer(); @@ -182,6 +194,12 @@ public int protect(PointerBase start, UnsignedWord nbytes, int access) { return mprotect(start, nbytes, accessAsProt(access)); } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void jitWriteProtect(boolean protect) { + DarwinPthread.pthread_jit_write_protect_np(protect ? 1 : 0); + } + @Override @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true) public int uncommit(PointerBase start, UnsignedWord nbytes) { diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Mman.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Mman.java index ce5b3b9abb7f..804036e35ce5 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Mman.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Mman.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.posix.headers; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.CContext; import org.graalvm.nativeimage.c.constant.CConstant; import org.graalvm.nativeimage.c.function.CFunction; @@ -67,6 +69,10 @@ public class Mman { @CConstant public static native int MAP_NORESERVE(); + @CConstant + @Platforms(Platform.DARWIN_AARCH64.class) + public static native int MAP_JIT(); + @CConstant public static native PointerBase MAP_FAILED(); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinPthread.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinPthread.java index 1109dfd322a4..e14fb2a353ba 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinPthread.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinPthread.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.posix.headers.darwin; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.CContext; import org.graalvm.nativeimage.c.function.CFunction; import org.graalvm.nativeimage.c.function.CFunction.Transition; @@ -49,4 +51,8 @@ public class DarwinPthread { @CFunction(transition = Transition.NO_TRANSITION) public static native Pointer pthread_get_stackaddr_np(Pthread.pthread_t thread); + + @Platforms(Platform.DARWIN_AARCH64.class) + @CFunction(transition = Transition.NO_TRANSITION) + public static native void pthread_jit_write_protect_np(int enabled); } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java index d0175dfaaddb..d0312aed9b5b 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java @@ -134,7 +134,7 @@ private static int accessAsProt(int access) { @Override @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true) - public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) { + public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { if (nbytes.equal(0)) { return WordFactory.nullPointer(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index a6c2ad8cc223..3bf5ae9eff01 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -26,8 +26,13 @@ import java.util.EnumSet; +import com.oracle.svm.core.os.VirtualMemoryProvider; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalInt; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -248,6 +253,32 @@ public static void makeCodeMemoryWriteableNonExecutable(CodePointer start, Unsig CommittedMemoryProvider.get().protect(start, size, EnumSet.of(CommittedMemoryProvider.Access.READ, CommittedMemoryProvider.Access.WRITE)); } + @Platforms(Platform.DARWIN_AARCH64.class) private static final FastThreadLocalInt jitProtectDepth = FastThreadLocalFactory.createInt("jitProtectDepth"); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void acquireThreadWriteAccess() { + if (Platform.includedIn(Platform.DARWIN_AARCH64.class)) { + // Disabling write protection can be nested, for example a GC can be triggered during + // code installation which in turn causes walk of references in code. Both need to + // disable write protection, but only the outer one should enable it again. + if (jitProtectDepth.get() == 0) { + VirtualMemoryProvider.get().jitWriteProtect(false); + } + jitProtectDepth.set(jitProtectDepth.get() + 1); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void releaseThreadWriteAccess() { + if (Platform.includedIn(Platform.DARWIN_AARCH64.class)) { + VMError.guarantee(jitProtectDepth.get() >= 1); + jitProtectDepth.set(jitProtectDepth.get() - 1); + if (jitProtectDepth.get() == 0) { + VirtualMemoryProvider.get().jitWriteProtect(true); + } + } + } + @Uninterruptible(reason = "Called from uninterruptible code", mayBeInlined = true) static void releaseMethodInfoOnTearDown(CodeInfo info) { InstalledCodeObserverSupport.removeObserversOnTearDown(getCodeObserverHandles(info)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index 90267d846d04..33821ea68681 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -203,18 +203,23 @@ public boolean walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { assert VMOperation.isGCInProgress() : "otherwise, we would need to make sure that the CodeInfo is not freeded by the GC"; if (table.isNonNull()) { int length = NonmovableArrays.lengthOf(table); - for (int i = 0; i < length;) { - UntetheredCodeInfo info = NonmovableArrays.getWord(table, i); - if (info.isNonNull()) { - visitor.visitCode(CodeInfoAccess.convert(info)); - } + RuntimeCodeInfoAccess.acquireThreadWriteAccess(); + try { + for (int i = 0; i < length;) { + UntetheredCodeInfo info = NonmovableArrays.getWord(table, i); + if (info.isNonNull()) { + visitor.visitCode(CodeInfoAccess.convert(info)); + } - // If the visitor removed the current entry from the table, then it is necessary to - // visit the now updated entry one more time. However, this could have the effect - // that some entries are visited more than once. - if (info == NonmovableArrays.getWord(table, i)) { - i++; + // If the visitor removed the current entry from the table, then it is necessary + // to visit the now updated entry one more time. However, this could have the + // effect that some entries are visited more than once. + if (info == NonmovableArrays.getWord(table, i)) { + i++; + } } + } finally { + RuntimeCodeInfoAccess.releaseThreadWriteAccess(); } } return true; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index 3acca7f3f2dd..3e22081c4c92 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -28,9 +28,11 @@ import static com.oracle.svm.core.Isolates.IMAGE_HEAP_END; import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_BEGIN; import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_END; +import static org.graalvm.word.WordFactory.nullPointer; import java.util.EnumSet; +import com.oracle.svm.core.util.UnsignedUtils; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; @@ -40,6 +42,7 @@ import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.heap.Heap; +import org.graalvm.word.WordFactory; public abstract class AbstractCommittedMemoryProvider implements CommittedMemoryProvider { @Fold @@ -94,4 +97,95 @@ public boolean protect(PointerBase start, UnsignedWord nbytes, EnumSet a int success = VirtualMemoryProvider.get().protect(start, nbytes, vmAccessBits); return success == 0; } + + @Override + public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { + return allocate(nbytes, alignment, false); + } + + @Override + public Pointer allocateUnalignedChunk(UnsignedWord nbytes) { + return allocate(nbytes, WordFactory.unsigned(1), false); + } + + @Override + public Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment) { + return allocate(nbytes, alignment, true); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) { + int access = VirtualMemoryProvider.Access.READ | VirtualMemoryProvider.Access.WRITE; + if (executable) { + access |= VirtualMemoryProvider.Access.EXECUTE; + } + Pointer reserved = WordFactory.nullPointer(); + if (!UnsignedUtils.isAMultiple(getGranularity(), alignment)) { + reserved = VirtualMemoryProvider.get().reserve(size, alignment, executable); + if (reserved.isNull()) { + return nullPointer(); + } + } + Pointer committed = VirtualMemoryProvider.get().commit(reserved, size, access); + if (committed.isNull()) { + if (reserved.isNonNull()) { + VirtualMemoryProvider.get().free(reserved, size); + } + return nullPointer(); + } + assert reserved.isNull() || reserved.equal(committed); + tracker.track(size); + return committed; + } + + @Override + public boolean areUnalignedChunksZeroed() { + return false; + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { + free(start, nbytes); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes) { + free(start, nbytes); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { + free(start, nbytes); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void free(PointerBase start, UnsignedWord nbytes) { + if (VirtualMemoryProvider.get().free(start, nbytes) == 0) { + tracker.untrack(nbytes); + } + } + + private final VirtualMemoryTracker tracker = new VirtualMemoryTracker(); + + public static class VirtualMemoryTracker { + + private UnsignedWord totalAllocated; + + public VirtualMemoryTracker() { + this.totalAllocated = WordFactory.zero(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void track(UnsignedWord size) { + totalAllocated = totalAllocated.add(size); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void untrack(UnsignedWord size) { + totalAllocated = totalAllocated.subtract(size); + } + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java index 4c440d13a0ba..080ca4ae3839 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/OSCommittedMemoryProvider.java @@ -33,10 +33,7 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.hosted.Feature; -import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; -import org.graalvm.word.UnsignedWord; -import org.graalvm.word.WordFactory; import com.oracle.svm.core.Isolates; import com.oracle.svm.core.SubstrateOptions; @@ -45,7 +42,6 @@ import com.oracle.svm.core.c.function.CEntryPointCreateIsolateParameters; import com.oracle.svm.core.c.function.CEntryPointErrors; import com.oracle.svm.core.c.function.CEntryPointSetup; -import com.oracle.svm.core.util.UnsignedUtils; public class OSCommittedMemoryProvider extends AbstractCommittedMemoryProvider { @Platforms(Platform.HOSTED_ONLY.class) @@ -75,95 +71,6 @@ public int tearDown() { PointerBase heapBase = Isolates.getHeapBase(CurrentIsolate.getIsolate()); return ImageHeapProvider.get().freeImageHeap(heapBase); } - - @Override - public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, false); - } - - @Override - public Pointer allocateUnalignedChunk(UnsignedWord nbytes) { - return allocate(nbytes, WordFactory.unsigned(1), false); - } - - @Override - public Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment) { - return allocate(nbytes, alignment, true); - } - - private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) { - int access = VirtualMemoryProvider.Access.READ | VirtualMemoryProvider.Access.WRITE; - if (executable) { - access |= VirtualMemoryProvider.Access.EXECUTE; - } - Pointer reserved = WordFactory.nullPointer(); - if (!UnsignedUtils.isAMultiple(getGranularity(), alignment)) { - reserved = VirtualMemoryProvider.get().reserve(size, alignment); - if (reserved.isNull()) { - return nullPointer(); - } - } - Pointer committed = VirtualMemoryProvider.get().commit(reserved, size, access); - if (committed.isNull()) { - if (reserved.isNonNull()) { - VirtualMemoryProvider.get().free(reserved, size); - } - return nullPointer(); - } - assert reserved.isNull() || reserved.equal(committed); - tracker.track(size); - return committed; - } - - @Override - public boolean areUnalignedChunksZeroed() { - return false; - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { - free(start, nbytes); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes) { - free(start, nbytes); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) { - free(start, nbytes); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private void free(PointerBase start, UnsignedWord nbytes) { - if (VirtualMemoryProvider.get().free(start, nbytes) == 0) { - tracker.untrack(nbytes); - } - } - - private final VirtualMemoryTracker tracker = new VirtualMemoryTracker(); - - protected static class VirtualMemoryTracker { - - private UnsignedWord totalAllocated; - - protected VirtualMemoryTracker() { - this.totalAllocated = WordFactory.zero(); - } - - public void track(UnsignedWord size) { - totalAllocated = totalAllocated.add(size); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public void untrack(UnsignedWord size) { - totalAllocated = totalAllocated.subtract(size); - } - } } @AutomaticFeature diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java index 5105df083780..cc7865fb7f92 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/VirtualMemoryProvider.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.os; +import com.oracle.svm.core.annotate.Uninterruptible; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.word.Pointer; @@ -83,10 +84,19 @@ default UnsignedWord getAlignment() { * to a multiple of the {@linkplain #getGranularity() granularity}. This value must * not be 0. * @param alignment The alignment in bytes of the start of the address range to be reserved. + * @param executable Indicates if memory for code is requested. * @return An {@linkplain #getAlignment aligned} pointer to the beginning of the reserved * address range, or {@link WordFactory#nullPointer()} in case of an error. */ - Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment); + default Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) { + // GR-36766: required for legacy code, remove as soon as no longer needed + return reserve(nbytes, alignment); + } + + @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true) + default Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) { + return reserve(nbytes, alignment, false); + } /** * Map a region of an open file to the specified address range. When {@linkplain Access#WRITE @@ -183,4 +193,15 @@ default UnsignedWord getAlignment() { * @return 0 when successful, or a non-zero implementation-specific error code. */ int free(PointerBase start, UnsignedWord nbytes); + + /** + * Toggle a thread-local flag that tells the OS our intention about the usage of pages that are + * mapped with MAP_JIT. Only applicable on macOS. Only enforced on AArch64. + * + * @param protect If write protection is enabled, pages mapped with MAP_JIT have effectively + * READ|EXEC permissions for the calling thread. If disabled, the permissions for the + * same pages turn to READ|WRITE. + */ + default void jitWriteProtect(boolean protect) { + } } diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java index 7a7a1a284fe8..23cdb6b05eb1 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/isolated/IsolatedRuntimeCodeInstaller.java @@ -129,8 +129,13 @@ private CodeInstallInfo doPrepareInstall() { private static void installPrepared(SharedMethod method, CodeInstallInfo installInfo, SubstrateInstalledCode installedCode) { IsolatedRuntimeMethodInfoAccess.startTrackingInCurrentIsolate(installInfo.getCodeInfo()); - IsolatedReferenceAdjuster.adjustAndDispose(installInfo.getAdjusterData(), IsolatedCompileClient.get().getHandleSet()); - installInfo.setAdjusterData(WordFactory.nullPointer()); + RuntimeCodeInfoAccess.acquireThreadWriteAccess(); + try { + IsolatedReferenceAdjuster.adjustAndDispose(installInfo.getAdjusterData(), IsolatedCompileClient.get().getHandleSet()); + installInfo.setAdjusterData(WordFactory.nullPointer()); + } finally { + RuntimeCodeInfoAccess.releaseThreadWriteAccess(); + } doInstallPrepared(method, installInfo.getCodeInfo(), installedCode); UnmanagedMemory.free(installInfo); diff --git a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/RuntimeCodeInstaller.java b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/RuntimeCodeInstaller.java index a42221021476..495e535e9553 100644 --- a/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/RuntimeCodeInstaller.java +++ b/substratevm/src/com.oracle.svm.graal/src/com/oracle/svm/graal/meta/RuntimeCodeInstaller.java @@ -182,53 +182,58 @@ protected void doPrepareInstall(ReferenceAdjuster adjuster, CodeInfo codeInfo) { "No direct calls permitted: patching of runtime-compiled code intentionally not supported"); } - prepareCodeMemory(); - - /* - * Object reference constants are stored in this holder first, then written and made visible - * in a single step that is atomic regarding to GC. - */ - ObjectConstantsHolder objectConstants = new ObjectConstantsHolder(compilation); - - // Build an index of PatchingAnnotations - Map patches = new HashMap<>(); - for (CodeAnnotation codeAnnotation : compilation.getCodeAnnotations()) { - if (codeAnnotation instanceof NativeImagePatcher) { - NativeImagePatcher priorValue = patches.put(codeAnnotation.getPosition(), (NativeImagePatcher) codeAnnotation); - VMError.guarantee(priorValue == null, "Registering two patchers for same position."); + RuntimeCodeInfoAccess.acquireThreadWriteAccess(); + try { + prepareCodeMemory(); + + /* + * Object reference constants are stored in this holder first, then written and made + * visible in a single step that is atomic regarding to GC. + */ + ObjectConstantsHolder objectConstants = new ObjectConstantsHolder(compilation); + + // Build an index of PatchingAnnotations + Map patches = new HashMap<>(); + for (CodeAnnotation codeAnnotation : compilation.getCodeAnnotations()) { + if (codeAnnotation instanceof NativeImagePatcher) { + NativeImagePatcher priorValue = patches.put(codeAnnotation.getPosition(), (NativeImagePatcher) codeAnnotation); + VMError.guarantee(priorValue == null, "Registering two patchers for same position."); + } } - } - int numPatchesHandled = patchData(patches, objectConstants); - VMError.guarantee(numPatchesHandled == patches.size(), "Not all patches applied."); - - // Store the compiled code - for (int index = 0; index < codeSize; index++) { - code.writeByte(index, compiledBytes[index]); - } - - // remove write access from code - if (!RuntimeCodeCache.Options.WriteableCodeCache.getValue()) { - makeCodeMemoryReadOnly(code, codeSize); - } + int numPatchesHandled = patchData(patches, objectConstants); + VMError.guarantee(numPatchesHandled == patches.size(), "Not all patches applied."); - /* Write primitive constants to the buffer, record object constants with offsets */ - ByteBuffer dataBuffer = CTypeConversion.asByteBuffer(code.add(dataOffset), compilation.getDataSection().getSectionSize()); - compilation.getDataSection().buildDataSection(dataBuffer, (position, constant) -> { - objectConstants.add(dataOffset + position, - ConfigurationValues.getObjectLayout().getReferenceSize(), - (SubstrateObjectConstant) constant); - }); - - NonmovableArray observerHandles = InstalledCodeObserverSupport.installObservers(codeObservers); - RuntimeCodeInfoAccess.initialize(codeInfo, code, codeSize, dataOffset, dataSize, codeAndDataMemorySize, tier, observerHandles, false); + // Store the compiled code + for (int index = 0; index < codeSize; index++) { + code.writeByte(index, compiledBytes[index]); + } - CodeReferenceMapEncoder encoder = new CodeReferenceMapEncoder(); - encoder.add(objectConstants.referenceMap); - RuntimeCodeInfoAccess.setCodeObjectConstantsInfo(codeInfo, encoder.encodeAll(), encoder.lookupEncoding(objectConstants.referenceMap)); - ImageSingletons.lookup(CodeInfoEncoder.Counters.class).addToReferenceMapSize(encoder.getEncodingSize()); - patchDirectObjectConstants(objectConstants, codeInfo, adjuster); + // remove write access from code + if (!RuntimeCodeCache.Options.WriteableCodeCache.getValue()) { + makeCodeMemoryReadOnly(code, codeSize); + } - createCodeChunkInfos(codeInfo, adjuster); + /* Write primitive constants to the buffer, record object constants with offsets */ + ByteBuffer dataBuffer = CTypeConversion.asByteBuffer(code.add(dataOffset), compilation.getDataSection().getSectionSize()); + compilation.getDataSection().buildDataSection(dataBuffer, (position, constant) -> { + objectConstants.add(dataOffset + position, + ConfigurationValues.getObjectLayout().getReferenceSize(), + (SubstrateObjectConstant) constant); + }); + + NonmovableArray observerHandles = InstalledCodeObserverSupport.installObservers(codeObservers); + RuntimeCodeInfoAccess.initialize(codeInfo, code, codeSize, dataOffset, dataSize, codeAndDataMemorySize, tier, observerHandles, false); + + CodeReferenceMapEncoder encoder = new CodeReferenceMapEncoder(); + encoder.add(objectConstants.referenceMap); + RuntimeCodeInfoAccess.setCodeObjectConstantsInfo(codeInfo, encoder.encodeAll(), encoder.lookupEncoding(objectConstants.referenceMap)); + ImageSingletons.lookup(CodeInfoEncoder.Counters.class).addToReferenceMapSize(encoder.getEncodingSize()); + patchDirectObjectConstants(objectConstants, codeInfo, adjuster); + + createCodeChunkInfos(codeInfo, adjuster); + } finally { + RuntimeCodeInfoAccess.releaseThreadWriteAccess(); + } compilation = null; }