darwin-aarch64: add support for MAP_JIT/pthread_jit_write_protect_np

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -788,6 +788,7 @@ private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list)
     @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false)
     private void blackenStackRoots() {
         Timer blackenStackRootsTimer = timers.blackenStackRoots.open();
+        RuntimeCodeInfoAccess.acquireThreadWriteAccess();
         try {
             Pointer sp = readCallerStackPointer();
             CodePointer ip = readReturnAddress();
@@ -818,6 +819,7 @@ private void blackenStackRoots() {
             }
         } finally {
             blackenStackRootsTimer.close();
+            RuntimeCodeInfoAccess.releaseThreadWriteAccess();
         }
     }
 

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixVirtualMemoryProvider.java

@@ -27,6 +27,7 @@
 import static com.oracle.svm.core.posix.headers.Mman.MAP_ANON;
 import static com.oracle.svm.core.posix.headers.Mman.MAP_FAILED;
 import static com.oracle.svm.core.posix.headers.Mman.MAP_FIXED;
+import static com.oracle.svm.core.posix.headers.Mman.MAP_JIT;
 import static com.oracle.svm.core.posix.headers.Mman.MAP_NORESERVE;
 import static com.oracle.svm.core.posix.headers.Mman.MAP_PRIVATE;
 import static com.oracle.svm.core.posix.headers.Mman.PROT_EXEC;
@@ -38,8 +39,10 @@
 import static com.oracle.svm.core.posix.headers.Mman.NoTransitions.munmap;
 import static org.graalvm.word.WordFactory.nullPointer;
 
+import com.oracle.svm.core.posix.headers.darwin.DarwinPthread;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.c.type.WordPointer;
 import org.graalvm.nativeimage.hosted.Feature;
 import org.graalvm.word.Pointer;
@@ -107,7 +110,7 @@ public UnsignedWord getGranularity() {
 
     @Override
     @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true)
-    public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) {
+    public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) {
         if (nbytes.equal(0)) {
             return WordFactory.nullPointer();
         }
@@ -119,7 +122,11 @@ public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) {
             mappingSize = mappingSize.add(alignment);
         }
         mappingSize = UnsignedUtils.roundUp(mappingSize, granularity);
-        Pointer mappingBegin = mmap(nullPointer(), mappingSize, PROT_NONE(), MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE(), NO_FD, NO_FD_OFFSET);
+        int flags = MAP_ANON() | MAP_PRIVATE() | MAP_NORESERVE();
+        if (Platform.includedIn(Platform.DARWIN_AARCH64.class) && executable) {
+            flags |= MAP_JIT();
+        }
+        Pointer mappingBegin = mmap(nullPointer(), mappingSize, PROT_NONE(), flags, NO_FD, NO_FD_OFFSET);
         if (mappingBegin.equal(MAP_FAILED())) {
             return nullPointer();
         }
@@ -167,6 +174,11 @@ public Pointer commit(PointerBase start, UnsignedWord nbytes, int access) {
         if (start.isNonNull()) {
             flags |= MAP_FIXED();
         }
+
+        boolean isWX = (access & Access.WRITE) != 0 && (access & Access.EXECUTE) != 0;
+        if (Platform.includedIn(Platform.DARWIN_AARCH64.class) && isWX) {
+            flags |= MAP_JIT();
+        }
         /* The memory returned by mmap is guaranteed to be zeroed. */
         final Pointer result = mmap(start, nbytes, accessAsProt(access), flags, NO_FD, NO_FD_OFFSET);
         return result.notEqual(MAP_FAILED()) ? result : nullPointer();
@@ -182,6 +194,12 @@ public int protect(PointerBase start, UnsignedWord nbytes, int access) {
         return mprotect(start, nbytes, accessAsProt(access));
     }
 
+    @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public void jitWriteProtect(boolean protect) {
+        DarwinPthread.pthread_jit_write_protect_np(protect ? 1 : 0);
+    }
+
     @Override
     @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true)
     public int uncommit(PointerBase start, UnsignedWord nbytes) {

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Mman.java

@@ -24,6 +24,8 @@
  */
 package com.oracle.svm.core.posix.headers;
 
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.constant.CConstant;
 import org.graalvm.nativeimage.c.function.CFunction;
@@ -67,6 +69,10 @@ public class Mman {
     @CConstant
     public static native int MAP_NORESERVE();
 
+    @CConstant
+    @Platforms(Platform.DARWIN_AARCH64.class)
+    public static native int MAP_JIT();
+
     @CConstant
     public static native PointerBase MAP_FAILED();
 

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinPthread.java

@@ -24,6 +24,8 @@
  */
 package com.oracle.svm.core.posix.headers.darwin;
 
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.function.CFunction;
 import org.graalvm.nativeimage.c.function.CFunction.Transition;
@@ -49,4 +51,8 @@ public class DarwinPthread {
 
     @CFunction(transition = Transition.NO_TRANSITION)
     public static native Pointer pthread_get_stackaddr_np(Pthread.pthread_t thread);
+
+    @Platforms(Platform.DARWIN_AARCH64.class)
+    @CFunction(transition = Transition.NO_TRANSITION)
+    public static native void pthread_jit_write_protect_np(int enabled);
 }

substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVirtualMemoryProvider.java

@@ -134,7 +134,7 @@ private static int accessAsProt(int access) {
 
     @Override
     @Uninterruptible(reason = "May be called from uninterruptible code.", mayBeInlined = true)
-    public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment) {
+    public Pointer reserve(UnsignedWord nbytes, UnsignedWord alignment, boolean executable) {
         if (nbytes.equal(0)) {
             return WordFactory.nullPointer();
         }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java

@@ -26,8 +26,13 @@
 
 import java.util.EnumSet;
 
+import com.oracle.svm.core.os.VirtualMemoryProvider;
+import com.oracle.svm.core.threadlocal.FastThreadLocalFactory;
+import com.oracle.svm.core.threadlocal.FastThreadLocalInt;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.UnmanagedMemory;
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.nativeimage.c.struct.SizeOf;
@@ -248,6 +253,32 @@ public static void makeCodeMemoryWriteableNonExecutable(CodePointer start, Unsig
         CommittedMemoryProvider.get().protect(start, size, EnumSet.of(CommittedMemoryProvider.Access.READ, CommittedMemoryProvider.Access.WRITE));
     }
 
+    @Platforms(Platform.DARWIN_AARCH64.class) private static final FastThreadLocalInt jitProtectDepth = FastThreadLocalFactory.createInt("jitProtectDepth");
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static void acquireThreadWriteAccess() {
+        if (Platform.includedIn(Platform.DARWIN_AARCH64.class)) {
+            // Disabling write protection can be nested, for example a GC can be triggered during
+            // code installation which in turn causes walk of references in code. Both need to
+            // disable write protection, but only the outer one should enable it again.
+            if (jitProtectDepth.get() == 0) {
+                VirtualMemoryProvider.get().jitWriteProtect(false);
+            }
+            jitProtectDepth.set(jitProtectDepth.get() + 1);
+        }
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static void releaseThreadWriteAccess() {
+        if (Platform.includedIn(Platform.DARWIN_AARCH64.class)) {
+            VMError.guarantee(jitProtectDepth.get() >= 1);
+            jitProtectDepth.set(jitProtectDepth.get() - 1);
+            if (jitProtectDepth.get() == 0) {
+                VirtualMemoryProvider.get().jitWriteProtect(true);
+            }
+        }
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code", mayBeInlined = true)
     static void releaseMethodInfoOnTearDown(CodeInfo info) {
         InstalledCodeObserverSupport.removeObserversOnTearDown(getCodeObserverHandles(info));

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java

@@ -203,18 +203,23 @@ public boolean walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) {
         assert VMOperation.isGCInProgress() : "otherwise, we would need to make sure that the CodeInfo is not freeded by the GC";
         if (table.isNonNull()) {
             int length = NonmovableArrays.lengthOf(table);
-            for (int i = 0; i < length;) {
-                UntetheredCodeInfo info = NonmovableArrays.getWord(table, i);
-                if (info.isNonNull()) {
-                    visitor.visitCode(CodeInfoAccess.convert(info));
-                }
+            RuntimeCodeInfoAccess.acquireThreadWriteAccess();
+            try {
+                for (int i = 0; i < length;) {
+                    UntetheredCodeInfo info = NonmovableArrays.getWord(table, i);
+                    if (info.isNonNull()) {
+                        visitor.visitCode(CodeInfoAccess.convert(info));
+                    }
 
-                // If the visitor removed the current entry from the table, then it is necessary to
-                // visit the now updated entry one more time. However, this could have the effect
-                // that some entries are visited more than once.
-                if (info == NonmovableArrays.getWord(table, i)) {
-                    i++;
+                    // If the visitor removed the current entry from the table, then it is necessary
+                    // to visit the now updated entry one more time. However, this could have the
+                    // effect that some entries are visited more than once.
+                    if (info == NonmovableArrays.getWord(table, i)) {
+                        i++;
+                    }
                 }
+            } finally {
+                RuntimeCodeInfoAccess.releaseThreadWriteAccess();
             }
         }
         return true;

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java

@@ -28,9 +28,11 @@
 import static com.oracle.svm.core.Isolates.IMAGE_HEAP_END;
 import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_BEGIN;
 import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_END;
+import static org.graalvm.word.WordFactory.nullPointer;
 
 import java.util.EnumSet;
 
+import com.oracle.svm.core.util.UnsignedUtils;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.PointerBase;
@@ -40,6 +42,7 @@
 import com.oracle.svm.core.annotate.Uninterruptible;
 import com.oracle.svm.core.c.function.CEntryPointErrors;
 import com.oracle.svm.core.heap.Heap;
+import org.graalvm.word.WordFactory;
 
 public abstract class AbstractCommittedMemoryProvider implements CommittedMemoryProvider {
     @Fold
@@ -94,4 +97,116 @@ public boolean protect(PointerBase start, UnsignedWord nbytes, EnumSet<Access> a
         int success = VirtualMemoryProvider.get().protect(start, nbytes, vmAccessBits);
         return success == 0;
     }
+
+    @Override
+    public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) {
+        return allocate(nbytes, alignment, false);
+    }
+
+    @Override
+    public Pointer allocateUnalignedChunk(UnsignedWord nbytes) {
+        return allocate(nbytes, WordFactory.unsigned(1), false);
+    }
+
+    @Override
+    public Pointer allocateExecutableMemory(UnsignedWord nbytes, UnsignedWord alignment) {
+        return allocate(nbytes, alignment, true);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private Pointer allocate(UnsignedWord size, UnsignedWord alignment, boolean executable) {
+        int access = VirtualMemoryProvider.Access.READ | VirtualMemoryProvider.Access.WRITE;
+        if (executable) {
+            access |= VirtualMemoryProvider.Access.EXECUTE;
+        }
+        Pointer reserved = WordFactory.nullPointer();
+        if (!UnsignedUtils.isAMultiple(getGranularity(), alignment)) {
+            reserved = VirtualMemoryProvider.get().reserve(size, alignment, executable);
+            if (reserved.isNull()) {
+                return nullPointer();
+            }
+        }
+        Pointer committed = VirtualMemoryProvider.get().commit(reserved, size, access);
+        if (committed.isNull()) {
+            if (reserved.isNonNull()) {
+                VirtualMemoryProvider.get().free(reserved, size);
+            }
+            return nullPointer();
+        }
+        assert reserved.isNull() || reserved.equal(committed);
+        track(size);
+        return committed;
+    }
+
+    @Override
+    public boolean areUnalignedChunksZeroed() {
+        return false;
+    }
+
+    @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public void freeAlignedChunk(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) {
+        free(start, nbytes);
+    }
+
+    @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public void freeUnalignedChunk(PointerBase start, UnsignedWord nbytes) {
+        free(start, nbytes);
+    }
+
+    @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public void freeExecutableMemory(PointerBase start, UnsignedWord nbytes, UnsignedWord alignment) {
+        free(start, nbytes);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private void free(PointerBase start, UnsignedWord nbytes) {
+        if (VirtualMemoryProvider.get().free(start, nbytes) == 0) {
+            untrack(nbytes);
+        }
+    }
+
+    private final VirtualMemoryTracker tracker = new VirtualMemoryTracker();
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public VirtualMemoryTracker getTracker() {
+        return tracker;
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private void track(UnsignedWord nbytes) {
+        if (getTracker() == null) {
+            return;
+        }
+        getTracker().track(nbytes);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private void untrack(UnsignedWord nbytes) {
+        if (getTracker() == null) {
+            return;
+        }
+        getTracker().untrack(nbytes);
+    }
+
+    public static class VirtualMemoryTracker {
+
+        private UnsignedWord totalAllocated;
+
+        public VirtualMemoryTracker() {
+            this.totalAllocated = WordFactory.zero();
+        }
+
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void track(UnsignedWord size) {
+            totalAllocated = totalAllocated.add(size);
+        }
+
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void untrack(UnsignedWord size) {
+            totalAllocated = totalAllocated.subtract(size);
+        }
+    }
 }