From aff3946a0e5f8d98838c2931b4c6bce8cb9f0cc2 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Fri, 1 Apr 2022 15:02:37 +0200
Subject: [PATCH 01/12] Implement StoredContinuation as @Hybrid class.

---
 .../genscavenge/ThreadLocalAllocation.java    |   4 +-
 .../graal/GenScavengeAllocationSnippets.java  |  70 ----------
 .../com/oracle/svm/core/annotate/Hybrid.java  |  21 ++-
 .../nodes/NewStoredContinuationNode.java      |  55 --------
 .../nodes/SubstrateNewHybridInstanceNode.java |   3 +
 .../svm/core/heap/StoredContinuation.java     |   8 ++
 .../svm/core/heap/StoredContinuationImpl.java | 124 ++++++------------
 .../com/oracle/svm/core/hub/DynamicHub.java   |   7 +-
 .../src/com/oracle/svm/core/hub/HubType.java  |   8 +-
 .../oracle/svm/core/hub/LayoutEncoding.java   |  26 +---
 .../svm/core/thread/ContinuationsFeature.java |   9 +-
 .../svm/hosted/NativeImageGenerator.java      |   3 -
 .../src/com/oracle/svm/hosted/SVMHost.java    |   3 -
 .../hosted/config/HybridLayoutSupport.java    |   9 +-
 .../svm/hosted/meta/UniverseBuilder.java      |   4 +-
 15 files changed, 67 insertions(+), 287 deletions(-)
 delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewStoredContinuationNode.java

diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
index f1afb6da2160..3001158d58ee 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
@@ -211,7 +211,7 @@ private static void runSlowPathHooks() {
     private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub) {
         DeoptTester.disableDeoptTesting();
         try {
-            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.allocateNewInstance", DynamicHub.toClass(hub).getName());
+            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewInstanceWithoutAllocating", DynamicHub.toClass(hub).getName());
             GCImpl.getGCImpl().maybeCollectOnAllocation();
 
             AlignedHeader newTlab = HeapImpl.getChunkProvider().produceAlignedChunk();
@@ -258,7 +258,7 @@ private static Object slowPathNewArray(Word objectHeader, int length, int fillSt
     private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset) {
         DeoptTester.disableDeoptTesting();
         try {
-            HeapImpl.exitIfAllocationDisallowed("Heap.allocateNewArray", DynamicHub.toClass(hub).getName());
+            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayWithoutAllocating", DynamicHub.toClass(hub).getName());
             GCImpl.getGCImpl().maybeCollectOnAllocation();
 
             if (size.aboveOrEqual(HeapParameters.getLargeArrayThreshold())) {
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
index fd9c80902811..dbb1d3cf308a 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
@@ -29,7 +29,6 @@
 import org.graalvm.compiler.api.replacements.Snippet;
 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
 import org.graalvm.compiler.graph.Node;
-import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.SnippetAnchorNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
@@ -40,10 +39,8 @@
 import org.graalvm.compiler.replacements.SnippetTemplate;
 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
-import org.graalvm.compiler.word.ObjectAccess;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
-import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.genscavenge.HeapParameters;
@@ -53,19 +50,13 @@
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
-import com.oracle.svm.core.graal.nodes.NewStoredContinuationNode;
 import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
 import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets;
 import com.oracle.svm.core.heap.Heap;
-import com.oracle.svm.core.heap.StoredContinuation;
-import com.oracle.svm.core.heap.StoredContinuationImpl;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.LayoutEncoding;
-import com.oracle.svm.core.meta.SharedType;
-import com.oracle.svm.core.meta.SubstrateObjectConstant;
 import com.oracle.svm.core.snippets.SnippetRuntime;
 import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
-import com.oracle.svm.core.thread.Continuation;
 
 final class GenScavengeAllocationSnippets extends SubstrateAllocationSnippets {
     private static final SubstrateForeignCallDescriptor SLOW_NEW_INSTANCE = SnippetRuntime.findForeignCall(ThreadLocalAllocation.class, "slowPathNewInstance", true);
@@ -112,30 +103,6 @@ private static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet,
         return ObjectHeaderImpl.encodeAsObjectHeader(hub, rememberedSet, unaligned);
     }
 
-    @Snippet
-    public Object allocateStoredContinuationInstance(@ConstantParameter DynamicHub hub, int payloadSize, @ConstantParameter DynamicHub byteArrayHub,
-                    @ConstantParameter int byteArrayBaseOffset, @ConstantParameter AllocationProfilingData profilingData) {
-        /*
-         * We allocate a byte[] first and then convert it to a StoredContinuation. We must pass
-         * parameters below that match the layout of a regular byte[], or we will run into problems
-         * because not all parameters are passed on to the slow path.
-         *
-         * Barrier code assumes that instance objects are always in aligned chunks, but a large
-         * StoredContinuation can end up in an unaligned chunk. Still, no barriers are needed
-         * because objects are immutable once filled and are then written only by GC.
-         */
-        int arrayLength = StoredContinuationImpl.PAYLOAD_OFFSET + payloadSize - byteArrayBaseOffset;
-        Object result = allocateArrayImpl(encodeAsTLABObjectHeader(byteArrayHub), arrayLength, byteArrayBaseOffset, 0, FillContent.WITH_GARBAGE_IF_ASSERTIONS_ENABLED,
-                        afterArrayLengthOffset(), false, false, false, false, profilingData);
-        UnsignedWord arrayHeader = ObjectHeaderImpl.readHeaderFromObject(result);
-        Word header = encodeAsObjectHeader(hub, ObjectHeaderImpl.hasRememberedSet(arrayHeader), ObjectHeaderImpl.isUnalignedHeader(arrayHeader));
-        initializeObjectHeader(Word.objectToUntrackedPointer(result), header, false);
-        ObjectAccess.writeObject(result, hub.getMonitorOffset(), null, LocationIdentity.init());
-        StoredContinuationImpl.initializeNewlyAllocated(result, payloadSize);
-        emitMemoryBarrierIf(true);
-        return PiNode.piCastToSnippetReplaceeStamp(result);
-    }
-
     @Override
     public void initializeObjectHeader(Word memory, Word objectHeader, boolean isArray) {
         Heap.getHeap().getObjectHeader().initializeHeaderOfNewObject(memory, objectHeader);
@@ -184,16 +151,12 @@ protected SubstrateForeignCallDescriptor getSlowNewArrayStub() {
     public static class Templates extends SubstrateAllocationSnippets.Templates {
         private final SnippetInfo formatObject;
         private final SnippetInfo formatArray;
-        private final SnippetInfo allocateStoredContinuationInstance;
 
         Templates(SubstrateAllocationSnippets receiver, OptionValues options, SnippetCounter.Group.Factory groupFactory, Providers providers) {
             super(receiver, options, groupFactory, providers);
 
             formatObject = snippet(GenScavengeAllocationSnippets.class, "formatObjectSnippet", null, receiver);
             formatArray = snippet(GenScavengeAllocationSnippets.class, "formatArraySnippet", null, receiver);
-
-            allocateStoredContinuationInstance = !Continuation.isSupported() ? null
-                            : snippet(GenScavengeAllocationSnippets.class, "allocateStoredContinuationInstance", null, receiver, ALLOCATION_LOCATIONS);
         }
 
         @Override
@@ -205,11 +168,6 @@ public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>
 
             FormatArrayLowering formatArrayLowering = new FormatArrayLowering();
             lowerings.put(FormatArrayNode.class, formatArrayLowering);
-
-            if (Continuation.isSupported()) {
-                NewStoredContinuationLowering newStoredContinuationLowering = new NewStoredContinuationLowering();
-                lowerings.put(NewStoredContinuationNode.class, newStoredContinuationLowering);
-            }
         }
 
         private class FormatObjectLowering implements NodeLoweringProvider<FormatObjectNode> {
@@ -252,33 +210,5 @@ public void lower(FormatArrayNode node, LoweringTool tool) {
                 template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
             }
         }
-
-        private class NewStoredContinuationLowering implements NodeLoweringProvider<NewStoredContinuationNode> {
-            @Override
-            public void lower(NewStoredContinuationNode node, LoweringTool tool) {
-                StructuredGraph graph = node.graph();
-
-                if (graph.getGuardsStage() != StructuredGraph.GuardsStage.AFTER_FSA) {
-                    return;
-                }
-
-                DynamicHub hub = ((SharedType) tool.getMetaAccess().lookupJavaType(StoredContinuation.class)).getHub();
-                assert hub.isStoredContinuationClass();
-                ConstantNode hubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(hub), providers.getMetaAccess(), graph);
-
-                DynamicHub byteArrayHub = ((SharedType) tool.getMetaAccess().lookupJavaType(byte[].class)).getHub();
-                ConstantNode byteArrayHubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(byteArrayHub), providers.getMetaAccess(), graph);
-                int byteArrayBaseOffset = getArrayBaseOffset(byteArrayHub.getLayoutEncoding());
-
-                Arguments args = new Arguments(allocateStoredContinuationInstance, graph.getGuardsStage(), tool.getLoweringStage());
-                args.addConst("hub", hubConstant);
-                args.add("payloadSize", node.getPayloadSize());
-                args.addConst("byteArrayHub", byteArrayHubConstant);
-                args.addConst("byteArrayBaseOffset", byteArrayBaseOffset);
-                args.addConst("profilingData", getProfilingData(node, null));
-
-                template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
-            }
-        }
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
index cb999f027cb9..8f0c0172e8a7 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
@@ -37,11 +37,6 @@
  * instance layouts and array layouts. The contents of a specified member array and (optional)
  * member type id slots are directly placed within the class layout. This saves one indirection when
  * accessing the array or type id slots.
- *
- * <p>
- * The array length is located directly after the HUB pointer, like in regular array. Then (if
- * present) the type id slots follow. Then the instance fields are placed. At the end of the layout,
- * the array elements are located.
  * 
  * <pre>
  *    +--------------------------------------------------+
@@ -73,15 +68,15 @@
 public @interface Hybrid {
 
     /**
-     * If {@code true}, we expect that the data in the hybrid fields can be duplicated between the
-     * hybrid object and a separate object for the array. For most objects, a duplication could
-     * occur if inlining and constant folding result in the internal reference to a hybrid field
-     * being constant folded to a constant value, which must be written into the image heap
-     * separately from the hybrid object.
-     * 
-     * If {@code false}, we expect that this duplication of the hybrid fields can never happen.
+     * If {@code true}, we allow the data in the hybrid fields to be duplicated between the hybrid
+     * object and a separate object for the array. For most objects, a duplication could occur if
+     * inlining and constant folding result in the internal reference to a hybrid field being
+     * constant folded to a constant value, which must be written into the image heap separately
+     * from the hybrid object.
+     *
+     * If {@code false}, a duplication of the hybrid fields must never happen.
      */
-    boolean canHybridFieldsBeDuplicated();
+    boolean canHybridFieldsBeDuplicated() default false;
 
     /**
      * Specifies a single member array as the hybrid array.
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewStoredContinuationNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewStoredContinuationNode.java
deleted file mode 100644
index ebcc21001f52..000000000000
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewStoredContinuationNode.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2020, 2020, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package com.oracle.svm.core.graal.nodes;
-
-import org.graalvm.compiler.core.common.type.StampFactory;
-import org.graalvm.compiler.graph.NodeClass;
-import org.graalvm.compiler.nodeinfo.NodeInfo;
-import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
-import org.graalvm.compiler.nodes.spi.Lowerable;
-
-import com.oracle.svm.core.heap.StoredContinuation;
-
-import jdk.vm.ci.meta.JavaKind;
-
-@NodeInfo
-public class NewStoredContinuationNode extends AbstractNewObjectNode implements Lowerable {
-
-    public static final NodeClass<NewStoredContinuationNode> TYPE = NodeClass.create(NewStoredContinuationNode.class);
-    @Input private ValueNode payloadSize;
-
-    public NewStoredContinuationNode(ValueNode payloadSize) {
-        super(TYPE, StampFactory.forKind(JavaKind.fromJavaClass(StoredContinuation.class)), false, null);
-        this.payloadSize = payloadSize;
-    }
-
-    public ValueNode getPayloadSize() {
-        return payloadSize;
-    }
-
-    @NodeIntrinsic
-    public static native StoredContinuation allocate(int payloadSize);
-}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateNewHybridInstanceNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateNewHybridInstanceNode.java
index 9847e1ed61cd..cd76c27a7ac1 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateNewHybridInstanceNode.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateNewHybridInstanceNode.java
@@ -69,4 +69,7 @@ public ResolvedJavaType instanceClass() {
     public ResolvedJavaType elementType() {
         return elementType;
     }
+
+    @NodeIntrinsic
+    public static native Object allocate(@ConstantNodeParameter Class<?> instanceType, @ConstantNodeParameter Class<?> elementType, int arrayLength);
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
index 147dd678263f..2a1605796571 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
@@ -24,10 +24,18 @@
  */
 package com.oracle.svm.core.heap;
 
+import com.oracle.svm.core.annotate.Hybrid;
+
 /**
  * This class is used for variably-sized objects that store continuation stack frames.
  *
  * For object layout and other implementation details, see {@link StoredContinuationImpl}.
  */
+@Hybrid
 public final class StoredContinuation {
+    @Hybrid.Array byte[] payload;
+
+    /** Must be allocated via {@link StoredContinuationImpl}. */
+    private StoredContinuation() {
+    }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationImpl.java
index 9b54c4eeff53..e0d2d6a08735 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationImpl.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuationImpl.java
@@ -31,11 +31,13 @@
 import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.compiler.core.common.util.TypeConversion;
+import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.c.function.CodePointer;
-import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.Pointer;
+import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.UnmanagedMemoryUtil;
@@ -48,7 +50,9 @@
 import com.oracle.svm.core.code.FrameInfoQueryResult;
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.deopt.DeoptimizedFrame;
-import com.oracle.svm.core.graal.nodes.NewStoredContinuationNode;
+import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
+import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.stack.JavaStackWalker;
 import com.oracle.svm.core.stack.StackFrameVisitor;
 import com.oracle.svm.core.thread.Continuation;
@@ -59,21 +63,17 @@
 import jdk.vm.ci.meta.JavaKind;
 
 /**
- * Helper class to access a {@link StoredContinuation}.
- *
- * Memory layout of a {@link StoredContinuation} instance:e Each {@link StoredContinuation} has a
- * fixed-size header with the size of the instance, followed by the data of the stored frame(s).
+ * Helper class to access a {@link StoredContinuation}, the payload layout of which is:
  *
  * <pre>
- *      +-0x0-|-0x4-|-0x8-|-0xa-+
- * 0x00 | hub | (1) | (2) |  -  | header: hub; 1) identity hash code; 2) object monitor;
- *      +-----------------------+               3) payload size in bytes; 4) number of frames n
- * 0x10 | (3) | (4) | (a1)|(b1) | 8-byte per-frame headers: a) size of frame in bytes; b) reference map index of frame
- *      +-----------------------+
- * 0x20 | (a2)|(b2) |  .. | ..  |
- *  .   | (an)|(bn) | c o n t i |
- *  .   : n u o u s   f r a m e :
- *  .   |  d a t a  |
+ *      +-0x0-|-0x4-|-0x8-|-0xc-+
+ * 0x00 | (1) | (a1)|(b1) | (a2)| 1) number of frames n; per-frame headers: a) size of frame,
+ *      +-----+-----+-----+-----+                                           b) reference map index of frame
+ * 0x10 |(b2) | ..  | ..  | (an)|
+ *      +-----+-----+-----+-----+
+ * 0x20 |(bn) | c o n t i n u o |
+ *  .   :  u s    f r a m e     :
+ * 0xnn |  d a t a  +-----------+
  *      +-----------+
  * </pre>
  */
@@ -82,62 +82,29 @@ public interface RawFrameVisitor {
         boolean visitRawFrame(int index, Pointer sp);
     }
 
-    private static final int HEADER_SIZE = 0x18;
-    public static final int PAYLOAD_OFFSET = HEADER_SIZE;
-    public static final int OBJECT_MONITOR_OFFSET = 8;
-
-    private static final int SIZE_OFFSET_TO_PAYLOAD = -HEADER_SIZE + 0x10;
-    private static final int FRAME_COUNT_OFFSET_TO_PAYLOAD = -HEADER_SIZE + 0x14;
-
-    private static final int VALID_OFFSET_START = SIZE_OFFSET_TO_PAYLOAD;
-
-    private static final int FRAME_META_START_OFFSET_TO_PAYLOAD = 0;
+    private static final int FRAME_COUNT_OFFSET = 0;
+    private static final int FRAME_METAS_START_OFFSET = 4;
     private static final int FRAME_META_SIZE = 8;
     private static final int SIZE_OFFSET_IN_FRAME_META = 0;
     private static final int REFERENCE_MAP_INDEX_OFFSET_IN_FRAME_META = 4;
 
-    @Uninterruptible(reason = "in allocation of StoredContinuation instance")
-    public static void initializeNewlyAllocated(Object obj, int payloadSize) {
-        Pointer p = Word.objectToUntrackedPointer(obj).add(PAYLOAD_OFFSET);
-        p.writeInt(SIZE_OFFSET_TO_PAYLOAD, payloadSize, LocationIdentity.init());
-        // Keep GC from taking a closer look until frames are initialized
-        p.writeInt(FRAME_COUNT_OFFSET_TO_PAYLOAD, 0, LocationIdentity.init());
-    }
-
-    /* All method calls in this function except `allocate` should be `@Uninterruptible` */
     private static StoredContinuation allocate(int payloadSize) {
-        assert payloadSize % 8 == 0;
-        StoredContinuation f = NewStoredContinuationNode.allocate(payloadSize);
+        StoredContinuation f = (StoredContinuation) SubstrateNewHybridInstanceNode.allocate(StoredContinuation.class, byte.class, payloadSize);
         assert readPayloadSize(f) == payloadSize;
         return f;
     }
 
-    // validator functions
-
-    @Uninterruptible(reason = "read StoredContinuation")
-    private static boolean checkOffset(StoredContinuation f, int offset) {
-        assert offset % 4 == 0;
-        if (offset >= 0) {
-            assert offset < readSize(f);
-        } else {
-            assert offset >= VALID_OFFSET_START;
-        }
-        return true;
-    }
-
     @Uninterruptible(reason = "read StoredContinuation")
     private static boolean checkPayloadOffset(StoredContinuation f, int offset) {
         assert offset % 4 == 0;
         assert offset >= 0;
-        assert offset < readSize(f);
+        assert offset < readPayloadSize(f);
         return true;
     }
 
-    // read/write functions
-
     @Uninterruptible(reason = "read StoredContinuation")
     private static int readPayloadInt(StoredContinuation f, int offset) {
-        assert checkOffset(f, offset);
+        assert checkPayloadOffset(f, offset);
         return payloadLocation(f).readInt(offset);
     }
 
@@ -147,56 +114,45 @@ private static void writePayloadInt(StoredContinuation f, int offset, int value)
         payloadLocation(f).writeInt(offset, value);
     }
 
-    // size of payload
-    @Uninterruptible(reason = "read StoredContinuation")
-    private static int readPayloadSize(StoredContinuation f) {
-        return readPayloadInt(f, SIZE_OFFSET_TO_PAYLOAD);
-    }
+    @NodeIntrinsic(ArrayLengthNode.class)
+    private static native int readPayloadSize(StoredContinuation f);
 
     // size of raw frame
     @Uninterruptible(reason = "read StoredContinuation")
     public static int readAllFrameSize(StoredContinuation f) {
-        return readPayloadSize(f) - readFrameMetaSize(f);
-    }
-
-    // size of object
-    @Uninterruptible(reason = "read StoredContinuation")
-    public static int readSize(StoredContinuation f) {
-        return readPayloadSize(f) + HEADER_SIZE;
+        return readPayloadSize(f) - readFrameMetasSize(f);
     }
 
     @Uninterruptible(reason = "read StoredContinuation")
     public static int readFrameCount(StoredContinuation f) {
-        return readPayloadInt(f, FRAME_COUNT_OFFSET_TO_PAYLOAD);
+        return readPayloadInt(f, FRAME_COUNT_OFFSET);
     }
 
-    // read frame meta-data
-
     @Uninterruptible(reason = "read StoredContinuation")
-    private static int readFrameMetaSize(StoredContinuation f) {
-        return FRAME_META_START_OFFSET_TO_PAYLOAD + readFrameCount(f) * FRAME_META_SIZE;
+    private static int readFrameMetasSize(StoredContinuation f) {
+        return FRAME_METAS_START_OFFSET + readFrameCount(f) * FRAME_META_SIZE;
     }
 
     @Uninterruptible(reason = "read StoredContinuation")
     public static int readFrameSize(StoredContinuation f, int frameIndex) {
-        return payloadLocation(f).readInt(FRAME_META_START_OFFSET_TO_PAYLOAD + frameIndex * FRAME_META_SIZE + SIZE_OFFSET_IN_FRAME_META);
+        return readPayloadInt(f, FRAME_METAS_START_OFFSET + frameIndex * FRAME_META_SIZE + SIZE_OFFSET_IN_FRAME_META);
     }
 
     @Uninterruptible(reason = "read StoredContinuation")
     private static int readReferenceMapIndex(StoredContinuation f, int frameIndex) {
-        return payloadLocation(f).readInt(FRAME_META_START_OFFSET_TO_PAYLOAD + frameIndex * FRAME_META_SIZE + REFERENCE_MAP_INDEX_OFFSET_IN_FRAME_META);
+        return readPayloadInt(f, FRAME_METAS_START_OFFSET + frameIndex * FRAME_META_SIZE + REFERENCE_MAP_INDEX_OFFSET_IN_FRAME_META);
     }
 
-    // Pointers
-
     @Uninterruptible(reason = "read/write StoredContinuation")
     private static Pointer payloadLocation(StoredContinuation f) {
-        return Word.objectToUntrackedPointer(f).add(PAYLOAD_OFFSET);
+        int layout = KnownIntrinsics.readHub(f).getLayoutEncoding();
+        UnsignedWord baseOffset = LayoutEncoding.getArrayBaseOffset(layout);
+        return Word.objectToUntrackedPointer(f).add(baseOffset);
     }
 
     @Uninterruptible(reason = "read/write StoredContinuation")
     public static Pointer payloadFrameStart(StoredContinuation f) {
-        return payloadLocation(f).add(readFrameMetaSize(f));
+        return payloadLocation(f).add(readFrameMetasSize(f));
     }
 
     public static int allocateFromCurrentStack(Continuation cont, Pointer rootSp, Pointer leafSp, CodePointer leafIp) {
@@ -235,7 +191,7 @@ private static int allocateFromStack(Continuation cont, Pointer rootSp, Pointer
         VMError.guarantee(resultLeafSP.isNonNull());
 
         int frameCount = visitor.frameSizeReferenceMapIndex.size();
-        int payloadSize = FRAME_META_SIZE * frameCount + UnsignedUtils.safeToInt(rootSp.subtract(resultLeafSP));
+        int payloadSize = FRAME_METAS_START_OFFSET + FRAME_META_SIZE * frameCount + UnsignedUtils.safeToInt(rootSp.subtract(resultLeafSP));
 
         cont.stored = allocate(payloadSize);
         /*
@@ -246,9 +202,9 @@ private static int allocateFromStack(Continuation cont, Pointer rootSp, Pointer
         int allFrameSize = 0;
         for (int i = 0; i < frameCount; i++) {
             Pair<Integer, Integer> frameSizeRefMapInxPair = visitor.frameSizeReferenceMapIndex.get(i);
-            writePayloadInt(cont.stored, FRAME_META_START_OFFSET_TO_PAYLOAD + i * FRAME_META_SIZE + SIZE_OFFSET_IN_FRAME_META,
+            writePayloadInt(cont.stored, FRAME_METAS_START_OFFSET + i * FRAME_META_SIZE + SIZE_OFFSET_IN_FRAME_META,
                             frameSizeRefMapInxPair.getLeft());
-            writePayloadInt(cont.stored, FRAME_META_START_OFFSET_TO_PAYLOAD + i * FRAME_META_SIZE + REFERENCE_MAP_INDEX_OFFSET_IN_FRAME_META,
+            writePayloadInt(cont.stored, FRAME_METAS_START_OFFSET + i * FRAME_META_SIZE + REFERENCE_MAP_INDEX_OFFSET_IN_FRAME_META,
                             frameSizeRefMapInxPair.getRight());
             allFrameSize += frameSizeRefMapInxPair.getLeft();
         }
@@ -259,7 +215,7 @@ private static int allocateFromStack(Continuation cont, Pointer rootSp, Pointer
     @Uninterruptible(reason = "Prevent modifications to the stack while copying.")
     private static void fillUninterruptibly(StoredContinuation stored, Pointer sp, int size, int frameCount) {
         Pointer p = payloadLocation(stored);
-        p.writeInt(FRAME_COUNT_OFFSET_TO_PAYLOAD, frameCount);
+        p.writeInt(FRAME_COUNT_OFFSET, frameCount);
 
         VMError.guarantee(size == readAllFrameSize(stored));
         Pointer frameStart = payloadFrameStart(stored);
@@ -286,13 +242,9 @@ private static void fillUninterruptibly(StoredContinuation stored, Pointer sp, i
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean walkStoredContinuationFromPointer(Pointer baseAddress, RawFrameVisitor frameVisitor, ObjectReferenceVisitor refVisitor, Object holderObject) {
         StoredContinuation f = (StoredContinuation) holderObject;
-
-        Pointer payloadStart = StoredContinuationImpl.payloadLocation(f);
-        assert payloadStart.subtract(baseAddress).equal(StoredContinuationImpl.HEADER_SIZE) : "base address not pointing to frame instance";
+        assert baseAddress.equal(Word.objectToUntrackedPointer(holderObject));
 
         int frameCount = StoredContinuationImpl.readFrameCount(f);
-        int size = StoredContinuationImpl.readPayloadSize(f);
-
         Pointer curFrame = StoredContinuationImpl.payloadFrameStart(f);
         int frameIndex = 0;
 
@@ -314,7 +266,7 @@ public static boolean walkStoredContinuationFromPointer(Pointer baseAddress, Raw
 
         assert frameIndex == frameCount;
         // NOTE: frameCount can be 0 if the object has just been allocated but not filled yet
-        assert frameCount == 0 || curFrame.subtract(payloadStart).equal(size);
+        assert frameCount == 0 || curFrame.subtract(payloadLocation(f)).equal(readPayloadSize(f));
 
         return true;
     }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java
index b09ed3becc83..6bd0312dc64b 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java
@@ -98,7 +98,7 @@
 import sun.reflect.generics.factory.GenericsFactory;
 import sun.reflect.generics.repository.ClassRepository;
 
-@Hybrid(canHybridFieldsBeDuplicated = false)
+@Hybrid
 @Substitute
 @TargetClass(java.lang.Class.class)
 @SuppressWarnings({"static-method", "serial"})
@@ -611,11 +611,6 @@ public boolean isInstanceClass() {
         return HubType.isInstance(hubType);
     }
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public boolean isStoredContinuationClass() {
-        return HubType.isStoredContinuation(hubType);
-    }
-
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean isReferenceInstanceClass() {
         return HubType.isReferenceInstance(hubType);
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java
index 54da8de209b4..20cf74467fbe 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java
@@ -34,7 +34,6 @@ public enum HubType {
     InstanceReference(1),
 
     // special hubs
-    StoredContinuation(2),
     Other(3),
 
     // array hubs
@@ -54,7 +53,7 @@ public int getValue() {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isInstance(int hubType) {
-        return hubType <= StoredContinuation.getValue();
+        return hubType <= InstanceReference.getValue();
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
@@ -62,11 +61,6 @@ public static boolean isReferenceInstance(int hubType) {
         return hubType == InstanceReference.getValue();
     }
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static boolean isStoredContinuation(int hubType) {
-        return hubType == StoredContinuation.getValue();
-    }
-
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArray(int hubType) {
         return hubType >= TypeArray.getValue();
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
index c1c600fc8254..78bfaf239f69 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
@@ -24,11 +24,9 @@
  */
 package com.oracle.svm.core.hub;
 
-import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.compiler.core.common.calc.UnsignedMath;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
-import org.graalvm.compiler.replacements.ReplacementsUtil;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
@@ -42,10 +40,7 @@
 import com.oracle.svm.core.annotate.Hybrid;
 import com.oracle.svm.core.annotate.Uninterruptible;
 import com.oracle.svm.core.config.ObjectLayout;
-import com.oracle.svm.core.heap.StoredContinuation;
-import com.oracle.svm.core.heap.StoredContinuationImpl;
 import com.oracle.svm.core.snippets.KnownIntrinsics;
-import com.oracle.svm.core.thread.Continuation;
 import com.oracle.svm.core.util.VMError;
 
 import jdk.vm.ci.meta.ResolvedJavaType;
@@ -81,8 +76,7 @@ public class LayoutEncoding {
     private static final int PRIMITIVE_VALUE = NEUTRAL_VALUE + 1;
     private static final int INTERFACE_VALUE = PRIMITIVE_VALUE + 1;
     private static final int ABSTRACT_VALUE = INTERFACE_VALUE + 1;
-    private static final int STORED_CONTINUATION_VALUE = ABSTRACT_VALUE + 1;
-    private static final int LAST_SPECIAL_VALUE = STORED_CONTINUATION_VALUE;
+    private static final int LAST_SPECIAL_VALUE = ABSTRACT_VALUE;
 
     private static final int ARRAY_INDEX_SHIFT_SHIFT = 0;
     private static final int ARRAY_INDEX_SHIFT_MASK = 0xff;
@@ -105,10 +99,6 @@ public static int forAbstract() {
         return ABSTRACT_VALUE;
     }
 
-    public static int forStoredContinuation() {
-        return STORED_CONTINUATION_VALUE;
-    }
-
     @Platforms(Platform.HOSTED_ONLY.class)
     private static void guaranteeEncoding(ResolvedJavaType type, boolean condition, String description) {
         if (!condition) {
@@ -125,7 +115,6 @@ public static int forInstance(ResolvedJavaType type, int size) {
         guaranteeEncoding(type, size > LAST_SPECIAL_VALUE, "Instance type size must be above special values for encoding: " + size);
         int encoding = size;
         guaranteeEncoding(type, isInstance(encoding), "Instance type encoding must denote an instance");
-        guaranteeEncoding(type, !Continuation.isSupported() || !isStoredContinuation(encoding), "Instance type encoding must not denote a stored continuation");
         guaranteeEncoding(type, !isArray(encoding), "Instance type encoding must not denote an array");
         guaranteeEncoding(type, !isObjectArray(encoding), "Instance type encoding must not denote an object array");
         guaranteeEncoding(type, !isPrimitiveArray(encoding), "Instance type encoding must not denote a primitive array");
@@ -168,20 +157,9 @@ public static boolean isInstance(int encoding) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static UnsignedWord getInstanceSize(int encoding) {
-        boolean isStoredContinuation = Continuation.isSupported() && isStoredContinuation(encoding);
-        if (GraalDirectives.inIntrinsic()) {
-            ReplacementsUtil.dynamicAssert(!isStoredContinuation, "size cannot be determined from encoding");
-        } else {
-            assert !isStoredContinuation : "size cannot be determined from encoding";
-        }
         return WordFactory.unsigned(encoding);
     }
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static boolean isStoredContinuation(int encoding) {
-        return encoding == STORED_CONTINUATION_VALUE;
-    }
-
     // May be inlined because it does not deal in Pointers.
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArray(int encoding) {
@@ -241,8 +219,6 @@ public static UnsignedWord getSizeFromObjectInline(Object obj) {
         int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding();
         if (isArray(encoding)) {
             return getArraySize(encoding, ArrayLengthNode.arrayLength(obj));
-        } else if (Continuation.isSupported() && isStoredContinuation(encoding)) {
-            return WordFactory.unsigned(StoredContinuationImpl.readSize((StoredContinuation) obj));
         } else {
             return getInstanceSize(encoding);
         }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java
index fcc81b591faa..d84388f3a0a9 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java
@@ -35,12 +35,10 @@
 
 import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.annotate.AutomaticFeature;
-import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.heap.StoredContinuation;
 import com.oracle.svm.core.heap.StoredContinuationImpl;
 import com.oracle.svm.core.option.SubstrateOptionsParser;
 import com.oracle.svm.core.util.UserError;
-import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.util.ReflectionUtil;
 
 @AutomaticFeature
@@ -73,12 +71,11 @@ public void afterRegistration(AfterRegistrationAccess access) {
 
     @Override
     public void beforeAnalysis(BeforeAnalysisAccess access) {
+        // Must currently exist as HostedField for HybridLayout to work
+        access.registerAsAccessed(ReflectionUtil.lookupField(StoredContinuation.class, "payload"));
+
         if (Continuation.isSupported()) {
             access.registerAsInHeap(StoredContinuation.class);
-            VMError.guarantee(ConfigurationValues.getObjectLayout().getFirstFieldOffset() == StoredContinuationImpl.OBJECT_MONITOR_OFFSET,
-                            "Unexpected monitor field offset. Continuation support currently needs option " +
-                                            SubstrateOptionsParser.commandArgument(SubstrateOptions.SpawnIsolates, "+") +
-                                            " and compressed 32-bit reference support.");
 
             if (LoomSupport.isEnabled()) {
                 RuntimeReflection.register(ReflectionUtil.lookupMethod(ForkJoinPool.class, "compensatedBlock", ForkJoinPool.ManagedBlocker.class));
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
index 983fb0e03ecd..f33fd3c0cd08 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
@@ -1676,9 +1676,6 @@ private void printTypes() {
             } else if (LayoutEncoding.isPrimitiveArray(le)) {
                 System.out.format("primitive array base %d shift %d scale %d  ", LayoutEncoding.getArrayBaseOffset(le).rawValue(), LayoutEncoding.getArrayIndexShift(le),
                                 LayoutEncoding.getArrayIndexScale(le));
-            } else if (LayoutEncoding.isStoredContinuation(le)) {
-                // can exist as HostedType even if unreachable and continuations are disabled
-                System.out.print("stored continuation  ");
             } else {
                 throw VMError.shouldNotReachHere();
             }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
index 2de8664974b8..10dd3ec49eba 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
@@ -92,7 +92,6 @@
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.stackvalue.StackValueNode;
 import com.oracle.svm.core.graal.thread.VMThreadLocalAccess;
-import com.oracle.svm.core.heap.StoredContinuation;
 import com.oracle.svm.core.heap.Target_java_lang_ref_Reference;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.HubType;
@@ -462,8 +461,6 @@ private static HubType computeHubType(AnalysisType type) {
         } else if (type.isInstanceClass()) {
             if (Reference.class.isAssignableFrom(type.getJavaClass())) {
                 return HubType.InstanceReference;
-            } else if (type.getJavaClass().equals(StoredContinuation.class)) {
-                return HubType.StoredContinuation;
             }
             assert !Target_java_lang_ref_Reference.class.isAssignableFrom(type.getJavaClass()) : "should not see substitution type here";
             return HubType.Instance;
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
index 0c36b60d62f7..19ff6d3c55f4 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
@@ -26,7 +26,6 @@
 
 import java.lang.reflect.Modifier;
 
-import org.graalvm.collections.Pair;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
 
@@ -56,13 +55,7 @@ public boolean canHybridFieldsBeDuplicated(HostedType clazz) {
         return clazz.getAnnotation(Hybrid.class).canHybridFieldsBeDuplicated();
     }
 
-    /**
-     * Finds the hybrid array and bitset fields of a class annotated with {@link Hybrid}.
-     *
-     * @param hybridClass A class annotated with {@link Hybrid}
-     * @return A {@link Pair} containing the (non-null) hybrid array field in the left position, and
-     *         the (nullable) hybrid bitset field in the right position.
-     */
+    /** Finds the hybrid array and bitset fields of a hybrid class. */
     public HybridFields findHybridFields(HostedInstanceClass hybridClass) {
         assert hybridClass.getAnnotation(Hybrid.class) != null;
         assert Modifier.isFinal(hybridClass.getModifiers());
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
index 5c32a040e76b..7de8c5796ab2 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
@@ -421,7 +421,7 @@ private void layoutInstanceFields(HostedInstanceClass clazz, int superSize, Host
 
             /*
              * Set start after typecheck slots field, if the hybrid class has one. For now, only
-             * DynamicHubs can this field(s).
+             * DynamicHubs can have such field(s).
              */
             if (clazz.equals(hMetaAccess.lookupJavaType(DynamicHub.class))) {
                 /* Each type check id slot is 2 bytes. */
@@ -900,8 +900,6 @@ private void buildHubs() {
                     JavaKind storageKind = hybridLayout.getArrayElementStorageKind();
                     boolean isObject = (storageKind == JavaKind.Object);
                     layoutHelper = LayoutEncoding.forArray(type, isObject, hybridLayout.getArrayBaseOffset(), ol.getArrayIndexShift(storageKind));
-                } else if (instanceClass.getJavaClass().equals(StoredContinuation.class)) {
-                    layoutHelper = LayoutEncoding.forStoredContinuation();
                 } else {
                     layoutHelper = LayoutEncoding.forInstance(type, ConfigurationValues.getObjectLayout().alignUp(instanceClass.getInstanceSize()));
                 }

From 1106b1b0e197c3d74dd330d14e2079444933b681 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Fri, 8 Apr 2022 11:01:56 +0200
Subject: [PATCH 02/12] Introduce pods: runtime-modeled data objects.

---
 ...SubstrateDynamicNewHybridInstanceNode.java |  56 ++++
 .../snippets/SubstrateAllocationSnippets.java |  38 ++-
 .../src/com/oracle/svm/core/heap/Pod.java     | 302 ++++++++++++++++++
 .../svm/core/heap/PodReferenceMapDecoder.java |  65 ++++
 .../com/oracle/svm/core/hub/DynamicHub.java   |   5 +
 .../src/com/oracle/svm/core/hub/HubType.java  |   8 +-
 .../svm/core/hub/InteriorObjRefWalker.java    |   5 +
 .../src/com/oracle/svm/hosted/SVMHost.java    |   3 +
 8 files changed, 476 insertions(+), 6 deletions(-)
 create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java
 create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
 create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java

diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java
new file mode 100644
index 000000000000..3f4a0c3ee496
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.graal.nodes;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.java.AbstractNewArrayNode;
+
+@NodeInfo
+public final class SubstrateDynamicNewHybridInstanceNode extends AbstractNewArrayNode {
+    public static final NodeClass<SubstrateDynamicNewHybridInstanceNode> TYPE = NodeClass.create(SubstrateDynamicNewHybridInstanceNode.class);
+
+    @Input ValueNode instanceClass;
+    @Input ValueNode elementType;
+
+    public SubstrateDynamicNewHybridInstanceNode(ValueNode instanceClass, ValueNode elementType, ValueNode arrayLength) {
+        super(TYPE, StampFactory.objectNonNull(), arrayLength, true, null);
+        this.instanceClass = instanceClass;
+        this.elementType = elementType;
+    }
+
+    public ValueNode instanceType() {
+        return instanceClass;
+    }
+
+    public ValueNode elementType() {
+        return elementType;
+    }
+
+    @NodeIntrinsic
+    public static native Object allocate(Class<?> instanceClass, Class<?> elementType, int arrayLength);
+}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
index 0e18f8343256..01a089e1da95 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
@@ -79,6 +79,7 @@
 import com.oracle.svm.core.configure.ConfigurationFile;
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.nodes.ForeignCallWithExceptionNode;
+import com.oracle.svm.core.graal.nodes.SubstrateDynamicNewHybridInstanceNode;
 import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
 import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.hub.DynamicHub;
@@ -146,16 +147,19 @@ public Object allocateInstanceDynamic(@NonNullParameter DynamicHub hub, @Constan
     }
 
     @Snippet
-    public Object allocateArrayDynamic(DynamicHub elementType, int length, @ConstantParameter FillContent fillContents, @ConstantParameter boolean emitMemoryBarrier,
+    public Object allocateArrayDynamic(DynamicHub hub, DynamicHub elementType, int length, @ConstantParameter FillContent fillContents, @ConstantParameter boolean emitMemoryBarrier,
                     @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationProfilingData profilingData) {
-        DynamicHub checkedArrayHub = getCheckedArrayHub(elementType);
+        DynamicHub arrayHub = hub;
+        if (arrayHub == null) {
+            arrayHub = getCheckedArrayHub(elementType);
+        }
 
-        int layoutEncoding = checkedArrayHub.getLayoutEncoding();
+        int layoutEncoding = arrayHub.getLayoutEncoding();
         int arrayBaseOffset = getArrayBaseOffset(layoutEncoding);
         int log2ElementSize = LayoutEncoding.getArrayIndexShift(layoutEncoding);
 
-        Object result = allocateArrayImpl(encodeAsTLABObjectHeader(checkedArrayHub), length, arrayBaseOffset, log2ElementSize, fillContents,
-                        arrayBaseOffset, emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, profilingData);
+        Object result = allocateArrayImpl(encodeAsTLABObjectHeader(arrayHub), length, arrayBaseOffset, log2ElementSize, fillContents,
+                        afterArrayLengthOffset(), emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, profilingData);
         return piArrayCastToSnippetReplaceeStamp(result, length);
     }
 
@@ -399,6 +403,7 @@ public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>
             lowerings.put(NewArrayNode.class, new NewArrayLowering());
             lowerings.put(DynamicNewInstanceNode.class, new DynamicNewInstanceLowering());
             lowerings.put(DynamicNewArrayNode.class, new DynamicNewArrayLowering());
+            lowerings.put(SubstrateDynamicNewHybridInstanceNode.class, new DynamicNewHybridInstanceLowering());
             lowerings.put(NewMultiArrayNode.class, new NewMultiArrayLowering());
             lowerings.put(ValidateNewInstanceClassNode.class, new ValidateNewInstanceClassLowering());
         }
@@ -578,6 +583,7 @@ public void lower(DynamicNewArrayNode node, LoweringTool tool) {
                 }
 
                 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
+                args.add("hub", ConstantNode.defaultForKind(JavaKind.Object, graph));
                 args.add("elementType", node.getElementType());
                 args.add("length", node.length());
                 args.addConst("fillContents", FillContent.fromBoolean(node.fillContents()));
@@ -590,6 +596,28 @@ public void lower(DynamicNewArrayNode node, LoweringTool tool) {
             }
         }
 
+        private class DynamicNewHybridInstanceLowering implements NodeLoweringProvider<SubstrateDynamicNewHybridInstanceNode> {
+            @Override
+            public void lower(SubstrateDynamicNewHybridInstanceNode node, LoweringTool tool) {
+                StructuredGraph graph = node.graph();
+                if (graph.getGuardsStage() != StructuredGraph.GuardsStage.AFTER_FSA) {
+                    return;
+                }
+
+                Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
+                args.add("hub", node.instanceType());
+                args.add("elementType", node.elementType());
+                args.add("length", node.length());
+                args.addConst("fillContents", FillContent.fromBoolean(node.fillContents()));
+                args.addConst("emitMemoryBarrier", node.emitMemoryBarrier());
+                args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
+                args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
+                args.addConst("profilingData", getProfilingData(node, null));
+
+                template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
+            }
+        }
+
         private class ValidateNewInstanceClassLowering implements NodeLoweringProvider<ValidateNewInstanceClassNode> {
             @Override
             public void lower(ValidateNewInstanceClassNode node, LoweringTool tool) {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
new file mode 100644
index 000000000000..59939f2d0ecc
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -0,0 +1,302 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.heap;
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import org.graalvm.nativeimage.hosted.Feature;
+import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
+
+import com.oracle.svm.core.JavaMemoryUtil;
+import com.oracle.svm.core.annotate.AutomaticFeature;
+import com.oracle.svm.core.annotate.Hybrid;
+import com.oracle.svm.core.config.ConfigurationValues;
+import com.oracle.svm.core.graal.nodes.SubstrateDynamicNewHybridInstanceNode;
+import com.oracle.svm.core.hub.DynamicHub;
+import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.util.UnsignedUtils;
+import com.oracle.svm.util.ReflectionUtil;
+
+import jdk.vm.ci.meta.JavaKind;
+
+@AutomaticFeature
+final class PodFeature implements Feature {
+    @Override
+    public void beforeAnalysis(BeforeAnalysisAccess access) {
+        // needed for HybridLayout
+        access.registerAsAccessed(ReflectionUtil.lookupField(Pod.Instance.class, "fields"));
+
+        access.registerReachabilityHandler(a -> a.registerAsInHeap(Pod.Instance.class),
+                        ReflectionUtil.lookupMethod(Pod.class, "newInstance"));
+    }
+}
+
+/**
+ * A structure of fields, including object references, that is {@linkplain Builder modeled} at image
+ * runtime and instances of which are allocated on the Java heap. The name <em>pod</em> refers to it
+ * storing "plain old data" without further object-oriented features such as method dispatching or
+ * type information, apart from having a Java superclass.
+ */
+public final class Pod<T> {
+    @Hybrid
+    public static final class Instance {
+        @Hybrid.Array byte[] fields;
+
+        private Instance() {
+        }
+    }
+
+    private final Class<T> superClass;
+    private final int fieldsSizeWithoutRefMap;
+    private final byte[] referenceMap;
+
+    private Pod(Class<T> superClass, int fieldsSize, byte[] referenceMap) {
+        this.superClass = superClass;
+        this.fieldsSizeWithoutRefMap = fieldsSize;
+        this.referenceMap = referenceMap;
+    }
+
+    public T newInstance() {
+        @SuppressWarnings("unchecked")
+        T instance = (T) SubstrateDynamicNewHybridInstanceNode.allocate(superClass, byte.class, fieldsSizeWithoutRefMap + referenceMap.length);
+
+        UnsignedWord podRefMapBase = getArrayBaseOffset(superClass).add(fieldsSizeWithoutRefMap);
+        JavaMemoryUtil.copy(referenceMap, getArrayBaseOffset(byte[].class), instance, podRefMapBase, WordFactory.unsigned(referenceMap.length));
+
+        return instance;
+    }
+
+    private static UnsignedWord getArrayBaseOffset(Class<?> clazz) {
+        DynamicHub hub = DynamicHub.fromClass(clazz);
+        return LayoutEncoding.getArrayBaseOffset(hub.getLayoutEncoding());
+    }
+
+    public static final class Builder {
+        private final Pod<?> superPod;
+        private final Class<Instance> superClass = Instance.class;
+        private final List<Field> fields = new ArrayList<>();
+        private boolean built = false;
+
+        public Builder() {
+            this(null);
+        }
+
+        public Builder(Pod<?> superPod) {
+            this.superPod = superPod;
+        }
+
+        private void guaranteeUnbuilt() {
+            if (built) {
+                throw new IllegalStateException();
+            }
+        }
+
+        public Field addField(Class<?> type) {
+            guaranteeUnbuilt();
+            Objects.requireNonNull(type);
+            if (type == void.class) {
+                throw new IllegalArgumentException("void is an illegal field type");
+            }
+
+            JavaKind kind = JavaKind.fromJavaClass(type);
+            int size = ConfigurationValues.getObjectLayout().sizeInBytes(kind);
+            Field f = new Field(size, kind.isObject());
+            fields.add(f);
+            return f;
+        }
+
+        public Pod<Instance> build() {
+            guaranteeUnbuilt();
+            built = true;
+
+            Collections.sort(fields);
+
+            UnsignedWord baseOffset = getArrayBaseOffset(superClass);
+
+            byte[] superRefMap = null;
+            UnsignedWord nextOffset = baseOffset;
+            if (superPod != null) {
+                nextOffset = nextOffset.add(superPod.fieldsSizeWithoutRefMap);
+                superRefMap = superPod.referenceMap;
+            }
+            ReferenceMapEncoder refMapEncoder = new ReferenceMapEncoder(superRefMap);
+            while (!fields.isEmpty()) {
+                boolean progress = false;
+                for (int i = 0; i < fields.size(); i++) {
+                    Field field = fields.get(i);
+
+                    if (nextOffset.unsignedRemainder(field.size).equal(0)) {
+                        field.initOffset(UnsignedUtils.safeToInt(nextOffset));
+
+                        if (field.isReference) {
+                            refMapEncoder.add(UnsignedUtils.safeToInt(nextOffset.subtract(baseOffset)), field.size);
+                        }
+
+                        fields.remove(i);
+                        nextOffset = nextOffset.add(field.size);
+                        progress = true;
+                        break;
+                    }
+                }
+                if (!progress) {
+                    nextOffset = nextOffset.add(1); // insert padding byte and retry
+                }
+            }
+
+            byte[] referenceMap = refMapEncoder.encode();
+            return new Pod<>(superClass, UnsignedUtils.safeToInt(nextOffset), referenceMap);
+        }
+    }
+
+    public static final class Field implements Comparable<Field> {
+        private final int size;
+        private final boolean isReference;
+        private int offset = -1;
+
+        Field(int size, boolean isReference) {
+            assert size > 0;
+            this.size = size;
+            this.isReference = isReference;
+        }
+
+        public int getSize() {
+            return size;
+        }
+
+        public boolean isReference() {
+            return isReference;
+        }
+
+        public int getOffset() {
+            if (offset == -1) {
+                throw new IllegalStateException("Pod must be built before field offsets are assigned");
+            }
+            return offset;
+        }
+
+        void initOffset(int value) {
+            assert this.offset == -1;
+            assert value >= 0;
+            this.offset = value;
+        }
+
+        @Override
+        public int compareTo(Field f) {
+            if (isReference != f.isReference) { // references first
+                return Boolean.compare(f.isReference, isReference);
+            }
+            return f.size - size; // larger fields first
+        }
+    }
+
+    private static final class ReferenceMapEncoder {
+        private final BitSet bitset = new BitSet();
+        private final int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
+
+        ReferenceMapEncoder(byte[] referenceMap) {
+            if (referenceMap != null) {
+                decode(referenceMap);
+            }
+        }
+
+        void add(int offset, int size) {
+            assert offset % referenceSize == 0;
+            assert size == referenceSize;
+
+            int index = offset / referenceSize;
+            assert !bitset.get(index);
+            bitset.set(index);
+        }
+
+        byte[] encode() {
+            if (bitset.isEmpty()) {
+                return new byte[]{0, 0};
+            }
+
+            ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+            int previous;
+            int index = 0;
+            for (;;) { // assume references are placed first
+                previous = index;
+                index = bitset.nextClearBit(previous);
+                int nrefs = index - previous;
+                putUV(buffer, nrefs);
+
+                previous = index;
+                index = bitset.nextSetBit(index);
+                if (index != -1) {
+                    putUV(buffer, index - previous); // gap
+                } else {
+                    buffer.write(0);
+                    if ((nrefs & 0xff) == 0) { // needs an explicit end marker
+                        buffer.write(0);
+                        buffer.write(0);
+                    }
+                    break;
+                }
+            }
+
+            // reverse so we can decode backwards from the known end
+            byte[] bytes = buffer.toByteArray();
+            for (int i = 0, j = bytes.length - 1; i < j; i++, j--) {
+                byte t = bytes[i];
+                bytes[i] = bytes[j];
+                bytes[j] = t;
+            }
+
+            assert bitset.equals(new ReferenceMapEncoder(bytes).bitset);
+            return bytes;
+        }
+
+        private static void putUV(ByteArrayOutputStream buffer, int value) {
+            int v = value;
+            for (; v > 0xff; v -= 0xff) { // should be rare
+                buffer.write(0xff);
+                buffer.write(0);
+            }
+            buffer.write(v);
+        }
+
+        private void decode(byte[] encoded) {
+            int nrefs;
+            int gap;
+            int bit = 0;
+            int i = encoded.length - 1;
+            do {
+                nrefs = Byte.toUnsignedInt(encoded[i]);
+                gap = Byte.toUnsignedInt(encoded[i - 1]);
+                i -= 2;
+                bitset.set(bit, bit + nrefs);
+                bit += nrefs + gap;
+            } while (gap != 0 || nrefs == 0xff);
+        }
+    }
+}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
new file mode 100644
index 000000000000..0d41c7027029
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.heap;
+
+import org.graalvm.compiler.nodes.java.ArrayLengthNode;
+import org.graalvm.word.Pointer;
+import org.graalvm.word.UnsignedWord;
+
+import com.oracle.svm.core.annotate.AlwaysInline;
+import com.oracle.svm.core.config.ConfigurationValues;
+import com.oracle.svm.core.hub.LayoutEncoding;
+
+public final class PodReferenceMapDecoder {
+    @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor")
+    public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) {
+        int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
+        boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences();
+
+        UnsignedWord refOffset = LayoutEncoding.getArrayBaseOffset(layoutEncoding);
+        UnsignedWord mapOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, ArrayLengthNode.arrayLength(obj) - 1);
+
+        int nrefs;
+        int gap;
+        do {
+            nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset));
+            gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.subtract(1)));
+            mapOffset = mapOffset.subtract(2);
+
+            for (int i = 0; i < nrefs; i++) {
+                if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) {
+                    return false;
+                }
+                refOffset = refOffset.add(referenceSize);
+            }
+            refOffset = refOffset.add(referenceSize * gap);
+        } while (gap != 0 || nrefs == 0xff);
+
+        return true;
+    }
+
+    private PodReferenceMapDecoder() {
+    }
+}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java
index 6bd0312dc64b..73ea3513ab45 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java
@@ -611,6 +611,11 @@ public boolean isInstanceClass() {
         return HubType.isInstance(hubType);
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public boolean isPodInstanceClass() {
+        return HubType.isPodInstance(hubType);
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean isReferenceInstanceClass() {
         return HubType.isReferenceInstance(hubType);
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java
index 20cf74467fbe..ca902e0e42a7 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/HubType.java
@@ -34,6 +34,7 @@ public enum HubType {
     InstanceReference(1),
 
     // special hubs
+    PodInstance(2),
     Other(3),
 
     // array hubs
@@ -53,7 +54,7 @@ public int getValue() {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isInstance(int hubType) {
-        return hubType <= InstanceReference.getValue();
+        return hubType <= PodInstance.getValue();
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
@@ -61,6 +62,11 @@ public static boolean isReferenceInstance(int hubType) {
         return hubType == InstanceReference.getValue();
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean isPodInstance(int hubType) {
+        return hubType == PodInstance.getValue();
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArray(int hubType) {
         return hubType >= TypeArray.getValue();
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
index 9522479008f7..632168282fb5 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
@@ -36,6 +36,7 @@
 import com.oracle.svm.core.heap.InstanceReferenceMapDecoder;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
+import com.oracle.svm.core.heap.PodReferenceMapDecoder;
 import com.oracle.svm.core.heap.ReferenceAccess;
 
 /**
@@ -79,6 +80,10 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi
                 }
                 pos = pos.add(referenceSize);
             }
+        } else if (objHub.isPodInstanceClass()) {
+            if (!PodReferenceMapDecoder.walkOffsetsFromPointer(objPointer, layoutEncoding, visitor, obj)) {
+                return false;
+            }
         }
 
         NonmovableArray<Byte> referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding();
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
index 10dd3ec49eba..91f2f3ec57cd 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
@@ -92,6 +92,7 @@
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.stackvalue.StackValueNode;
 import com.oracle.svm.core.graal.thread.VMThreadLocalAccess;
+import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.Target_java_lang_ref_Reference;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.HubType;
@@ -461,6 +462,8 @@ private static HubType computeHubType(AnalysisType type) {
         } else if (type.isInstanceClass()) {
             if (Reference.class.isAssignableFrom(type.getJavaClass())) {
                 return HubType.InstanceReference;
+            } else if (Pod.Instance.class.isAssignableFrom(type.getJavaClass())) {
+                return HubType.PodInstance;
             }
             assert !Target_java_lang_ref_Reference.class.isAssignableFrom(type.getJavaClass()) : "should not see substitution type here";
             return HubType.Instance;

From 199a9002bbe55e21ff354a610af3a175c0113186 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Tue, 12 Apr 2022 16:29:38 +0200
Subject: [PATCH 03/12] Do not require a @Hybrid.Array field in a hybrid class.

---
 .../com/oracle/svm/core/annotate/Hybrid.java  | 29 +++++++++--------
 .../src/com/oracle/svm/core/heap/Pod.java     |  7 +---
 .../svm/core/heap/StoredContinuation.java     |  4 +--
 .../svm/core/thread/ContinuationsFeature.java |  6 ++--
 .../svm/hosted/config/HybridLayout.java       | 30 ++++++++---------
 .../hosted/config/HybridLayoutSupport.java    | 32 +++++++++++++------
 .../svm/hosted/image/NativeImageHeap.java     |  2 +-
 .../svm/hosted/meta/UniverseBuilder.java      |  2 +-
 .../svm/jni/access/JNIAccessibleField.java    |  5 +--
 9 files changed, 63 insertions(+), 54 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
index 8f0c0172e8a7..96a01de95af8 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
@@ -68,29 +68,32 @@
 public @interface Hybrid {
 
     /**
-     * If {@code true}, we allow the data in the hybrid fields to be duplicated between the hybrid
-     * object and a separate object for the array. For most objects, a duplication could occur if
-     * inlining and constant folding result in the internal reference to a hybrid field being
-     * constant folded to a constant value, which must be written into the image heap separately
-     * from the hybrid object.
+     * The type of the array part of the hybrid class. Must be specified if no field annotated
+     * with @{@link Hybrid.Array} is declared, otherwise that field's type determines the type of
+     * the array part.
+     */
+    Class<?> arrayType() default Array.class;
+
+    /**
+     * If {@code true}, allow the data in the hybrid fields to be duplicated between the hybrid
+     * object and a separate object for the array. For image heap objects, a duplication can occur
+     * if inlining and constant folding result in the internal reference to a hybrid field being
+     * folded to a constant value, which must be written into the image heap separately from the
+     * hybrid object.
      *
      * If {@code false}, a duplication of the hybrid fields must never happen.
      */
     boolean canHybridFieldsBeDuplicated() default false;
 
-    /**
-     * Specifies a single member array as the hybrid array.
-     */
+    /** Designates at most one field that refers to the array part of the hybrid object. */
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.FIELD)
-    public @interface Array {
+    @interface Array {
     }
 
-    /**
-     * Specifies a single member type slots.
-     */
+    /** Designates at most one field that refers to the type ID slots of the hybrid object. */
     @Retention(RetentionPolicy.RUNTIME)
     @Target(ElementType.FIELD)
-    public @interface TypeIDSlots {
+    @interface TypeIDSlots {
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index 59939f2d0ecc..c5fe1ae65b6c 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -51,9 +51,6 @@
 final class PodFeature implements Feature {
     @Override
     public void beforeAnalysis(BeforeAnalysisAccess access) {
-        // needed for HybridLayout
-        access.registerAsAccessed(ReflectionUtil.lookupField(Pod.Instance.class, "fields"));
-
         access.registerReachabilityHandler(a -> a.registerAsInHeap(Pod.Instance.class),
                         ReflectionUtil.lookupMethod(Pod.class, "newInstance"));
     }
@@ -66,10 +63,8 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
  * type information, apart from having a Java superclass.
  */
 public final class Pod<T> {
-    @Hybrid
+    @Hybrid(arrayType = byte[].class)
     public static final class Instance {
-        @Hybrid.Array byte[] fields;
-
         private Instance() {
         }
     }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
index 2a1605796571..2e4b9159d522 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
@@ -31,10 +31,8 @@
  *
  * For object layout and other implementation details, see {@link StoredContinuationImpl}.
  */
-@Hybrid
+@Hybrid(arrayType = byte[].class)
 public final class StoredContinuation {
-    @Hybrid.Array byte[] payload;
-
     /** Must be allocated via {@link StoredContinuationImpl}. */
     private StoredContinuation() {
     }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java
index d84388f3a0a9..060e0036d472 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ContinuationsFeature.java
@@ -71,11 +71,9 @@ public void afterRegistration(AfterRegistrationAccess access) {
 
     @Override
     public void beforeAnalysis(BeforeAnalysisAccess access) {
-        // Must currently exist as HostedField for HybridLayout to work
-        access.registerAsAccessed(ReflectionUtil.lookupField(StoredContinuation.class, "payload"));
-
         if (Continuation.isSupported()) {
-            access.registerAsInHeap(StoredContinuation.class);
+            access.registerReachabilityHandler(a -> access.registerAsInHeap(StoredContinuation.class),
+                            ReflectionUtil.lookupMethod(StoredContinuationImpl.class, "allocate", int.class));
 
             if (LoomSupport.isEnabled()) {
                 RuntimeReflection.register(ReflectionUtil.lookupMethod(ForkJoinPool.class, "compensatedBlock", ForkJoinPool.ManagedBlocker.class));
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java
index 94262cb89309..8a7a57784ee2 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayout.java
@@ -34,15 +34,13 @@
 import com.oracle.svm.hosted.meta.HostedType;
 
 import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
- * Defines the layout for a hybrid class.
+ * Provides sizes and offsets of a hybrid class.
  *
  * @see Hybrid
- *
- * @param <T> The class which has a layout in hybrid form. It must be annotated with the
- *            {@link Hybrid} annotation.
  */
 public class HybridLayout<T> {
 
@@ -59,25 +57,30 @@ public static boolean canHybridFieldsBeDuplicated(HostedType clazz) {
     }
 
     private final ObjectLayout layout;
+    private final HostedType arrayType;
     private final HostedField arrayField;
     private final HostedField typeIDSlotsField;
     private final int arrayBaseOffset;
 
     public HybridLayout(Class<T> hybridClass, ObjectLayout layout, HostedMetaAccess metaAccess) {
-        this((HostedInstanceClass) metaAccess.lookupJavaType(hybridClass), layout);
+        this((HostedInstanceClass) metaAccess.lookupJavaType(hybridClass), layout, metaAccess);
     }
 
-    public HybridLayout(HostedInstanceClass hybridClass, ObjectLayout layout) {
+    public HybridLayout(HostedInstanceClass hybridClass, ObjectLayout layout, MetaAccessProvider metaAccess) {
         this.layout = layout;
-        HybridLayoutSupport utils = HybridLayoutSupport.singleton();
-        HybridLayoutSupport.HybridFields hybridFields = utils.findHybridFields(hybridClass);
-        arrayField = hybridFields.arrayField;
-        typeIDSlotsField = hybridFields.typeIDSlotsField;
-        arrayBaseOffset = NumUtil.roundUp(hybridClass.getAfterFieldsOffset(), layout.sizeInBytes(getArrayElementStorageKind()));
+        HybridLayoutSupport.HybridInfo hybridInfo = HybridLayoutSupport.singleton().inspectHybrid(hybridClass, metaAccess);
+        this.arrayType = hybridInfo.arrayType;
+        this.arrayField = hybridInfo.arrayField;
+        this.typeIDSlotsField = hybridInfo.typeIDSlotsField;
+        this.arrayBaseOffset = NumUtil.roundUp(hybridClass.getAfterFieldsOffset(), layout.sizeInBytes(getArrayElementStorageKind()));
+    }
+
+    public HostedType getArrayType() {
+        return arrayType;
     }
 
     public JavaKind getArrayElementStorageKind() {
-        return arrayField.getType().getComponentType().getStorageKind();
+        return arrayType.getComponentType().getStorageKind();
     }
 
     public int getArrayBaseOffset() {
@@ -100,9 +103,6 @@ public HostedField getTypeIDSlotsField() {
         return typeIDSlotsField;
     }
 
-    /**
-     * In a given build, only the bit field or the type id slot array field will exist.
-     */
     public static int getTypeIDSlotsFieldOffset(ObjectLayout layout) {
         return layout.getArrayLengthOffset() + layout.sizeInBytes(JavaKind.Int);
     }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
index 19ff6d3c55f4..cab39a4fe50b 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
@@ -30,10 +30,12 @@
 import org.graalvm.nativeimage.ImageSingletons;
 
 import com.oracle.svm.core.annotate.Hybrid;
+import com.oracle.svm.hosted.meta.HostedArrayClass;
 import com.oracle.svm.hosted.meta.HostedField;
 import com.oracle.svm.hosted.meta.HostedInstanceClass;
 import com.oracle.svm.hosted.meta.HostedType;
 
+import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 public class HybridLayoutSupport {
@@ -55,17 +57,17 @@ public boolean canHybridFieldsBeDuplicated(HostedType clazz) {
         return clazz.getAnnotation(Hybrid.class).canHybridFieldsBeDuplicated();
     }
 
-    /** Finds the hybrid array and bitset fields of a hybrid class. */
-    public HybridFields findHybridFields(HostedInstanceClass hybridClass) {
-        assert hybridClass.getAnnotation(Hybrid.class) != null;
+    /** Determines characteristics of a hybrid class. */
+    protected HybridInfo inspectHybrid(HostedInstanceClass hybridClass, MetaAccessProvider metaAccess) {
+        Hybrid annotation = hybridClass.getAnnotation(Hybrid.class);
+        assert annotation != null;
         assert Modifier.isFinal(hybridClass.getModifiers());
 
         HostedField foundArrayField = null;
         HostedField foundTypeIDSlotsField = null;
         for (HostedField field : hybridClass.getInstanceFields(true)) {
             if (field.getAnnotation(Hybrid.Array.class) != null) {
-                assert foundArrayField == null : "must have exactly one hybrid array field";
-                assert field.getType().isArray();
+                assert foundArrayField == null : "must have at most one hybrid array field";
                 foundArrayField = field;
             }
             if (field.getAnnotation(Hybrid.TypeIDSlots.class) != null) {
@@ -74,15 +76,27 @@ public HybridFields findHybridFields(HostedInstanceClass hybridClass) {
                 foundTypeIDSlotsField = field;
             }
         }
-        assert foundArrayField != null : "must have exactly one hybrid array field";
-        return new HybridFields(foundArrayField, foundTypeIDSlotsField);
+
+        HostedType arrayType;
+        boolean arrayTypeIsSet = (annotation.arrayType() != Hybrid.Array.class);
+        if (foundArrayField != null) {
+            arrayType = foundArrayField.getType();
+            assert !arrayTypeIsSet || arrayType.equals(metaAccess.lookupJavaType(annotation.arrayType())) : "@Hybrid.arrayType must match the type of a @Hybrid.Array field when both are present";
+        } else {
+            assert arrayTypeIsSet : "@Hybrid.arrayType must be set when no @Hybrid.Array field is present (if present, ensure it is reachable)";
+            arrayType = (HostedArrayClass) metaAccess.lookupJavaType(annotation.arrayType());
+        }
+        assert arrayType.isArray();
+        return new HybridInfo(arrayType, foundArrayField, foundTypeIDSlotsField);
     }
 
-    public static class HybridFields {
+    public static class HybridInfo {
+        public final HostedType arrayType;
         public final HostedField arrayField;
         public final HostedField typeIDSlotsField;
 
-        public HybridFields(HostedField arrayField, HostedField typeIDSlotsField) {
+        public HybridInfo(HostedType arrayType, HostedField arrayField, HostedField typeIDSlotsField) {
+            this.arrayType = arrayType;
             this.arrayField = arrayField;
             this.typeIDSlotsField = typeIDSlotsField;
         }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java
index ef5e14f0e7bc..054e701d5fd5 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java
@@ -392,7 +392,7 @@ private void addObjectToImageHeap(final Object object, boolean immutableFromPare
             if (HybridLayout.isHybrid(clazz)) {
                 HybridLayout<?> hybridLayout = hybridLayouts.get(clazz);
                 if (hybridLayout == null) {
-                    hybridLayout = new HybridLayout<>(clazz, objectLayout);
+                    hybridLayout = new HybridLayout<>(clazz, objectLayout, metaAccess);
                     hybridLayouts.put(clazz, hybridLayout);
                 }
 
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
index 7de8c5796ab2..8991afbbf0c7 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
@@ -896,7 +896,7 @@ private void buildHubs() {
                 if (instanceClass.isAbstract()) {
                     layoutHelper = LayoutEncoding.forAbstract();
                 } else if (HybridLayout.isHybrid(type)) {
-                    HybridLayout<?> hybridLayout = new HybridLayout<>(instanceClass, ol);
+                    HybridLayout<?> hybridLayout = new HybridLayout<>(instanceClass, ol, hMetaAccess);
                     JavaKind storageKind = hybridLayout.getArrayElementStorageKind();
                     boolean isObject = (storageKind == JavaKind.Object);
                     layoutHelper = LayoutEncoding.forArray(type, isObject, hybridLayout.getArrayBaseOffset(), ol.getArrayIndexShift(storageKind));
diff --git a/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/access/JNIAccessibleField.java b/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/access/JNIAccessibleField.java
index b85e7756af29..d93da05f3056 100644
--- a/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/access/JNIAccessibleField.java
+++ b/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/access/JNIAccessibleField.java
@@ -111,8 +111,9 @@ void finishBeforeCompilation(CompilationAccessImpl access) {
             int offset;
             if (HybridLayout.isHybridField(field)) {
                 assert !field.hasLocation();
-                HybridLayout<?> hybridLayout = new HybridLayout<>((HostedInstanceClass) field.getDeclaringClass(), ImageSingletons.lookup(ObjectLayout.class));
-                assert field.equals(hybridLayout.getArrayField()) : "JNI access to hybrid bitset field is not implemented";
+                HybridLayout<?> hybridLayout = new HybridLayout<>((HostedInstanceClass) field.getDeclaringClass(),
+                                ImageSingletons.lookup(ObjectLayout.class), access.getMetaAccess());
+                assert field.equals(hybridLayout.getArrayField()) : "JNI access to hybrid objects is implemented only for the array field";
                 offset = hybridLayout.getArrayBaseOffset();
             } else {
                 assert field.hasLocation();

From 90b37158693647bce49d64d0704342b0f65ec299 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Tue, 12 Apr 2022 13:33:22 +0200
Subject: [PATCH 04/12] Implement arbitrary superclasses for Pods (known at
 build time) with subclasses generation.

---
 .../src/com/oracle/svm/core/heap/Pod.java     |  96 ++++++----
 .../src/com/oracle/svm/hosted/SVMHost.java    |   4 +-
 .../oracle/svm/hosted/heap/PodSupport.java    | 166 ++++++++++++++++++
 .../svm/hosted/meta/UniverseBuilder.java      |   7 +-
 4 files changed, 233 insertions(+), 40 deletions(-)
 create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java

diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index c5fe1ae65b6c..5b15b8076970 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -31,31 +31,24 @@
 import java.util.List;
 import java.util.Objects;
 
-import org.graalvm.nativeimage.hosted.Feature;
+import org.graalvm.collections.EconomicMap;
+import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.Platform;
+import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.JavaMemoryUtil;
-import com.oracle.svm.core.annotate.AutomaticFeature;
-import com.oracle.svm.core.annotate.Hybrid;
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.graal.nodes.SubstrateDynamicNewHybridInstanceNode;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.util.ImageHeapMap;
 import com.oracle.svm.core.util.UnsignedUtils;
-import com.oracle.svm.util.ReflectionUtil;
+import com.oracle.svm.core.util.VMError;
 
 import jdk.vm.ci.meta.JavaKind;
 
-@AutomaticFeature
-final class PodFeature implements Feature {
-    @Override
-    public void beforeAnalysis(BeforeAnalysisAccess access) {
-        access.registerReachabilityHandler(a -> a.registerAsInHeap(Pod.Instance.class),
-                        ReflectionUtil.lookupMethod(Pod.class, "newInstance"));
-    }
-}
-
 /**
  * A structure of fields, including object references, that is {@linkplain Builder modeled} at image
  * runtime and instances of which are allocated on the Java heap. The name <em>pod</em> refers to it
@@ -63,27 +56,21 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
  * type information, apart from having a Java superclass.
  */
 public final class Pod<T> {
-    @Hybrid(arrayType = byte[].class)
-    public static final class Instance {
-        private Instance() {
-        }
-    }
-
-    private final Class<T> superClass;
+    private final Class<? extends T> podClass;
     private final int fieldsSizeWithoutRefMap;
     private final byte[] referenceMap;
 
-    private Pod(Class<T> superClass, int fieldsSize, byte[] referenceMap) {
-        this.superClass = superClass;
+    private Pod(Class<? extends T> podClass, int fieldsSize, byte[] referenceMap) {
+        this.podClass = podClass;
         this.fieldsSizeWithoutRefMap = fieldsSize;
         this.referenceMap = referenceMap;
     }
 
     public T newInstance() {
         @SuppressWarnings("unchecked")
-        T instance = (T) SubstrateDynamicNewHybridInstanceNode.allocate(superClass, byte.class, fieldsSizeWithoutRefMap + referenceMap.length);
+        T instance = (T) SubstrateDynamicNewHybridInstanceNode.allocate(podClass, byte.class, fieldsSizeWithoutRefMap + referenceMap.length);
 
-        UnsignedWord podRefMapBase = getArrayBaseOffset(superClass).add(fieldsSizeWithoutRefMap);
+        UnsignedWord podRefMapBase = getArrayBaseOffset(podClass).add(fieldsSizeWithoutRefMap);
         JavaMemoryUtil.copy(referenceMap, getArrayBaseOffset(byte[].class), instance, podRefMapBase, WordFactory.unsigned(referenceMap.length));
 
         return instance;
@@ -94,17 +81,37 @@ private static UnsignedWord getArrayBaseOffset(Class<?> clazz) {
         return LayoutEncoding.getArrayBaseOffset(hub.getLayoutEncoding());
     }
 
-    public static final class Builder {
-        private final Pod<?> superPod;
-        private final Class<Instance> superClass = Instance.class;
-        private final List<Field> fields = new ArrayList<>();
-        private boolean built = false;
+    public static final class Builder<T> {
+        public static Builder<Object> create() {
+            return new Builder<>(Object.class, null);
+        }
+
+        public static <T> Builder<T> createExtending(Pod<T> superPod) {
+            return new Builder<>(null, superPod);
+        }
 
-        public Builder() {
-            this(null);
+        public static <T> Builder<T> createExtending(Class<T> superClass) {
+            return new Builder<>(superClass, null);
         }
 
-        public Builder(Pod<?> superPod) {
+        private final Pod<T> superPod;
+        private final Class<? extends T> podClass;
+        private final List<Field> fields = new ArrayList<>();
+        private boolean built = false;
+
+        private Builder(Class<T> superClass, Pod<T> superPod) {
+            if (superClass == null && superPod == null) {
+                throw new NullPointerException();
+            }
+            if (superPod != null) {
+                this.podClass = superPod.podClass;
+            } else {
+                this.podClass = ImageSingletons.lookup(RuntimeSupport.class).get(superClass);
+                if (this.podClass == null) {
+                    throw new IllegalArgumentException("Pod superclass was not registered during image build: " + superClass);
+                }
+            }
+            assert DynamicHub.fromClass(this.podClass).isPodInstanceClass();
             this.superPod = superPod;
         }
 
@@ -128,13 +135,13 @@ public Field addField(Class<?> type) {
             return f;
         }
 
-        public Pod<Instance> build() {
+        public Pod<T> build() {
             guaranteeUnbuilt();
             built = true;
 
             Collections.sort(fields);
 
-            UnsignedWord baseOffset = getArrayBaseOffset(superClass);
+            UnsignedWord baseOffset = getArrayBaseOffset(podClass);
 
             byte[] superRefMap = null;
             UnsignedWord nextOffset = baseOffset;
@@ -167,7 +174,7 @@ public Pod<Instance> build() {
             }
 
             byte[] referenceMap = refMapEncoder.encode();
-            return new Pod<>(superClass, UnsignedUtils.safeToInt(nextOffset), referenceMap);
+            return new Pod<>(podClass, UnsignedUtils.safeToInt(nextOffset), referenceMap);
         }
     }
 
@@ -212,6 +219,25 @@ public int compareTo(Field f) {
         }
     }
 
+    public static final class RuntimeSupport {
+        private final EconomicMap<Class<?>, Class<?>> superClasses = ImageHeapMap.create();
+
+        @Platforms(Platform.HOSTED_ONLY.class)
+        public RuntimeSupport() {
+        }
+
+        @Platforms(Platform.HOSTED_ONLY.class)
+        public boolean registerClass(Class<?> superClass, Class<?> podClass) {
+            VMError.guarantee(podClass.getSuperclass() == superClass);
+            return superClasses.putIfAbsent(superClass, podClass) == null;
+        }
+
+        @SuppressWarnings("unchecked")
+        public <T> Class<? extends T> get(Class<T> clazz) {
+            return (Class<? extends T>) superClasses.get(clazz);
+        }
+    }
+
     private static final class ReferenceMapEncoder {
         private final BitSet bitset = new BitSet();
         private final int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
index 91f2f3ec57cd..773866199635 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
@@ -92,7 +92,6 @@
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.stackvalue.StackValueNode;
 import com.oracle.svm.core.graal.thread.VMThreadLocalAccess;
-import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.Target_java_lang_ref_Reference;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.HubType;
@@ -107,6 +106,7 @@
 import com.oracle.svm.hosted.classinitialization.ClassInitializationSupport;
 import com.oracle.svm.hosted.code.InliningUtilities;
 import com.oracle.svm.hosted.code.UninterruptibleAnnotationChecker;
+import com.oracle.svm.hosted.heap.PodSupport;
 import com.oracle.svm.hosted.meta.HostedType;
 import com.oracle.svm.hosted.phases.AnalysisGraphBuilderPhase;
 import com.oracle.svm.hosted.phases.ImplicitAssertionsPhase;
@@ -462,7 +462,7 @@ private static HubType computeHubType(AnalysisType type) {
         } else if (type.isInstanceClass()) {
             if (Reference.class.isAssignableFrom(type.getJavaClass())) {
                 return HubType.InstanceReference;
-            } else if (Pod.Instance.class.isAssignableFrom(type.getJavaClass())) {
+            } else if (PodSupport.singleton().isPodClass(type.getJavaClass())) {
                 return HubType.PodInstance;
             }
             assert !Target_java_lang_ref_Reference.class.isAssignableFrom(type.getJavaClass()) : "should not see substitution type here";
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
new file mode 100644
index 000000000000..514507f02dd6
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted.heap;
+
+import static jdk.internal.org.objectweb.asm.Opcodes.ACC_FINAL;
+import static jdk.internal.org.objectweb.asm.Opcodes.ACC_PUBLIC;
+import static jdk.internal.org.objectweb.asm.Opcodes.ACC_SUPER;
+import static jdk.internal.org.objectweb.asm.Opcodes.ACC_SYNTHETIC;
+import static jdk.internal.org.objectweb.asm.Opcodes.V11;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.hosted.Feature;
+
+import com.oracle.svm.core.annotate.AutomaticFeature;
+import com.oracle.svm.core.annotate.Hybrid;
+import com.oracle.svm.core.heap.Pod;
+import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.util.UserError;
+import com.oracle.svm.hosted.NativeImageSystemClassLoader;
+import com.oracle.svm.util.ReflectionUtil;
+
+import jdk.internal.org.objectweb.asm.ClassWriter;
+import jdk.internal.org.objectweb.asm.Type;
+
+/** Support for preparing the creation of {@link Pod} objects during the image build. */
+public interface PodSupport {
+    static PodSupport singleton() {
+        return ImageSingletons.lookup(PodSupport.class);
+    }
+
+    /**
+     * Registers a class so it will be available as a superclass of {@link Pod}s at runtime via
+     * {@link com.oracle.svm.core.heap.Pod.Builder#createExtending(Class)}.
+     */
+    void registerSuperclass(Class<?> clazz);
+
+    boolean isPodClass(Class<?> clazz);
+
+    boolean isPodSuperclass(Class<?> clazz);
+}
+
+@AutomaticFeature
+final class PodFeature implements PodSupport, Feature {
+    private static final AtomicInteger GENERATED_COUNTER = new AtomicInteger();
+
+    private final Set<Class<?>> generated = ConcurrentHashMap.newKeySet();
+
+    /** These classes have to reserve the space where the length field in the pod class will be. */
+    private final Set<Class<?>> superClasses = ConcurrentHashMap.newKeySet();
+
+    private BeforeAnalysisAccess analysisAccess;
+    private volatile boolean instantiated = false;
+    private boolean sealed = false;
+
+    @Override
+    public void afterRegistration(AfterRegistrationAccess access) {
+        ImageSingletons.add(PodSupport.class, this);
+        ImageSingletons.add(Pod.RuntimeSupport.class, new Pod.RuntimeSupport());
+
+        registerSuperclass(Object.class);
+    }
+
+    @Override
+    public void beforeAnalysis(Feature.BeforeAnalysisAccess access) {
+        this.analysisAccess = access;
+        access.registerReachabilityHandler(this::registerAsInstantiated, ReflectionUtil.lookupMethod(Pod.class, "newInstance"));
+    }
+
+    private void registerAsInstantiated(DuringAnalysisAccess access) {
+        instantiated = true;
+        generated.forEach(clazz -> registerClassAsInstantiated(access, clazz));
+    }
+
+    private static void registerClassAsInstantiated(BeforeAnalysisAccess access, Class<?> podClass) {
+        access.registerAsInHeap(podClass);
+        runtimeSupport().registerClass(podClass.getSuperclass(), podClass);
+    }
+
+    private static Pod.RuntimeSupport runtimeSupport() {
+        return ImageSingletons.lookup(Pod.RuntimeSupport.class);
+    }
+
+    @Override
+    public void registerSuperclass(Class<?> superClass) {
+        if (sealed) {
+            throw UserError.abort("Pod superclasses can not be registered after analysis has finished");
+        }
+        if (runtimeSupport().get(superClass) != null) {
+            return;
+        }
+        Class<?> podClass = generatePod(superClass);
+        if (instantiated) {
+            registerClassAsInstantiated(analysisAccess, podClass);
+        }
+        Class<?> sup = superClass;
+        while (sup != Object.class) {
+            superClasses.add(sup);
+            sup = sup.getSuperclass();
+        }
+    }
+
+    /**
+     * For {@link Pod}s, we need a designated class that (1) is a hybrid class and so must not be
+     * allocated as an instance class anywhere because its {@link LayoutEncoding} describes it as an
+     * array and (2) is not subclassed since fields in a subclass would overlap with the array part.
+     */
+    private Class<?> generatePod(Class<?> superClass) {
+        String className = Pod.class.getName() + "$$Generated" + GENERATED_COUNTER.incrementAndGet();
+
+        ClassWriter writer = new ClassWriter(0);
+        int access = ACC_PUBLIC | ACC_SUPER | ACC_SYNTHETIC | ACC_FINAL;
+        writer.visit(V11, access, className.replace('.', '/'), null, Type.getInternalName(superClass), null);
+        writer.visitAnnotation(Type.getDescriptor(Hybrid.class), true)
+                        .visit("arrayType", Type.getType(byte[].class));
+        writer.visitEnd();
+        byte[] data = writer.toByteArray();
+
+        Class<?> podClass = NativeImageSystemClassLoader.singleton().predefineClass(className, data, 0, data.length);
+        assert podClass.getSuperclass() == superClass;
+
+        runtimeSupport().registerClass(superClass, podClass);
+        generated.add(podClass);
+        return podClass;
+    }
+
+    @Override
+    public boolean isPodClass(Class<?> clazz) {
+        return generated.contains(clazz);
+    }
+
+    @Override
+    public boolean isPodSuperclass(Class<?> clazz) {
+        return superClasses.contains(clazz);
+    }
+
+    @Override
+    public void afterAnalysis(AfterAnalysisAccess access) {
+        sealed = true;
+    }
+}
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
index 8991afbbf0c7..5523b369731a 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
@@ -91,6 +91,7 @@
 import com.oracle.svm.hosted.NativeImageOptions;
 import com.oracle.svm.hosted.annotation.CustomSubstitutionMethod;
 import com.oracle.svm.hosted.config.HybridLayout;
+import com.oracle.svm.hosted.heap.PodSupport;
 import com.oracle.svm.hosted.substitute.AnnotationSubstitutionProcessor;
 import com.oracle.svm.hosted.substitute.ComputedValueField;
 import com.oracle.svm.hosted.substitute.DeletedMethod;
@@ -413,9 +414,9 @@ private void layoutInstanceFields(HostedInstanceClass clazz, int superSize, Host
             startSize = DeoptimizedFrame.getScratchSpaceOffset() + layout.getDeoptScratchSpace();
         }
 
-        if (HybridLayout.isHybrid(clazz)) {
-            /* Set start after array length field */
-            assert startSize == layout.getArrayLengthOffset();
+        if (HybridLayout.isHybrid(clazz) || PodSupport.singleton().isPodSuperclass(clazz.getJavaClass())) {
+            // Reserve space for the array length field
+            VMError.guarantee(startSize == layout.getArrayLengthOffset());
             int fieldSize = layout.sizeInBytes(JavaKind.Int);
             startSize += fieldSize;
 

From 9e8b3784b0b1364d4bd36556f28008622d631056 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Thu, 14 Apr 2022 19:26:11 +0200
Subject: [PATCH 05/12] Support factory interfaces for superconstructor
 invocation on pods.

---
 .../impl/RuntimeReflectionSupport.java        |  40 +---
 ...SubstrateDynamicNewHybridInstanceNode.java |  56 ------
 .../snippets/SubstrateAllocationSnippets.java |  38 +---
 .../src/com/oracle/svm/core/heap/Pod.java     | 183 ++++++++++++++----
 .../oracle/svm/hosted/ProgressReporter.java   |   4 +-
 .../heap/PodFactorySubstitutionMethod.java    | 145 ++++++++++++++
 .../oracle/svm/hosted/heap/PodSupport.java    | 171 +++++++++++++---
 .../svm/hosted/heap/SVMImageHeapScanner.java  |   6 +-
 .../svm/hosted/heap/SVMImageHeapVerifier.java |   4 +-
 .../hosted/image/NativeImageCodeCache.java    |   4 +-
 .../InternalRuntimeReflectionSupport.java     |  69 +++++++
 .../svm/hosted/meta/UniverseBuilder.java      |   6 +-
 .../reflect/hosted/ReflectionDataBuilder.java |   4 +-
 .../svm/reflect/hosted/ReflectionFeature.java |   2 +
 .../hosted/ReflectionMetadataEncoderImpl.java |   4 +-
 15 files changed, 518 insertions(+), 218 deletions(-)
 delete mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java
 create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
 create mode 100644 substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/InternalRuntimeReflectionSupport.java

diff --git a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java
index 6b4e7aeaacbe..b4862955d1cc 100644
--- a/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java
+++ b/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/RuntimeReflectionSupport.java
@@ -40,44 +40,6 @@
  */
 package org.graalvm.nativeimage.impl;
 
-import java.lang.reflect.AccessibleObject;
-import java.lang.reflect.Executable;
-import java.lang.reflect.Field;
-import java.util.Map;
-import java.util.Set;
-
 public interface RuntimeReflectionSupport extends ReflectionRegistry {
-    Map<Class<?>, Set<Class<?>>> getReflectionInnerClasses();
-
-    Set<Field> getReflectionFields();
-
-    Set<Executable> getReflectionExecutables();
-
-    Object getAccessor(Executable method);
-
-    /*
-     * Returns the methods and fields that shadow a superclass element registered for reflection, to
-     * be excluded from reflection queries.
-     */
-    Set<?> getHidingReflectionFields();
-
-    Set<?> getHidingReflectionMethods();
-
-    Object[] getRecordComponents(Class<?> type);
-
-    void registerHeapDynamicHub(Object hub);
-
-    Set<?> getHeapDynamicHubs();
-
-    void registerHeapReflectionObject(AccessibleObject object);
-
-    Set<AccessibleObject> getHeapReflectionObjects();
-
-    int getReflectionClassesCount();
-
-    int getReflectionMethodsCount();
-
-    int getReflectionFieldsCount();
-
-    boolean requiresProcessing();
+    // needed as reflection-specific ImageSingletons key
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java
deleted file mode 100644
index 3f4a0c3ee496..000000000000
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/SubstrateDynamicNewHybridInstanceNode.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package com.oracle.svm.core.graal.nodes;
-
-import org.graalvm.compiler.core.common.type.StampFactory;
-import org.graalvm.compiler.graph.NodeClass;
-import org.graalvm.compiler.nodeinfo.NodeInfo;
-import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.nodes.java.AbstractNewArrayNode;
-
-@NodeInfo
-public final class SubstrateDynamicNewHybridInstanceNode extends AbstractNewArrayNode {
-    public static final NodeClass<SubstrateDynamicNewHybridInstanceNode> TYPE = NodeClass.create(SubstrateDynamicNewHybridInstanceNode.class);
-
-    @Input ValueNode instanceClass;
-    @Input ValueNode elementType;
-
-    public SubstrateDynamicNewHybridInstanceNode(ValueNode instanceClass, ValueNode elementType, ValueNode arrayLength) {
-        super(TYPE, StampFactory.objectNonNull(), arrayLength, true, null);
-        this.instanceClass = instanceClass;
-        this.elementType = elementType;
-    }
-
-    public ValueNode instanceType() {
-        return instanceClass;
-    }
-
-    public ValueNode elementType() {
-        return elementType;
-    }
-
-    @NodeIntrinsic
-    public static native Object allocate(Class<?> instanceClass, Class<?> elementType, int arrayLength);
-}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
index 01a089e1da95..0e18f8343256 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
@@ -79,7 +79,6 @@
 import com.oracle.svm.core.configure.ConfigurationFile;
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.nodes.ForeignCallWithExceptionNode;
-import com.oracle.svm.core.graal.nodes.SubstrateDynamicNewHybridInstanceNode;
 import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
 import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.hub.DynamicHub;
@@ -147,19 +146,16 @@ public Object allocateInstanceDynamic(@NonNullParameter DynamicHub hub, @Constan
     }
 
     @Snippet
-    public Object allocateArrayDynamic(DynamicHub hub, DynamicHub elementType, int length, @ConstantParameter FillContent fillContents, @ConstantParameter boolean emitMemoryBarrier,
+    public Object allocateArrayDynamic(DynamicHub elementType, int length, @ConstantParameter FillContent fillContents, @ConstantParameter boolean emitMemoryBarrier,
                     @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationProfilingData profilingData) {
-        DynamicHub arrayHub = hub;
-        if (arrayHub == null) {
-            arrayHub = getCheckedArrayHub(elementType);
-        }
+        DynamicHub checkedArrayHub = getCheckedArrayHub(elementType);
 
-        int layoutEncoding = arrayHub.getLayoutEncoding();
+        int layoutEncoding = checkedArrayHub.getLayoutEncoding();
         int arrayBaseOffset = getArrayBaseOffset(layoutEncoding);
         int log2ElementSize = LayoutEncoding.getArrayIndexShift(layoutEncoding);
 
-        Object result = allocateArrayImpl(encodeAsTLABObjectHeader(arrayHub), length, arrayBaseOffset, log2ElementSize, fillContents,
-                        afterArrayLengthOffset(), emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, profilingData);
+        Object result = allocateArrayImpl(encodeAsTLABObjectHeader(checkedArrayHub), length, arrayBaseOffset, log2ElementSize, fillContents,
+                        arrayBaseOffset, emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, profilingData);
         return piArrayCastToSnippetReplaceeStamp(result, length);
     }
 
@@ -403,7 +399,6 @@ public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>
             lowerings.put(NewArrayNode.class, new NewArrayLowering());
             lowerings.put(DynamicNewInstanceNode.class, new DynamicNewInstanceLowering());
             lowerings.put(DynamicNewArrayNode.class, new DynamicNewArrayLowering());
-            lowerings.put(SubstrateDynamicNewHybridInstanceNode.class, new DynamicNewHybridInstanceLowering());
             lowerings.put(NewMultiArrayNode.class, new NewMultiArrayLowering());
             lowerings.put(ValidateNewInstanceClassNode.class, new ValidateNewInstanceClassLowering());
         }
@@ -583,7 +578,6 @@ public void lower(DynamicNewArrayNode node, LoweringTool tool) {
                 }
 
                 Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
-                args.add("hub", ConstantNode.defaultForKind(JavaKind.Object, graph));
                 args.add("elementType", node.getElementType());
                 args.add("length", node.length());
                 args.addConst("fillContents", FillContent.fromBoolean(node.fillContents()));
@@ -596,28 +590,6 @@ public void lower(DynamicNewArrayNode node, LoweringTool tool) {
             }
         }
 
-        private class DynamicNewHybridInstanceLowering implements NodeLoweringProvider<SubstrateDynamicNewHybridInstanceNode> {
-            @Override
-            public void lower(SubstrateDynamicNewHybridInstanceNode node, LoweringTool tool) {
-                StructuredGraph graph = node.graph();
-                if (graph.getGuardsStage() != StructuredGraph.GuardsStage.AFTER_FSA) {
-                    return;
-                }
-
-                Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage());
-                args.add("hub", node.instanceType());
-                args.add("elementType", node.elementType());
-                args.add("length", node.length());
-                args.addConst("fillContents", FillContent.fromBoolean(node.fillContents()));
-                args.addConst("emitMemoryBarrier", node.emitMemoryBarrier());
-                args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
-                args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
-                args.addConst("profilingData", getProfilingData(node, null));
-
-                template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
-            }
-        }
-
         private class ValidateNewInstanceClassLowering implements NodeLoweringProvider<ValidateNewInstanceClassNode> {
             @Override
             public void lower(ValidateNewInstanceClassNode node, LoweringTool tool) {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index 5b15b8076970..0a4c864e2207 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -25,28 +25,32 @@
 package com.oracle.svm.core.heap;
 
 import java.io.ByteArrayOutputStream;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Supplier;
 
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.UnsignedWord;
-import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.JavaMemoryUtil;
 import com.oracle.svm.core.config.ConfigurationValues;
-import com.oracle.svm.core.graal.nodes.SubstrateDynamicNewHybridInstanceNode;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.LayoutEncoding;
 import com.oracle.svm.core.util.ImageHeapMap;
 import com.oracle.svm.core.util.UnsignedUtils;
-import com.oracle.svm.core.util.VMError;
 
+import jdk.internal.misc.Unsafe;
 import jdk.vm.ci.meta.JavaKind;
 
 /**
@@ -54,64 +58,107 @@
  * runtime and instances of which are allocated on the Java heap. The name <em>pod</em> refers to it
  * storing "plain old data" without further object-oriented features such as method dispatching or
  * type information, apart from having a Java superclass.
+ *
+ * @param <T> The interface of the {@linkplain #getFactory() factory} that allocates instances.
  */
 public final class Pod<T> {
-    private final Class<? extends T> podClass;
+    private final RuntimeSupport.PodInfo podInfo;
     private final int fieldsSizeWithoutRefMap;
     private final byte[] referenceMap;
-
-    private Pod(Class<? extends T> podClass, int fieldsSize, byte[] referenceMap) {
-        this.podClass = podClass;
+    private final T factory;
+
+    private Pod(RuntimeSupport.PodInfo podInfo, int fieldsSize, byte[] referenceMap) {
+        this.podInfo = podInfo;
+        try {
+            @SuppressWarnings("unchecked")
+            T factoryInstance = (T) podInfo.factoryCtor.newInstance(this);
+            this.factory = factoryInstance;
+        } catch (ReflectiveOperationException e) {
+            throw new RuntimeException(e);
+        }
         this.fieldsSizeWithoutRefMap = fieldsSize;
         this.referenceMap = referenceMap;
     }
 
-    public T newInstance() {
-        @SuppressWarnings("unchecked")
-        T instance = (T) SubstrateDynamicNewHybridInstanceNode.allocate(podClass, byte.class, fieldsSizeWithoutRefMap + referenceMap.length);
-
-        UnsignedWord podRefMapBase = getArrayBaseOffset(podClass).add(fieldsSizeWithoutRefMap);
-        JavaMemoryUtil.copy(referenceMap, getArrayBaseOffset(byte[].class), instance, podRefMapBase, WordFactory.unsigned(referenceMap.length));
-
-        return instance;
+    public T getFactory() {
+        return factory;
     }
 
-    private static UnsignedWord getArrayBaseOffset(Class<?> clazz) {
-        DynamicHub hub = DynamicHub.fromClass(clazz);
-        return LayoutEncoding.getArrayBaseOffset(hub.getLayoutEncoding());
+    /**
+     * Used from generated graphs: write {@link #referenceMap} to a newly allocated pod instance.
+     *
+     * {@link JavaMemoryUtil} copying is uninterruptible, so it cannot be inlined here and in our
+     * caller, and reference maps are likely too small to benefit from its optimizations. Instead,
+     * we use a simple interruptible copy loop that is safe because it does not use pointers and
+     * because a reference map decoder will not attempt to make sense of the reference map while the
+     * last two bytes (the first two it reads) remain zero, which is not a problem because all
+     * references are null at this point.
+     */
+    @SuppressWarnings("unused")
+    private void initInstanceRefMap(Object instance) {
+        Unsafe unsafe = Unsafe.getUnsafe();
+        int refMapArrayBase = unsafe.arrayBaseOffset(byte[].class);
+        int podRefMapBase = unsafe.arrayBaseOffset(podInfo.podClass) + fieldsSizeWithoutRefMap;
+        for (int offset = 0; offset < referenceMap.length; offset += 2) {
+            short entry = unsafe.getShort(referenceMap, refMapArrayBase + offset);
+            unsafe.putShort(instance, podRefMapBase + offset, entry);
+        }
     }
 
+    /**
+     * A builder for constructing pods with a specific superpod or superclass and factory interface
+     * and {@linkplain #addField additional fields}.
+     */
     public static final class Builder<T> {
-        public static Builder<Object> create() {
-            return new Builder<>(Object.class, null);
+        /**
+         * Create a builder for a pod that is derived directly from {@link Object} and
+         * {@link Supplier} as factory interface. This is supported without explicitly registering
+         * Object as superclass with {@code PodSupport}.
+         */
+        public static Builder<Supplier<Object>> create() {
+            return new Builder<>(Object.class, Supplier.class, null);
         }
 
+        /**
+         * Create a builder for a pod derived from the given superpod, which has the same
+         * superclass, uses the same factory interface for initialization, and has the same fields
+         * and in addition fields that are added with {@link #addField}. There are no guarantees
+         * with regard to Java type checks other than that {@link Object#getClass} on instances of
+         * the pods will return a {@link Class} to which the superclass of the two pods
+         * {@linkplain Class#isAssignableFrom is assignable from}.
+         */
         public static <T> Builder<T> createExtending(Pod<T> superPod) {
-            return new Builder<>(null, superPod);
+            return new Builder<>(null, null, superPod);
         }
 
-        public static <T> Builder<T> createExtending(Class<T> superClass) {
-            return new Builder<>(superClass, null);
+        /**
+         * Create a builder for a pod that extends the given superclass and instances of which can
+         * be allocated via the given factory interface. The specific pair of superclass and factory
+         * interface must have been registered during the image build with {@code PodSupport}.
+         */
+        public static <T> Builder<T> createExtending(Class<?> superClass, Class<T> factoryInterface) {
+            return new Builder<>(superClass, factoryInterface, null);
         }
 
         private final Pod<T> superPod;
-        private final Class<? extends T> podClass;
+        private final RuntimeSupport.PodInfo podInfo;
         private final List<Field> fields = new ArrayList<>();
         private boolean built = false;
 
-        private Builder(Class<T> superClass, Pod<T> superPod) {
-            if (superClass == null && superPod == null) {
-                throw new NullPointerException();
-            }
+        private Builder(Class<?> superClass, Class<?> factoryInterface, Pod<T> superPod) {
+            assert superPod == null || (superClass == null && factoryInterface == null);
             if (superPod != null) {
-                this.podClass = superPod.podClass;
-            } else {
-                this.podClass = ImageSingletons.lookup(RuntimeSupport.class).get(superClass);
-                if (this.podClass == null) {
-                    throw new IllegalArgumentException("Pod superclass was not registered during image build: " + superClass);
+                this.podInfo = superPod.podInfo;
+            } else if (superClass != null && factoryInterface != null) {
+                RuntimeSupport.PodSpec spec = new RuntimeSupport.PodSpec(superClass, factoryInterface);
+                this.podInfo = ImageSingletons.lookup(RuntimeSupport.class).getInfo(spec);
+                if (this.podInfo == null) {
+                    throw new IllegalArgumentException("Pod superclass/factory interface pair was not registered during image build: " + superClass + ", " + factoryInterface);
                 }
+            } else {
+                throw new NullPointerException();
             }
-            assert DynamicHub.fromClass(this.podClass).isPodInstanceClass();
+            assert DynamicHub.fromClass(this.podInfo.podClass).isPodInstanceClass();
             this.superPod = superPod;
         }
 
@@ -121,6 +168,11 @@ private void guaranteeUnbuilt() {
             }
         }
 
+        /**
+         * Add a field of a specified type to a pod. Once the pod has been {@linkplain #build
+         * built}, its offset is available via {@link Field#getOffset} and values of the given type
+         * can be written to instances, for example using {@code Unsafe}.
+         */
         public Field addField(Class<?> type) {
             guaranteeUnbuilt();
             Objects.requireNonNull(type);
@@ -135,13 +187,19 @@ public Field addField(Class<?> type) {
             return f;
         }
 
+        /**
+         * Create and return a pod with the superclass/superpod given during builder creation and
+         * the {@linkplain #addField added fields}. This method can be called only once, after which
+         * the builder can no longer be used.
+         */
         public Pod<T> build() {
             guaranteeUnbuilt();
             built = true;
 
             Collections.sort(fields);
 
-            UnsignedWord baseOffset = getArrayBaseOffset(podClass);
+            UnsignedWord baseOffset = LayoutEncoding.getArrayBaseOffset(
+                            DynamicHub.fromClass(podInfo.podClass).getLayoutEncoding());
 
             byte[] superRefMap = null;
             UnsignedWord nextOffset = baseOffset;
@@ -174,7 +232,7 @@ public Pod<T> build() {
             }
 
             byte[] referenceMap = refMapEncoder.encode();
-            return new Pod<>(podClass, UnsignedUtils.safeToInt(nextOffset), referenceMap);
+            return new Pod<>(podInfo, UnsignedUtils.safeToInt(nextOffset), referenceMap);
         }
     }
 
@@ -220,21 +278,60 @@ public int compareTo(Field f) {
     }
 
     public static final class RuntimeSupport {
-        private final EconomicMap<Class<?>, Class<?>> superClasses = ImageHeapMap.create();
+        @Retention(RetentionPolicy.RUNTIME)
+        @Target(ElementType.TYPE)
+        public @interface PodFactory {
+            Class<?> podClass();
+        }
+
+        public static final class PodSpec {
+            final Class<?> superClass;
+            final Class<?> factoryInterface;
+
+            public PodSpec(Class<?> superClass, Class<?> factoryInterface) {
+                assert superClass != null && factoryInterface != null;
+                this.superClass = superClass;
+                this.factoryInterface = factoryInterface;
+            }
+
+            @Override
+            public boolean equals(Object obj) {
+                if (obj != this && getClass() == obj.getClass()) {
+                    PodSpec other = (PodSpec) obj;
+                    return superClass.equals(other.superClass) && factoryInterface.equals(other.factoryInterface);
+                }
+                return (obj == this);
+            }
+
+            @Override
+            public int hashCode() {
+                return 31 * superClass.hashCode() + factoryInterface.hashCode();
+            }
+        }
+
+        public static final class PodInfo {
+            public final Class<?> podClass;
+            public final Constructor<?> factoryCtor;
+
+            public PodInfo(Class<?> podClass, Constructor<?> factoryCtor) {
+                this.podClass = podClass;
+                this.factoryCtor = factoryCtor;
+            }
+        }
+
+        private final EconomicMap<PodSpec, PodInfo> pods = ImageHeapMap.create();
 
         @Platforms(Platform.HOSTED_ONLY.class)
         public RuntimeSupport() {
         }
 
         @Platforms(Platform.HOSTED_ONLY.class)
-        public boolean registerClass(Class<?> superClass, Class<?> podClass) {
-            VMError.guarantee(podClass.getSuperclass() == superClass);
-            return superClasses.putIfAbsent(superClass, podClass) == null;
+        public boolean registerPod(PodSpec spec, PodInfo info) {
+            return pods.putIfAbsent(spec, info) == null;
         }
 
-        @SuppressWarnings("unchecked")
-        public <T> Class<? extends T> get(Class<T> clazz) {
-            return (Class<? extends T>) superClasses.get(clazz);
+        PodInfo getInfo(PodSpec spec) {
+            return pods.get(spec);
         }
     }
 
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java
index 3abe294326db..f4da70cd97df 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java
@@ -51,7 +51,6 @@
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.hosted.Feature;
 import org.graalvm.nativeimage.impl.ImageSingletonsSupport;
-import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
 
 import com.oracle.graal.pointsto.BigBang;
 import com.oracle.graal.pointsto.meta.AnalysisField;
@@ -75,6 +74,7 @@
 import com.oracle.svm.hosted.code.CompileQueue.CompileTask;
 import com.oracle.svm.hosted.image.AbstractImage.NativeImageKind;
 import com.oracle.svm.hosted.image.NativeImageHeap.ObjectInfo;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 import com.oracle.svm.util.ImageBuildStatistics;
 import com.oracle.svm.util.ReflectionUtil;
 
@@ -310,7 +310,7 @@ private void printAnalysisStatistics(AnalysisUniverse universe) {
                             .a(" methods included for ").doclink("runtime compilation", "#glossary-runtime-methods").println();
         }
         String classesFieldsMethodFormat = "%,8d classes, %,5d fields, and %,5d methods ";
-        RuntimeReflectionSupport rs = ImageSingletons.lookup(RuntimeReflectionSupport.class);
+        InternalRuntimeReflectionSupport rs = ImageSingletons.lookup(InternalRuntimeReflectionSupport.class);
         l().a(classesFieldsMethodFormat, rs.getReflectionClassesCount(), rs.getReflectionFieldsCount(), rs.getReflectionMethodsCount())
                         .doclink("registered for reflection", "#glossary-reflection-registrations").println();
         if (numJNIClasses > 0) {
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
new file mode 100644
index 000000000000..b89cc777f57f
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted.heap;
+
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.graalvm.compiler.debug.DebugContext;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.calc.AddNode;
+import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
+import org.graalvm.compiler.nodes.java.ArrayLengthNode;
+
+import com.oracle.graal.pointsto.infrastructure.SubstitutionProcessor;
+import com.oracle.graal.pointsto.infrastructure.UniverseMetaAccess;
+import com.oracle.graal.pointsto.meta.HostedProviders;
+import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
+import com.oracle.svm.core.heap.Pod;
+import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodFactory;
+import com.oracle.svm.hosted.annotation.CustomSubstitutionMethod;
+import com.oracle.svm.hosted.phases.HostedGraphKit;
+
+import jdk.vm.ci.meta.ResolvedJavaField;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+import jdk.vm.ci.meta.ResolvedJavaType;
+
+final class PodFactorySubstitutionProcessor extends SubstitutionProcessor {
+    private final ConcurrentMap<ResolvedJavaMethod, PodFactorySubstitutionMethod> substitutions = new ConcurrentHashMap<>();
+
+    @Override
+    public ResolvedJavaMethod lookup(ResolvedJavaMethod method) {
+        if (method.isSynthetic() && method.getDeclaringClass().isAnnotationPresent(PodFactory.class) && !method.isConstructor()) {
+            assert !(method instanceof CustomSubstitutionMethod);
+            return substitutions.computeIfAbsent(method, PodFactorySubstitutionMethod::new);
+        }
+        return method;
+    }
+
+    @Override
+    public ResolvedJavaMethod resolve(ResolvedJavaMethod method) {
+        if (method instanceof PodFactorySubstitutionMethod) {
+            return ((PodFactorySubstitutionMethod) method).getOriginal();
+        }
+        return method;
+    }
+}
+
+final class PodFactorySubstitutionMethod extends CustomSubstitutionMethod {
+    PodFactorySubstitutionMethod(ResolvedJavaMethod original) {
+        super(original);
+    }
+
+    @Override
+    public int getModifiers() {
+        return super.getModifiers() & ~Modifier.NATIVE;
+    }
+
+    @Override
+    public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method, HostedProviders providers, Purpose purpose) {
+        UniverseMetaAccess metaAccess = (UniverseMetaAccess) providers.getMetaAccess();
+        HostedGraphKit kit = new HostedGraphKit(debug, providers, method);
+
+        ValueNode[] originalArgs = kit.loadArguments(method.toParameterTypes()).toArray(ValueNode.EMPTY_ARRAY);
+
+        ResolvedJavaType podType = metaAccess.lookupJavaType(Pod.class);
+        ValueNode podObject = kit.maybeCreateExplicitNullCheck(kit.createLoadField(originalArgs[0], findField(method.getDeclaringClass(), "pod")));
+        ValueNode sizeWithoutRefMap = kit.createLoadField(podObject, findField(podType, "fieldsSizeWithoutRefMap"));
+        ValueNode refMapArray = kit.createLoadField(podObject, findField(podType, "referenceMap"));
+        ValueNode refMapLength = kit.append(new ArrayLengthNode(refMapArray));
+        ValueNode hybridArrayLength = kit.append(AddNode.add(sizeWithoutRefMap, refMapLength));
+
+        PodFactory annotation = method.getDeclaringClass().getAnnotation(PodFactory.class);
+        ResolvedJavaType podConcreteType = metaAccess.lookupJavaType(annotation.podClass());
+        ResolvedJavaType elementType = metaAccess.lookupJavaType(byte.class);
+
+        AbstractNewObjectNode instance = kit.append(new SubstrateNewHybridInstanceNode(podConcreteType, elementType, hybridArrayLength));
+        kit.createInvokeWithExceptionAndUnwind(Pod.class, "initInstanceRefMap", InvokeKind.Virtual, podObject, instance);
+
+        ResolvedJavaMethod targetCtor = null;
+        for (ResolvedJavaMethod ctor : podConcreteType.getSuperclass().getDeclaredConstructors()) {
+            if (parameterTypesMatch(method, ctor)) {
+                targetCtor = ctor;
+                break;
+            }
+        }
+        GraalError.guarantee(targetCtor != null, "Matching constructor not found: %s", getSignature());
+
+        ValueNode[] invokeArgs = Arrays.copyOf(originalArgs, originalArgs.length);
+        invokeArgs[0] = instance;
+        kit.createInvokeWithExceptionAndUnwind(targetCtor, InvokeKind.Special, kit.getFrameState(), kit.bci(), invokeArgs);
+
+        kit.createReturn(instance, instance.getStackKind());
+        return kit.finalizeGraph();
+    }
+
+    private static boolean parameterTypesMatch(ResolvedJavaMethod method, ResolvedJavaMethod ctor) {
+        int paramsCount = method.getSignature().getParameterCount(false);
+        if (paramsCount != ctor.getSignature().getParameterCount(false)) {
+            return false;
+        }
+        for (int i = 0; i < paramsCount; i++) {
+            if (!ctor.getSignature().getParameterType(i, ctor.getDeclaringClass())
+                            .equals(method.getSignature().getParameterType(i, method.getDeclaringClass()))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private static ResolvedJavaField findField(ResolvedJavaType type, String name) {
+        for (ResolvedJavaField field : type.getInstanceFields(false)) {
+            if (field.getName().equals(name)) {
+                return field;
+            }
+        }
+        throw GraalError.shouldNotReachHere("Required field " + name + " not found in " + type);
+    }
+}
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
index 514507f02dd6..063ef39dc98f 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
@@ -25,14 +25,25 @@
 package com.oracle.svm.hosted.heap;
 
 import static jdk.internal.org.objectweb.asm.Opcodes.ACC_FINAL;
+import static jdk.internal.org.objectweb.asm.Opcodes.ACC_PRIVATE;
 import static jdk.internal.org.objectweb.asm.Opcodes.ACC_PUBLIC;
 import static jdk.internal.org.objectweb.asm.Opcodes.ACC_SUPER;
 import static jdk.internal.org.objectweb.asm.Opcodes.ACC_SYNTHETIC;
+import static jdk.internal.org.objectweb.asm.Opcodes.ACONST_NULL;
+import static jdk.internal.org.objectweb.asm.Opcodes.ALOAD;
+import static jdk.internal.org.objectweb.asm.Opcodes.ARETURN;
+import static jdk.internal.org.objectweb.asm.Opcodes.INVOKESPECIAL;
+import static jdk.internal.org.objectweb.asm.Opcodes.PUTFIELD;
+import static jdk.internal.org.objectweb.asm.Opcodes.RETURN;
 import static jdk.internal.org.objectweb.asm.Opcodes.V11;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.hosted.Feature;
@@ -40,8 +51,13 @@
 import com.oracle.svm.core.annotate.AutomaticFeature;
 import com.oracle.svm.core.annotate.Hybrid;
 import com.oracle.svm.core.heap.Pod;
+import com.oracle.svm.core.heap.Pod.Builder;
+import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodFactory;
+import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodInfo;
+import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodSpec;
 import com.oracle.svm.core.hub.LayoutEncoding;
 import com.oracle.svm.core.util.UserError;
+import com.oracle.svm.hosted.FeatureImpl.DuringSetupAccessImpl;
 import com.oracle.svm.hosted.NativeImageSystemClassLoader;
 import com.oracle.svm.util.ReflectionUtil;
 
@@ -55,24 +71,36 @@ static PodSupport singleton() {
     }
 
     /**
-     * Registers a class so it will be available as a superclass of {@link Pod}s at runtime via
-     * {@link com.oracle.svm.core.heap.Pod.Builder#createExtending(Class)}.
+     * Registers a superclass and factory interface so that they are available to build {@link Pod}s
+     * at runtime via {@link Builder#createExtending(Class, Class)}. The provided superclass must be
+     * non-abstract, non-final and accessible (usually public and in an exported package). The
+     * factory interface consist solely of methods with a return type to which the superclass
+     * {@linkplain Class#isAssignableFrom is assignable to} and their parameter list musts match
+     * exactly the parameter list of one of the constructors of the superclass.
+     * <p>
+     * This method can be called multiple times for the same superclass and different factory
+     * interfaces, and for the same factory interface and different superclasses.
      */
-    void registerSuperclass(Class<?> clazz);
+    void registerSuperclass(Class<?> superClass, Class<?> factoryInterface);
 
     boolean isPodClass(Class<?> clazz);
 
-    boolean isPodSuperclass(Class<?> clazz);
+    boolean mustReserveLengthField(Class<?> clazz);
 }
 
 @AutomaticFeature
 final class PodFeature implements PodSupport, Feature {
     private static final AtomicInteger GENERATED_COUNTER = new AtomicInteger();
 
-    private final Set<Class<?>> generated = ConcurrentHashMap.newKeySet();
+    private final Map<PodSpec, PodInfo> pods = new ConcurrentHashMap<>();
 
-    /** These classes have to reserve the space where the length field in the pod class will be. */
-    private final Set<Class<?>> superClasses = ConcurrentHashMap.newKeySet();
+    /**
+     * These classes are ancestors of pod classes which directly inherit from {@link Object}, and so
+     * they must reserve space for a length field so subclasses (or the class itself) don't place
+     * other fields where the length field must be. If a pod subclasses {@link Object} itself, it is
+     * itself included in this set.
+     */
+    private final Set<Class<?>> classesNeedingLengthFields = ConcurrentHashMap.newKeySet();
 
     private BeforeAnalysisAccess analysisAccess;
     private volatile boolean instantiated = false;
@@ -83,23 +111,28 @@ public void afterRegistration(AfterRegistrationAccess access) {
         ImageSingletons.add(PodSupport.class, this);
         ImageSingletons.add(Pod.RuntimeSupport.class, new Pod.RuntimeSupport());
 
-        registerSuperclass(Object.class);
+        registerSuperclass(Object.class, Supplier.class);
+    }
+
+    @Override
+    public void duringSetup(DuringSetupAccess access) {
+        ((DuringSetupAccessImpl) access).registerSubstitutionProcessor(new PodFactorySubstitutionProcessor());
     }
 
     @Override
     public void beforeAnalysis(Feature.BeforeAnalysisAccess access) {
         this.analysisAccess = access;
-        access.registerReachabilityHandler(this::registerAsInstantiated, ReflectionUtil.lookupMethod(Pod.class, "newInstance"));
+        access.registerReachabilityHandler(this::registerAsInstantiated, (Object[]) Builder.class.getDeclaredConstructors());
     }
 
     private void registerAsInstantiated(DuringAnalysisAccess access) {
         instantiated = true;
-        generated.forEach(clazz -> registerClassAsInstantiated(access, clazz));
+        pods.forEach((spec, podInfo) -> registerPodAsInstantiated(access, spec, podInfo));
     }
 
-    private static void registerClassAsInstantiated(BeforeAnalysisAccess access, Class<?> podClass) {
-        access.registerAsInHeap(podClass);
-        runtimeSupport().registerClass(podClass.getSuperclass(), podClass);
+    private static void registerPodAsInstantiated(BeforeAnalysisAccess access, PodSpec spec, PodInfo info) {
+        access.registerAsInHeap(info.podClass);
+        runtimeSupport().registerPod(spec, info);
     }
 
     private static Pod.RuntimeSupport runtimeSupport() {
@@ -107,60 +140,134 @@ private static Pod.RuntimeSupport runtimeSupport() {
     }
 
     @Override
-    public void registerSuperclass(Class<?> superClass) {
+    public void registerSuperclass(Class<?> superClass, Class<?> factoryInterface) {
         if (sealed) {
             throw UserError.abort("Pod superclasses can not be registered after analysis has finished");
         }
-        if (runtimeSupport().get(superClass) != null) {
+        if (superClass == null || factoryInterface == null) {
+            throw new NullPointerException();
+        }
+        var spec = new PodSpec(superClass, factoryInterface);
+        if (pods.containsKey(spec)) {
             return;
         }
-        Class<?> podClass = generatePod(superClass);
+        if (!factoryInterface.isInterface()) {
+            throw new IllegalArgumentException("Factory is not an interface: " + factoryInterface);
+        }
+        for (Method method : factoryInterface.getMethods()) {
+            if (!method.getReturnType().isAssignableFrom(superClass)) {
+                throw new IllegalArgumentException("The return type of " + method + " is not assignable from " + superClass);
+            }
+            try {
+                superClass.getDeclaredConstructor(method.getParameterTypes());
+            } catch (NoSuchMethodException e) {
+                throw new IllegalArgumentException("Method " + method + " does not match any constructor in '" + superClass, e);
+            }
+        }
+        Class<?> podClass = generatePodClass(superClass);
+        Constructor<?> factoryCtor = generatePodFactory(podClass, factoryInterface);
+
+        var info = new PodInfo(podClass, factoryCtor);
+        if (pods.putIfAbsent(spec, info) != null) {
+            return; // lost a race with another thread
+        }
+
         if (instantiated) {
-            registerClassAsInstantiated(analysisAccess, podClass);
+            registerPodAsInstantiated(analysisAccess, spec, info);
         }
-        Class<?> sup = superClass;
-        while (sup != Object.class) {
-            superClasses.add(sup);
+        Class<?> sup = podClass;
+        while (sup.getSuperclass() != Object.class) {
             sup = sup.getSuperclass();
         }
+        classesNeedingLengthFields.add(sup);
     }
 
     /**
-     * For {@link Pod}s, we need a designated class that (1) is a hybrid class and so must not be
-     * allocated as an instance class anywhere because its {@link LayoutEncoding} describes it as an
-     * array and (2) is not subclassed since fields in a subclass would overlap with the array part.
+     * For pods, we need a designated class that (1) is a {@link Hybrid} class and so must never be
+     * allocated as an instance class because its {@link LayoutEncoding} describes it as an array
+     * and (2) is not subclassed since fields in a subclass would overlap with the array part.
      */
-    private Class<?> generatePod(Class<?> superClass) {
+    private static Class<?> generatePodClass(Class<?> superClass) {
         String className = Pod.class.getName() + "$$Generated" + GENERATED_COUNTER.incrementAndGet();
 
         ClassWriter writer = new ClassWriter(0);
         int access = ACC_PUBLIC | ACC_SUPER | ACC_SYNTHETIC | ACC_FINAL;
         writer.visit(V11, access, className.replace('.', '/'), null, Type.getInternalName(superClass), null);
-        writer.visitAnnotation(Type.getDescriptor(Hybrid.class), true)
-                        .visit("arrayType", Type.getType(byte[].class));
+        var annotation = writer.visitAnnotation(Type.getDescriptor(Hybrid.class), true);
+        annotation.visit("arrayType", Type.getType(byte[].class));
+        annotation.visitEnd();
         writer.visitEnd();
         byte[] data = writer.toByteArray();
 
         Class<?> podClass = NativeImageSystemClassLoader.singleton().predefineClass(className, data, 0, data.length);
         assert podClass.getSuperclass() == superClass;
-
-        runtimeSupport().registerClass(superClass, podClass);
-        generated.add(podClass);
         return podClass;
     }
 
+    /**
+     * Generate a concrete subclass of the provided factory interface with dummy methods that are
+     * subsequently substituted with {@link PodFactorySubstitutionMethod} to allocate pod instances
+     * and invoke the matching superclass constructor on them.
+     */
+    private static Constructor<?> generatePodFactory(Class<?> podClass, Class<?> factoryInterface) {
+        String name = Pod.class.getName() + "$$GeneratedFactory" + GENERATED_COUNTER.incrementAndGet();
+        String internalName = name.replace('.', '/');
+        String podDescriptor = Type.getDescriptor(Pod.class);
+
+        ClassWriter writer = new ClassWriter(0);
+        int access = ACC_PUBLIC | ACC_SUPER | ACC_SYNTHETIC | ACC_FINAL;
+        writer.visit(V11, access, internalName, null, Type.getInternalName(Object.class), new String[]{Type.getInternalName(factoryInterface)});
+        var annotation = writer.visitAnnotation(Type.getDescriptor(PodFactory.class), true);
+        annotation.visit("podClass", Type.getType(podClass));
+        annotation.visitEnd();
+        writer.visitField(ACC_PRIVATE | ACC_FINAL | ACC_SYNTHETIC, "pod", podDescriptor, null, null).visitEnd();
+        var init = writer.visitMethod(ACC_PUBLIC | ACC_SYNTHETIC, "<init>", Type.getMethodDescriptor(Type.VOID_TYPE, Type.getType(podDescriptor)), null, null);
+        init.visitCode();
+        init.visitVarInsn(ALOAD, 0);
+        init.visitMethodInsn(INVOKESPECIAL, Type.getInternalName(Object.class), "<init>", "()V", false);
+        init.visitVarInsn(ALOAD, 0);
+        init.visitVarInsn(ALOAD, 1);
+        init.visitFieldInsn(PUTFIELD, internalName, "pod", podDescriptor);
+        init.visitInsn(RETURN);
+        init.visitMaxs(2, 2);
+        init.visitEnd();
+        for (Method method : factoryInterface.getMethods()) {
+            int methodAccess = ACC_PUBLIC | ACC_FINAL | ACC_SYNTHETIC;
+            var impl = writer.visitMethod(methodAccess, method.getName(), Type.getMethodDescriptor(method), null, null);
+            impl.visitCode(); // substituted with custom graph
+            impl.visitInsn(ACONST_NULL);
+            impl.visitInsn(ARETURN);
+            impl.visitMaxs(1, method.getParameterCount() + 1);
+            impl.visitEnd();
+        }
+        writer.visitEnd();
+        byte[] data = writer.toByteArray();
+
+        Class<?> factoryClass = NativeImageSystemClassLoader.singleton().predefineClass(name, data, 0, data.length);
+        assert factoryInterface.isAssignableFrom(factoryClass);
+        return ReflectionUtil.lookupConstructor(factoryClass, Pod.class);
+    }
+
     @Override
     public boolean isPodClass(Class<?> clazz) {
-        return generated.contains(clazz);
+        if ((clazz.getModifiers() & ACC_SYNTHETIC) != 0 && clazz.isAnnotationPresent(Hybrid.class)) {
+            for (PodInfo info : pods.values()) {
+                if (info.podClass == clazz) {
+                    return true;
+                }
+            }
+        }
+        return false;
     }
 
     @Override
-    public boolean isPodSuperclass(Class<?> clazz) {
-        return superClasses.contains(clazz);
+    public boolean mustReserveLengthField(Class<?> clazz) {
+        return classesNeedingLengthFields.contains(clazz);
     }
 
     @Override
     public void afterAnalysis(AfterAnalysisAccess access) {
+        analysisAccess = null;
         sealed = true;
     }
 }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java
index 2e5273c7502e..94625fb2cd9f 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapScanner.java
@@ -32,7 +32,6 @@
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
 import org.graalvm.nativeimage.ImageSingletons;
-import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
 
 import com.oracle.graal.pointsto.ObjectScanner.ScanReason;
 import com.oracle.graal.pointsto.ObjectScanningObserver;
@@ -50,6 +49,7 @@
 import com.oracle.svm.hosted.ImageClassLoader;
 import com.oracle.svm.hosted.ameta.AnalysisConstantReflectionProvider;
 import com.oracle.svm.hosted.meta.HostedMetaAccess;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 import com.oracle.svm.util.ReflectionUtil;
 
 import jdk.vm.ci.meta.ConstantReflectionProvider;
@@ -62,7 +62,7 @@ public class SVMImageHeapScanner extends ImageHeapScanner {
     private final Class<?> economicMapImpl;
     private final Field economicMapImplEntriesField;
     private final Field economicMapImplHashArrayField;
-    private final RuntimeReflectionSupport reflectionSupport;
+    private final InternalRuntimeReflectionSupport reflectionSupport;
 
     public SVMImageHeapScanner(ImageHeap imageHeap, ImageClassLoader loader, AnalysisMetaAccess metaAccess,
                     SnippetReflectionProvider snippetReflection, ConstantReflectionProvider aConstantReflection, ObjectScanningObserver aScanningObserver) {
@@ -72,7 +72,7 @@ public SVMImageHeapScanner(ImageHeap imageHeap, ImageClassLoader loader, Analysi
         economicMapImplEntriesField = ReflectionUtil.lookupField(economicMapImpl, "entries");
         economicMapImplHashArrayField = ReflectionUtil.lookupField(economicMapImpl, "hashArray");
         ImageSingletons.add(ImageHeapScanner.class, this);
-        reflectionSupport = ImageSingletons.lookup(RuntimeReflectionSupport.class);
+        reflectionSupport = ImageSingletons.lookup(InternalRuntimeReflectionSupport.class);
     }
 
     public static ImageHeapScanner instance() {
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapVerifier.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapVerifier.java
index 839b78cf2037..b85c4a0a2c0c 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapVerifier.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/SVMImageHeapVerifier.java
@@ -25,7 +25,6 @@
 package com.oracle.svm.hosted.heap;
 
 import org.graalvm.nativeimage.ImageSingletons;
-import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
 
 import com.oracle.graal.pointsto.BigBang;
 import com.oracle.graal.pointsto.ObjectScanner;
@@ -37,6 +36,7 @@
 import com.oracle.svm.core.meta.SubstrateObjectConstant;
 import com.oracle.svm.hosted.SVMHost;
 import com.oracle.svm.hosted.analysis.NativeImagePointsToAnalysis;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 
 import jdk.vm.ci.meta.JavaConstant;
 
@@ -63,7 +63,7 @@ public boolean requireAnalysisIteration(CompletionExecutor executor) throws Inte
      * 
      */
     private static boolean imageStateModified() {
-        return ImageSingletons.lookup(RuntimeReflectionSupport.class).requiresProcessing() ||
+        return ImageSingletons.lookup(InternalRuntimeReflectionSupport.class).requiresProcessing() ||
                         ImageSingletons.lookup(ImageHeapMapFeature.class).imageHeapMapNeedsUpdate();
     }
 
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java
index 115c72ff56b4..e67f561fbaf4 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageCodeCache.java
@@ -54,7 +54,6 @@
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.c.function.CFunctionPointer;
-import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
 
@@ -93,6 +92,7 @@
 import com.oracle.svm.hosted.meta.HostedMethod;
 import com.oracle.svm.hosted.meta.HostedType;
 import com.oracle.svm.hosted.meta.HostedUniverse;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 
 import jdk.vm.ci.code.BytecodeFrame;
 import jdk.vm.ci.code.site.Call;
@@ -245,7 +245,7 @@ public void buildRuntimeMetadata(CFunctionPointer firstMethod, UnsignedWord code
         }
 
         ReflectionMetadataEncoder reflectionMetadataEncoder = ImageSingletons.lookup(ReflectionMetadataEncoderFactory.class).create(encoders);
-        RuntimeReflectionSupport reflectionSupport = ImageSingletons.lookup(RuntimeReflectionSupport.class);
+        InternalRuntimeReflectionSupport reflectionSupport = ImageSingletons.lookup(InternalRuntimeReflectionSupport.class);
         HostedUniverse hUniverse = imageHeap.getUniverse();
         HostedMetaAccess hMetaAccess = imageHeap.getMetaAccess();
 
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/InternalRuntimeReflectionSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/InternalRuntimeReflectionSupport.java
new file mode 100644
index 000000000000..8aaaf8722e6d
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/InternalRuntimeReflectionSupport.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.hosted.meta;
+
+import java.lang.reflect.AccessibleObject;
+import java.lang.reflect.Executable;
+import java.lang.reflect.Field;
+import java.util.Map;
+import java.util.Set;
+
+import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
+
+public interface InternalRuntimeReflectionSupport extends RuntimeReflectionSupport {
+    Map<Class<?>, Set<Class<?>>> getReflectionInnerClasses();
+
+    Set<Field> getReflectionFields();
+
+    Set<Executable> getReflectionExecutables();
+
+    Object getAccessor(Executable method);
+
+    /*
+     * Returns the methods and fields that shadow a superclass element registered for reflection, to
+     * be excluded from reflection queries.
+     */
+    Set<?> getHidingReflectionFields();
+
+    Set<?> getHidingReflectionMethods();
+
+    Object[] getRecordComponents(Class<?> type);
+
+    void registerHeapDynamicHub(Object hub);
+
+    Set<?> getHeapDynamicHubs();
+
+    void registerHeapReflectionObject(AccessibleObject object);
+
+    Set<AccessibleObject> getHeapReflectionObjects();
+
+    int getReflectionClassesCount();
+
+    int getReflectionMethodsCount();
+
+    int getReflectionFieldsCount();
+
+    boolean requiresProcessing();
+}
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
index 5523b369731a..f7d6e13ffd41 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
@@ -414,8 +414,10 @@ private void layoutInstanceFields(HostedInstanceClass clazz, int superSize, Host
             startSize = DeoptimizedFrame.getScratchSpaceOffset() + layout.getDeoptScratchSpace();
         }
 
-        if (HybridLayout.isHybrid(clazz) || PodSupport.singleton().isPodSuperclass(clazz.getJavaClass())) {
-            // Reserve space for the array length field
+        PodSupport pods = PodSupport.singleton();
+        if ((HybridLayout.isHybrid(clazz) && !pods.isPodClass(clazz.getJavaClass())) || pods.mustReserveLengthField(clazz.getJavaClass())) {
+            // Reserve space for the array length field. For pods, the ancestor subclassing Object
+            // must have already reserved this space (unless the pod subclasses Object itself).
             VMError.guarantee(startSize == layout.getArrayLengthOffset());
             int fieldSize = layout.sizeInBytes(JavaKind.Int);
             startSize += fieldSize;
diff --git a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java
index 0416d4056273..23b00f48c568 100644
--- a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java
+++ b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionDataBuilder.java
@@ -58,7 +58,6 @@
 import org.graalvm.nativeimage.hosted.Feature.DuringAnalysisAccess;
 import org.graalvm.nativeimage.hosted.RuntimeReflection;
 import org.graalvm.nativeimage.impl.ConfigurationCondition;
-import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
 import org.graalvm.util.GuardedAnnotationAccess;
 
 import com.oracle.graal.pointsto.constraints.UnsupportedFeatureException;
@@ -78,6 +77,7 @@
 import com.oracle.svm.hosted.ConditionalConfigurationRegistry;
 import com.oracle.svm.hosted.FeatureImpl.DuringAnalysisAccessImpl;
 import com.oracle.svm.hosted.annotation.AnnotationSubstitutionType;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 import com.oracle.svm.hosted.substitute.SubstitutionReflectivityFilter;
 import com.oracle.svm.util.ModuleSupport;
 
@@ -90,7 +90,7 @@
 import sun.reflect.annotation.TypeAnnotation;
 import sun.reflect.annotation.TypeNotPresentExceptionProxy;
 
-public class ReflectionDataBuilder extends ConditionalConfigurationRegistry implements RuntimeReflectionSupport {
+public class ReflectionDataBuilder extends ConditionalConfigurationRegistry implements InternalRuntimeReflectionSupport {
 
     private final Set<Class<?>> modifiedClasses = ConcurrentHashMap.newKeySet();
     private boolean sealed;
diff --git a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionFeature.java b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionFeature.java
index f8a374c615c6..25bf875b5413 100644
--- a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionFeature.java
+++ b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionFeature.java
@@ -66,6 +66,7 @@
 import com.oracle.svm.hosted.analysis.Inflation;
 import com.oracle.svm.hosted.code.FactoryMethodSupport;
 import com.oracle.svm.hosted.config.ConfigurationParserUtils;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 import com.oracle.svm.hosted.snippets.ReflectionPlugins;
 import com.oracle.svm.hosted.substitute.AnnotationSubstitutionProcessor;
 import com.oracle.svm.util.ModuleSupport;
@@ -203,6 +204,7 @@ public void afterRegistration(AfterRegistrationAccess access) {
 
         reflectionData = new ReflectionDataBuilder();
         ImageSingletons.add(RuntimeReflectionSupport.class, reflectionData);
+        ImageSingletons.add(InternalRuntimeReflectionSupport.class, reflectionData);
     }
 
     @Override
diff --git a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionMetadataEncoderImpl.java b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionMetadataEncoderImpl.java
index d31f195df871..b291b008d8f6 100644
--- a/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionMetadataEncoderImpl.java
+++ b/substratevm/src/com.oracle.svm.reflect/src/com/oracle/svm/reflect/hosted/ReflectionMetadataEncoderImpl.java
@@ -60,7 +60,6 @@
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
 import org.graalvm.nativeimage.ImageSingletons;
-import org.graalvm.nativeimage.impl.RuntimeReflectionSupport;
 import org.graalvm.util.GuardedAnnotationAccess;
 
 import com.oracle.graal.pointsto.meta.AnalysisField;
@@ -79,6 +78,7 @@
 import com.oracle.svm.hosted.meta.HostedMetaAccess;
 import com.oracle.svm.hosted.meta.HostedMethod;
 import com.oracle.svm.hosted.meta.HostedType;
+import com.oracle.svm.hosted.meta.InternalRuntimeReflectionSupport;
 import com.oracle.svm.hosted.substitute.DeletedElementException;
 import com.oracle.svm.reflect.hosted.ReflectionMetadata.AccessibleObjectMetadata;
 import com.oracle.svm.reflect.hosted.ReflectionMetadata.ClassMetadata;
@@ -656,7 +656,7 @@ private static ReflectParameterMetadata[] getReflectParameters(Executable reflec
     }
 
     private RecordComponentMetadata[] getRecordComponents(MetaAccessProvider metaAccess, HostedType declaringType, Class<?> clazz) {
-        Object[] recordComponents = ImageSingletons.lookup(RuntimeReflectionSupport.class).getRecordComponents(clazz);
+        Object[] recordComponents = ImageSingletons.lookup(InternalRuntimeReflectionSupport.class).getRecordComponents(clazz);
         if (recordComponents == null) {
             return null;
         }

From bfcc30b8435d50b0d37c438ed9eeeb496d0d38c2 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Tue, 26 Apr 2022 12:49:05 +0200
Subject: [PATCH 06/12] Support runtime compilation of pod factory methods.

---
 .../heap/PodFactorySubstitutionMethod.java    | 164 +++++++++++++++---
 .../oracle/svm/hosted/heap/PodSupport.java    |   4 +
 2 files changed, 146 insertions(+), 22 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
index b89cc777f57f..42ce8538343e 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
@@ -32,21 +32,34 @@
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
+import org.graalvm.compiler.nodes.FixedNode;
+import org.graalvm.compiler.nodes.FixedWithNextNode;
+import org.graalvm.compiler.nodes.FrameState;
+import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.UnreachableBeginNode;
+import org.graalvm.compiler.nodes.UnwindNode;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.AddNode;
-import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
+import org.graalvm.compiler.nodes.java.ExceptionObjectNode;
 
 import com.oracle.graal.pointsto.infrastructure.SubstitutionProcessor;
-import com.oracle.graal.pointsto.infrastructure.UniverseMetaAccess;
 import com.oracle.graal.pointsto.meta.HostedProviders;
+import com.oracle.svm.core.annotate.DeoptTest;
+import com.oracle.svm.core.graal.nodes.DeoptEntryBeginNode;
+import com.oracle.svm.core.graal.nodes.DeoptEntryNode;
+import com.oracle.svm.core.graal.nodes.LoweredDeadEndNode;
 import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
+import com.oracle.svm.core.graal.nodes.TestDeoptimizeNode;
 import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodFactory;
+import com.oracle.svm.core.meta.SharedMethod;
 import com.oracle.svm.hosted.annotation.CustomSubstitutionMethod;
+import com.oracle.svm.hosted.nodes.DeoptProxyNode;
 import com.oracle.svm.hosted.phases.HostedGraphKit;
 
+import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.ResolvedJavaField;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
@@ -77,6 +90,11 @@ final class PodFactorySubstitutionMethod extends CustomSubstitutionMethod {
         super(original);
     }
 
+    @Override
+    public boolean allowRuntimeCompilation() {
+        return true;
+    }
+
     @Override
     public int getModifiers() {
         return super.getModifiers() & ~Modifier.NATIVE;
@@ -84,25 +102,13 @@ public int getModifiers() {
 
     @Override
     public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method, HostedProviders providers, Purpose purpose) {
-        UniverseMetaAccess metaAccess = (UniverseMetaAccess) providers.getMetaAccess();
         HostedGraphKit kit = new HostedGraphKit(debug, providers, method);
+        boolean isDeoptTarget = (method instanceof SharedMethod) && ((SharedMethod) method).isDeoptTarget();
 
-        ValueNode[] originalArgs = kit.loadArguments(method.toParameterTypes()).toArray(ValueNode.EMPTY_ARRAY);
-
-        ResolvedJavaType podType = metaAccess.lookupJavaType(Pod.class);
-        ValueNode podObject = kit.maybeCreateExplicitNullCheck(kit.createLoadField(originalArgs[0], findField(method.getDeclaringClass(), "pod")));
-        ValueNode sizeWithoutRefMap = kit.createLoadField(podObject, findField(podType, "fieldsSizeWithoutRefMap"));
-        ValueNode refMapArray = kit.createLoadField(podObject, findField(podType, "referenceMap"));
-        ValueNode refMapLength = kit.append(new ArrayLengthNode(refMapArray));
-        ValueNode hybridArrayLength = kit.append(AddNode.add(sizeWithoutRefMap, refMapLength));
-
-        PodFactory annotation = method.getDeclaringClass().getAnnotation(PodFactory.class);
-        ResolvedJavaType podConcreteType = metaAccess.lookupJavaType(annotation.podClass());
-        ResolvedJavaType elementType = metaAccess.lookupJavaType(byte.class);
-
-        AbstractNewObjectNode instance = kit.append(new SubstrateNewHybridInstanceNode(podConcreteType, elementType, hybridArrayLength));
-        kit.createInvokeWithExceptionAndUnwind(Pod.class, "initInstanceRefMap", InvokeKind.Virtual, podObject, instance);
-
+        ResolvedJavaType factoryType = method.getDeclaringClass();
+        PodFactory annotation = factoryType.getAnnotation(PodFactory.class);
+        ResolvedJavaType podConcreteType = kit.getMetaAccess().lookupJavaType(annotation.podClass());
+        ResolvedJavaType elementType = kit.getMetaAccess().lookupJavaType(byte.class);
         ResolvedJavaMethod targetCtor = null;
         for (ResolvedJavaMethod ctor : podConcreteType.getSuperclass().getDeclaredConstructors()) {
             if (parameterTypesMatch(method, ctor)) {
@@ -112,12 +118,126 @@ public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method,
         }
         GraalError.guarantee(targetCtor != null, "Matching constructor not found: %s", getSignature());
 
+        /*
+         * The graph must be safe for runtime compilation and so for compilation as a deoptimization
+         * target. We must be careful to use values only from the frame state, so we keep the
+         * allocated instance in a local and load it after each step during which a deopt can occur.
+         */
+        int instanceLocal = method.getSignature().getParameterCount(true);
+        int nextDeoptIndex = startMethod(kit, isDeoptTarget, 0);
+        instantiatePod(kit, factoryType, podConcreteType, elementType, instanceLocal);
+        if (isAnnotationPresent(DeoptTest.class)) {
+            kit.append(new TestDeoptimizeNode());
+        }
+        nextDeoptIndex = initInstanceRefMap(kit, method, isDeoptTarget, nextDeoptIndex, instanceLocal);
+        nextDeoptIndex = invokeConstructor(kit, method, isDeoptTarget, nextDeoptIndex, targetCtor, instanceLocal);
+
+        kit.createReturn(kit.loadLocal(instanceLocal, JavaKind.Object), JavaKind.Object);
+        return kit.finalizeGraph();
+    }
+
+    private static int startMethod(HostedGraphKit kit, boolean isDeoptTarget, int nextDeoptIndex) {
+        if (!isDeoptTarget) {
+            return nextDeoptIndex;
+        }
+        FrameState initialState = kit.getGraph().start().stateAfter();
+        return appendDeoptWithExceptionUnwind(kit, initialState, initialState.bci, nextDeoptIndex);
+    }
+
+    private static void instantiatePod(HostedGraphKit kit, ResolvedJavaType factoryType, ResolvedJavaType podConcreteType, ResolvedJavaType elementType, int instanceLocal) {
+        ResolvedJavaType podType = kit.getMetaAccess().lookupJavaType(Pod.class);
+        ValueNode pod = loadPod(kit, factoryType);
+        ValueNode sizeWithoutRefMap = kit.createLoadField(pod, findField(podType, "fieldsSizeWithoutRefMap"));
+        ValueNode refMapArray = kit.createLoadField(pod, findField(podType, "referenceMap"));
+        ValueNode refMapLength = kit.append(new ArrayLengthNode(refMapArray));
+        ValueNode hybridArrayLength = kit.append(AddNode.add(sizeWithoutRefMap, refMapLength));
+        ValueNode instance = kit.append(new SubstrateNewHybridInstanceNode(podConcreteType, elementType, hybridArrayLength));
+        kit.storeLocal(instanceLocal, JavaKind.Object, instance);
+    }
+
+    private static int initInstanceRefMap(HostedGraphKit kit, ResolvedJavaMethod method, boolean isDeoptTarget, int nextDeoptIndex, int instanceLocal) {
+        ValueNode instance = kit.loadLocal(instanceLocal, JavaKind.Object);
+        ResolvedJavaMethod initInstanceRefMap = kit.findMethod(Pod.class, "initInstanceRefMap", false);
+        ValueNode pod = loadPod(kit, method.getDeclaringClass());
+        return invokeWithDeoptAndExceptionUnwind(kit, isDeoptTarget, nextDeoptIndex, initInstanceRefMap, InvokeKind.Virtual, pod, instance);
+    }
+
+    private static int invokeConstructor(HostedGraphKit kit, ResolvedJavaMethod method, boolean isDeoptTarget, int nextDeoptIndex, ResolvedJavaMethod targetCtor, int instanceLocal) {
+        ValueNode instance = kit.loadLocal(instanceLocal, JavaKind.Object);
+        ValueNode[] originalArgs = kit.loadArguments(method.toParameterTypes()).toArray(ValueNode.EMPTY_ARRAY);
         ValueNode[] invokeArgs = Arrays.copyOf(originalArgs, originalArgs.length);
         invokeArgs[0] = instance;
-        kit.createInvokeWithExceptionAndUnwind(targetCtor, InvokeKind.Special, kit.getFrameState(), kit.bci(), invokeArgs);
+        return invokeWithDeoptAndExceptionUnwind(kit, isDeoptTarget, nextDeoptIndex, targetCtor, InvokeKind.Special, invokeArgs);
+    }
 
-        kit.createReturn(instance, instance.getStackKind());
-        return kit.finalizeGraph();
+    private static ValueNode loadPod(HostedGraphKit kit, ResolvedJavaType declaringClass) {
+        ValueNode receiver = kit.loadLocal(0, JavaKind.Object);
+        return kit.maybeCreateExplicitNullCheck(kit.createLoadField(receiver, findField(declaringClass, "pod")));
+    }
+
+    /** @see com.oracle.svm.hosted.phases.HostedGraphBuilderPhase */
+    private static int invokeWithDeoptAndExceptionUnwind(HostedGraphKit kit, boolean isDeoptTarget, int initialNextDeoptIndex, ResolvedJavaMethod target, InvokeKind invokeKind, ValueNode... args) {
+        InvokeWithExceptionNode invoke = kit.startInvokeWithException(target, invokeKind, kit.getFrameState(), kit.bci(), args);
+        kit.exceptionPart();
+        ExceptionObjectNode exception = kit.exceptionObject();
+
+        if (!isDeoptTarget) {
+            kit.append(new UnwindNode(exception));
+            kit.endInvokeWithException();
+            return initialNextDeoptIndex;
+        }
+
+        int nextDeoptIndex = initialNextDeoptIndex;
+
+        // Exception during invoke
+
+        var exceptionDeopt = kit.add(new DeoptEntryNode());
+        exceptionDeopt.setStateAfter(exception.stateAfter().duplicate());
+        var exceptionDeoptBegin = kit.add(new DeoptEntryBeginNode());
+        int exceptionDeoptIndex = nextDeoptIndex++;
+        ValueNode exceptionProxy = createDeoptProxy(kit, exceptionDeoptIndex, exceptionDeopt, exception);
+        var unwind = kit.append(new UnwindNode(exceptionProxy));
+        exception.setNext(exceptionDeopt);
+        exceptionDeopt.setNext(exceptionDeoptBegin);
+        exceptionDeoptBegin.setNext(unwind);
+
+        var exceptionDeoptExceptionEdge = kit.add(new UnreachableBeginNode());
+        exceptionDeoptExceptionEdge.setNext(kit.add(new LoweredDeadEndNode()));
+        exceptionDeopt.setExceptionEdge(exceptionDeoptExceptionEdge);
+
+        // Deopt entry after invoke without exception
+
+        kit.noExceptionPart();
+        nextDeoptIndex = appendDeoptWithExceptionUnwind(kit, invoke.stateAfter(), invoke.stateAfter().bci, nextDeoptIndex);
+        kit.endInvokeWithException();
+
+        return nextDeoptIndex;
+    }
+
+    /** @see com.oracle.svm.hosted.phases.HostedGraphBuilderPhase */
+    private static int appendDeoptWithExceptionUnwind(HostedGraphKit kit, FrameState state, int exceptionBci, int nextDeoptIndex) {
+        var entry = kit.add(new DeoptEntryNode());
+        entry.setStateAfter(state.duplicate());
+        var begin = kit.append(new DeoptEntryBeginNode());
+        ((FixedWithNextNode) begin.predecessor()).setNext(entry);
+        entry.setNext(begin);
+
+        ExceptionObjectNode exception = kit.add(new ExceptionObjectNode(kit.getMetaAccess()));
+        entry.setExceptionEdge(exception);
+        var exState = kit.getFrameState().copy();
+        exState.clearStack();
+        exState.push(JavaKind.Object, exception);
+        exState.setRethrowException(true);
+        exception.setStateAfter(exState.create(exceptionBci, exception));
+        exception.setNext(kit.add(new UnwindNode(exception)));
+
+        // Ensure later nodes see values from potential deoptimization
+        kit.getFrameState().insertProxies(value -> createDeoptProxy(kit, nextDeoptIndex, entry, value));
+        return nextDeoptIndex + 1;
+    }
+
+    private static ValueNode createDeoptProxy(HostedGraphKit kit, int nextDeoptIndex, FixedNode deoptTarget, ValueNode value) {
+        return kit.getGraph().addOrUniqueWithInputs(DeoptProxyNode.create(value, deoptTarget, nextDeoptIndex));
     }
 
     private static boolean parameterTypesMatch(ResolvedJavaMethod method, ResolvedJavaMethod ctor) {
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
index 063ef39dc98f..ff42cc993da9 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
@@ -49,6 +49,7 @@
 import org.graalvm.nativeimage.hosted.Feature;
 
 import com.oracle.svm.core.annotate.AutomaticFeature;
+import com.oracle.svm.core.annotate.DeoptTest;
 import com.oracle.svm.core.annotate.Hybrid;
 import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.Pod.Builder;
@@ -234,6 +235,9 @@ private static Constructor<?> generatePodFactory(Class<?> podClass, Class<?> fac
         for (Method method : factoryInterface.getMethods()) {
             int methodAccess = ACC_PUBLIC | ACC_FINAL | ACC_SYNTHETIC;
             var impl = writer.visitMethod(methodAccess, method.getName(), Type.getMethodDescriptor(method), null, null);
+            if (method.isAnnotationPresent(DeoptTest.class)) {
+                impl.visitAnnotation(Type.getDescriptor(DeoptTest.class), true).visitEnd();
+            }
             impl.visitCode(); // substituted with custom graph
             impl.visitInsn(ACONST_NULL);
             impl.visitInsn(ARETURN);

From ed42260cf16e4f6dcefca23e4886d7e173efff24 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Thu, 28 Apr 2022 08:37:36 +0200
Subject: [PATCH 07/12] Fully initialize pods in allocation snippet or slow
 path.

---
 .../replacements/AllocationSnippets.java      |  12 +-
 .../genscavenge/ThreadLocalAllocation.java    |  41 +++--
 .../graal/GenScavengeAllocationSnippets.java  | 157 +++++++++++++++++-
 .../graal/nodes/FormatPodNode.java            |  99 +++++++++++
 .../core/graal/nodes/NewPodInstanceNode.java  |  68 ++++++++
 .../src/com/oracle/svm/core/heap/Pod.java     |  23 ---
 .../heap/PodFactorySubstitutionMethod.java    |  35 ++--
 7 files changed, 364 insertions(+), 71 deletions(-)
 create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
 create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java

diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/AllocationSnippets.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/AllocationSnippets.java
index cf7632e4a4bb..26d20bd54465 100644
--- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/AllocationSnippets.java
+++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/AllocationSnippets.java
@@ -112,7 +112,7 @@ protected Object newMultiArrayImpl(Word hub, int rank, int[] dimensions) {
         return callNewMultiArrayStub(hub, rank, dims);
     }
 
-    private UnsignedWord arrayAllocationSize(int length, int arrayBaseOffset, int log2ElementSize) {
+    protected UnsignedWord arrayAllocationSize(int length, int arrayBaseOffset, int log2ElementSize) {
         int alignment = objectAlignment();
         return WordFactory.unsigned(arrayAllocationSize(length, arrayBaseOffset, log2ElementSize, alignment));
     }
@@ -379,7 +379,7 @@ public static FillContent fromBoolean(boolean fillContents) {
     }
 
     public static class AllocationProfilingData {
-        final AllocationSnippetCounters snippetCounters;
+        public final AllocationSnippetCounters snippetCounters;
 
         public AllocationProfilingData(AllocationSnippetCounters snippetCounters) {
             this.snippetCounters = snippetCounters;
@@ -395,9 +395,9 @@ public AllocationSnippetCounters(SnippetCounter.Group.Factory factory) {
             stub = new SnippetCounter(allocations, "stub", "alloc and zeroing via stub");
         }
 
-        final SnippetCounter unrolledInit;
-        final SnippetCounter loopInit;
-        final SnippetCounter bulkInit;
-        final SnippetCounter stub;
+        public final SnippetCounter unrolledInit;
+        public final SnippetCounter loopInit;
+        public final SnippetCounter bulkInit;
+        public final SnippetCounter stub;
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
index 3001158d58ee..61b72a603998 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
@@ -52,6 +52,7 @@
 import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
+import com.oracle.svm.core.genscavenge.graal.nodes.FormatPodNode;
 import com.oracle.svm.core.graal.snippets.DeoptTester;
 import com.oracle.svm.core.heap.OutOfMemoryUtil;
 import com.oracle.svm.core.hub.DynamicHub;
@@ -223,6 +224,15 @@ private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub) {
 
     @SubstrateForeignCallTarget(stubCallingConvention = false)
     private static Object slowPathNewArray(Word objectHeader, int length, int fillStartOffset) {
+        return slowPathNewArrayImpl(objectHeader, length, fillStartOffset, null);
+    }
+
+    @SubstrateForeignCallTarget(stubCallingConvention = false)
+    private static Object slowPathNewPodInstance(Word objectHeader, int sizeWithoutRefMap, int fillStartOffset, byte[] referenceMap) {
+        return slowPathNewArrayImpl(objectHeader, sizeWithoutRefMap, fillStartOffset, referenceMap);
+    }
+
+    private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fillStartOffset, byte[] podReferenceMap) {
         /*
          * Avoid stack overflow errors while producing memory chunks, because that could leave the
          * heap in an inconsistent state.
@@ -246,7 +256,7 @@ private static Object slowPathNewArray(Word objectHeader, int length, int fillSt
                 throw OutOfMemoryUtil.reportOutOfMemoryError(outOfMemoryError);
             }
 
-            Object result = slowPathNewArrayWithoutAllocating(hub, length, size, fillStartOffset);
+            Object result = slowPathNewArrayWithoutAllocating(hub, length, size, fillStartOffset, podReferenceMap);
             runSlowPathHooks();
             return result;
         } finally {
@@ -255,7 +265,7 @@ private static Object slowPathNewArray(Word objectHeader, int length, int fillSt
     }
 
     @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in the implementation of allocation.")
-    private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset) {
+    private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
         DeoptTester.disableDeoptTesting();
         try {
             HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayWithoutAllocating", DynamicHub.toClass(hub).getName());
@@ -265,16 +275,16 @@ private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int leng
                 /* Large arrays go into their own unaligned chunk. */
                 boolean needsZeroing = !HeapChunkProvider.areUnalignedChunksZeroed();
                 UnalignedHeapChunk.UnalignedHeader newTlabChunk = HeapImpl.getChunkProvider().produceUnalignedChunk(size);
-                return allocateLargeArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, needsZeroing);
+                return allocateLargeArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, needsZeroing, podReferenceMap);
             }
             /* Small arrays go into the regular aligned chunk. */
 
             // We might have allocated in the caller and acquired a TLAB with enough space already
             // (but we need to check in an uninterruptible method to be safe)
-            Object array = allocateSmallArrayInCurrentTlab(hub, length, size, fillStartOffset);
+            Object array = allocateSmallArrayInCurrentTlab(hub, length, size, fillStartOffset, podReferenceMap);
             if (array == null) { // We need a new chunk.
                 AlignedHeader newTlabChunk = HeapImpl.getChunkProvider().produceAlignedChunk();
-                array = allocateSmallArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk);
+                array = allocateSmallArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, podReferenceMap);
             }
             return array;
         } finally {
@@ -290,22 +300,22 @@ private static Object allocateInstanceInNewTlab(DynamicHub hub, AlignedHeader ne
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
-    private static Object allocateSmallArrayInCurrentTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset) {
+    private static Object allocateSmallArrayInCurrentTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
         if (size.aboveThan(availableTlabMemory(getTlab()))) {
             return null;
         }
         Pointer memory = allocateRawMemoryInTlab(size, getTlab());
-        return FormatArrayNode.formatArray(memory, DynamicHub.toClass(hub), length, false, false, FillContent.WITH_ZEROES, fillStartOffset, true);
+        return formatArrayOrPod(memory, hub, length, false, FillContent.WITH_ZEROES, fillStartOffset, podReferenceMap);
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
-    private static Object allocateSmallArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, AlignedHeader newTlabChunk) {
+    private static Object allocateSmallArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, AlignedHeader newTlabChunk, byte[] podReferenceMap) {
         Pointer memory = allocateRawMemoryInNewTlab(size, newTlabChunk);
-        return FormatArrayNode.formatArray(memory, DynamicHub.toClass(hub), length, false, false, FillContent.WITH_ZEROES, fillStartOffset, true);
+        return formatArrayOrPod(memory, hub, length, false, FillContent.WITH_ZEROES, fillStartOffset, podReferenceMap);
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory, modifies TLAB")
-    private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, UnalignedHeapChunk.UnalignedHeader newTlabChunk, boolean needsZeroing) {
+    private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, UnalignedHeader newTlabChunk, boolean needsZeroing, byte[] podReferenceMap) {
         ThreadLocalAllocation.Descriptor tlab = getTlab();
 
         HeapChunk.setNext(newTlabChunk, tlab.getUnalignedChunk());
@@ -327,7 +337,16 @@ private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, Un
          * any way.
          */
         FillContent fillKind = needsZeroing ? FillContent.WITH_ZEROES : FillContent.DO_NOT_FILL;
-        return FormatArrayNode.formatArray(memory, DynamicHub.toClass(hub), length, false, true, fillKind, fillStartOffset, true);
+        return formatArrayOrPod(memory, hub, length, true, fillKind, fillStartOffset, podReferenceMap);
+    }
+
+    @Uninterruptible(reason = "Holds uninitialized memory")
+    private static Object formatArrayOrPod(Pointer memory, DynamicHub hub, int length, boolean unaligned, FillContent fillContent, int fillStartOffset, byte[] podReferenceMap) {
+        Class<?> clazz = DynamicHub.toClass(hub);
+        if (podReferenceMap != null) {
+            return FormatPodNode.formatPod(memory, clazz, length, podReferenceMap, false, unaligned, fillStartOffset, true);
+        }
+        return FormatArrayNode.formatArray(memory, clazz, length, false, unaligned, fillContent, fillStartOffset, true);
     }
 
     @Uninterruptible(reason = "Returns uninitialized memory, modifies TLAB", callerMustBe = true)
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
index dbb1d3cf308a..7db796b9b53a 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
@@ -24,32 +24,50 @@
  */
 package com.oracle.svm.core.genscavenge.graal;
 
+import static org.graalvm.compiler.nodes.PiArrayNode.piArrayCastToSnippetReplaceeStamp;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
+
+import java.util.Arrays;
 import java.util.Map;
 
+import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.compiler.api.replacements.Snippet;
 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
+import org.graalvm.compiler.api.replacements.Snippet.NonNullParameter;
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
 import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
+import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.SnippetAnchorNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.util.Providers;
+import org.graalvm.compiler.replacements.ReplacementsUtil;
 import org.graalvm.compiler.replacements.SnippetCounter;
 import org.graalvm.compiler.replacements.SnippetTemplate;
 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
+import org.graalvm.compiler.word.ObjectAccess;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.UnsignedWord;
 
+import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.genscavenge.HeapParameters;
 import com.oracle.svm.core.genscavenge.ObjectHeaderImpl;
 import com.oracle.svm.core.genscavenge.ThreadLocalAllocation;
 import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
 import com.oracle.svm.core.genscavenge.graal.nodes.FormatObjectNode;
+import com.oracle.svm.core.genscavenge.graal.nodes.FormatPodNode;
 import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
+import com.oracle.svm.core.graal.nodes.NewPodInstanceNode;
 import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
 import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets;
 import com.oracle.svm.core.heap.Heap;
@@ -58,10 +76,13 @@
 import com.oracle.svm.core.snippets.SnippetRuntime;
 import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
 
+import jdk.vm.ci.meta.JavaKind;
+
 final class GenScavengeAllocationSnippets extends SubstrateAllocationSnippets {
     private static final SubstrateForeignCallDescriptor SLOW_NEW_INSTANCE = SnippetRuntime.findForeignCall(ThreadLocalAllocation.class, "slowPathNewInstance", true);
     private static final SubstrateForeignCallDescriptor SLOW_NEW_ARRAY = SnippetRuntime.findForeignCall(ThreadLocalAllocation.class, "slowPathNewArray", true);
-    private static final SubstrateForeignCallDescriptor[] FOREIGN_CALLS = new SubstrateForeignCallDescriptor[]{SLOW_NEW_INSTANCE, SLOW_NEW_ARRAY};
+    private static final SubstrateForeignCallDescriptor SLOW_NEW_POD_INSTANCE = SnippetRuntime.findForeignCall(ThreadLocalAllocation.class, "slowPathNewPodInstance", true);
+    private static final SubstrateForeignCallDescriptor[] FOREIGN_CALLS = new SubstrateForeignCallDescriptor[]{SLOW_NEW_INSTANCE, SLOW_NEW_ARRAY, SLOW_NEW_POD_INSTANCE};
 
     public static void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) {
         SubstrateAllocationSnippets.registerForeignCalls(foreignCalls);
@@ -93,12 +114,75 @@ public Object formatArraySnippet(Word memory, DynamicHub hub, int length, boolea
         int layoutEncoding = hubNonNull.getLayoutEncoding();
         UnsignedWord size = LayoutEncoding.getArraySize(layoutEncoding, length);
         Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, unaligned);
-        Object obj = formatArray(objectHeader, size, length, memory, fillContents, fillStartOffset,
-                        false, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
+        return formatArray(objectHeader, size, length, memory, fillContents, fillStartOffset,
+                        emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
+    }
+
+    @Snippet
+    public Object allocatePod(@NonNullParameter DynamicHub hub, int sizeWithoutRefMap, byte[] referenceMap, @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter boolean maybeUnroll,
+                    @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationProfilingData profilingData) {
+
+        Word thread = getTLABInfo();
+        Word top = readTlabTop(thread);
+        Word end = readTlabEnd(thread);
+        ReplacementsUtil.dynamicAssert(end.subtract(top).belowOrEqual(Integer.MAX_VALUE), "TLAB is too large");
+
+        int arrayBaseOffset = LayoutEncoding.getArrayBaseOffsetAsInt(hub.getLayoutEncoding());
+        UnsignedWord allocationSize = arrayAllocationSize(sizeWithoutRefMap, arrayBaseOffset, 0);
+        Word newTop = top.add(allocationSize);
+
+        Object instance;
+        if (useTLAB() && probability(FAST_PATH_PROBABILITY, shouldAllocateInTLAB(allocationSize, true)) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
+            writeTlabTop(thread, newTop);
+            emitPrefetchAllocate(newTop, true);
+            instance = formatPod(top, hub, sizeWithoutRefMap, referenceMap, false, false, afterArrayLengthOffset(),
+                            emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, profilingData.snippetCounters);
+        } else {
+            profilingData.snippetCounters.stub.inc();
+            instance = callSlowNewPodInstance(SLOW_NEW_POD_INSTANCE, encodeAsTLABObjectHeader(hub), sizeWithoutRefMap, afterArrayLengthOffset(), referenceMap);
+        }
+        profileAllocation(profilingData, allocationSize);
+        return piArrayCastToSnippetReplaceeStamp(verifyOop(instance), sizeWithoutRefMap);
+    }
+
+    @Snippet
+    public Object formatPodSnippet(Word memory, DynamicHub hub, int sizeWithoutRefMap, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier,
+                    @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationSnippetCounters snippetCounters) {
+
+        DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
+        byte[] refMapNonNull = (byte[]) PiNode.piCastNonNull(referenceMap, SnippetAnchorNode.anchor());
+        return formatPod(memory, hubNonNull, sizeWithoutRefMap, refMapNonNull, rememberedSet, unaligned, fillStartOffset,
+                        emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
+    }
+
+    private Object formatPod(Word memory, DynamicHub hub, int sizeWithoutRefMap, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset,
+                    boolean emitMemoryBarrier, boolean maybeUnroll, boolean supportsBulkZeroing, boolean supportsOptimizedFilling, AllocationSnippetCounters snippetCounters) {
+
+        int layoutEncoding = hub.getLayoutEncoding();
+        UnsignedWord allocationSize = LayoutEncoding.getArraySize(layoutEncoding, sizeWithoutRefMap);
+
+        Word objectHeader = encodeAsObjectHeader(hub, rememberedSet, unaligned);
+        Object instance = formatArray(objectHeader, allocationSize, sizeWithoutRefMap, memory, FillContent.WITH_ZEROES, fillStartOffset,
+                        false, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
+
+        int fromOffset = ConfigurationValues.getObjectLayout().getArrayBaseOffset(JavaKind.Byte);
+        int toOffset = LayoutEncoding.getArrayBaseOffsetAsInt(layoutEncoding) + sizeWithoutRefMap - referenceMap.length;
+        for (int i = 0; i < referenceMap.length; i++) {
+            byte b = ObjectAccess.readByte(referenceMap, fromOffset + i, byteArrayIdentity());
+            ObjectAccess.writeByte(instance, toOffset + i, b, LocationIdentity.INIT_LOCATION);
+        }
         emitMemoryBarrierIf(emitMemoryBarrier);
-        return obj;
+        return instance;
+    }
+
+    @Fold
+    static LocationIdentity byteArrayIdentity() {
+        return NamedLocationIdentity.getArrayLocation(JavaKind.Byte);
     }
 
+    @NodeIntrinsic(value = ForeignCallNode.class)
+    private static native Object callSlowNewPodInstance(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word hub, int length, int fillStartOffset, byte[] referenceMap);
+
     private static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet, boolean unaligned) {
         return ObjectHeaderImpl.encodeAsObjectHeader(hub, rememberedSet, unaligned);
     }
@@ -151,23 +235,29 @@ protected SubstrateForeignCallDescriptor getSlowNewArrayStub() {
     public static class Templates extends SubstrateAllocationSnippets.Templates {
         private final SnippetInfo formatObject;
         private final SnippetInfo formatArray;
+        private final SnippetInfo formatPod;
+        private final SnippetInfo allocatePod;
 
         Templates(SubstrateAllocationSnippets receiver, OptionValues options, SnippetCounter.Group.Factory groupFactory, Providers providers) {
             super(receiver, options, groupFactory, providers);
 
             formatObject = snippet(GenScavengeAllocationSnippets.class, "formatObjectSnippet", null, receiver);
             formatArray = snippet(GenScavengeAllocationSnippets.class, "formatArraySnippet", null, receiver);
+
+            Object[] podAllocLocations = Arrays.copyOf(ALLOCATION_LOCATIONS, ALLOCATION_LOCATIONS.length + 1);
+            podAllocLocations[podAllocLocations.length - 1] = byteArrayIdentity();
+            allocatePod = snippet(GenScavengeAllocationSnippets.class, "allocatePod", null, receiver, podAllocLocations);
+            formatPod = snippet(GenScavengeAllocationSnippets.class, "formatPodSnippet", null, receiver, byteArrayIdentity());
         }
 
         @Override
         public void registerLowerings(Map<Class<? extends Node>, NodeLoweringProvider<?>> lowerings) {
             super.registerLowerings(lowerings);
 
-            FormatObjectLowering formatObjectLowering = new FormatObjectLowering();
-            lowerings.put(FormatObjectNode.class, formatObjectLowering);
-
-            FormatArrayLowering formatArrayLowering = new FormatArrayLowering();
-            lowerings.put(FormatArrayNode.class, formatArrayLowering);
+            lowerings.put(FormatObjectNode.class, new FormatObjectLowering());
+            lowerings.put(FormatArrayNode.class, new FormatArrayLowering());
+            lowerings.put(FormatPodNode.class, new FormatPodLowering());
+            lowerings.put(NewPodInstanceNode.class, new NewPodInstanceLowering());
         }
 
         private class FormatObjectLowering implements NodeLoweringProvider<FormatObjectNode> {
@@ -210,5 +300,54 @@ public void lower(FormatArrayNode node, LoweringTool tool) {
                 template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
             }
         }
+
+        private class NewPodInstanceLowering implements NodeLoweringProvider<NewPodInstanceNode> {
+            @Override
+            public void lower(NewPodInstanceNode node, LoweringTool tool) {
+                StructuredGraph graph = node.graph();
+                if (graph.getGuardsStage() != StructuredGraph.GuardsStage.AFTER_FSA) {
+                    return;
+                }
+
+                assert node.getKnownInstanceType() == null || (node.getHub().isConstant() &&
+                                providers.getConstantReflection().asJavaType(node.getHub().asConstant()).equals(node.getKnownInstanceType()));
+                assert node.fillContents() : "fillContents must be true for hybrid allocations";
+
+                Arguments args = new Arguments(allocatePod, graph.getGuardsStage(), tool.getLoweringStage());
+                args.add("hub", node.getHub());
+                args.add("sizeWithoutRefMap", node.getSizeWithoutRefMap());
+                args.add("referenceMap", node.getReferenceMap());
+                args.addConst("emitMemoryBarrier", node.emitMemoryBarrier());
+                args.addConst("maybeUnroll", node.getSizeWithoutRefMap().isConstant());
+                args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
+                args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
+                args.addConst("profilingData", getProfilingData(node, node.getKnownInstanceType()));
+
+                template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
+            }
+        }
+
+        private class FormatPodLowering implements NodeLoweringProvider<FormatPodNode> {
+            @Override
+            public void lower(FormatPodNode node, LoweringTool tool) {
+                StructuredGraph graph = node.graph();
+                if (graph.getGuardsStage() != StructuredGraph.GuardsStage.AFTER_FSA) {
+                    return;
+                }
+                Arguments args = new Arguments(formatPod, graph.getGuardsStage(), tool.getLoweringStage());
+                args.add("memory", node.getMemory());
+                args.add("hub", node.getHub());
+                args.add("sizeWithoutRefMap", node.getSizeWithoutRefMap());
+                args.add("referenceMap", node.getReferenceMap());
+                args.add("rememberedSet", node.getRememberedSet());
+                args.add("unaligned", node.getUnaligned());
+                args.add("fillStartOffset", node.getFillStartOffset());
+                args.add("emitMemoryBarrier", node.getEmitMemoryBarrier());
+                args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
+                args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
+                args.addConst("snippetCounters", snippetCounters);
+                template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
+            }
+        }
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
new file mode 100644
index 000000000000..2f1a6f7cac29
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.genscavenge.graal.nodes;
+
+import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64;
+import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.FixedWithNextNode;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.word.Pointer;
+
+@NodeInfo(cycles = CYCLES_64, size = SIZE_64)
+public class FormatPodNode extends FixedWithNextNode implements Lowerable {
+    public static final NodeClass<FormatPodNode> TYPE = NodeClass.create(FormatPodNode.class);
+
+    @Input protected ValueNode memory;
+    @Input protected ValueNode hub;
+    @Input protected ValueNode sizeWithoutRefMap;
+    @Input protected ValueNode referenceMap;
+    @Input protected ValueNode rememberedSet;
+    @Input protected ValueNode unaligned;
+    @Input protected ValueNode fillStartOffset;
+    @Input protected ValueNode emitMemoryBarrier;
+
+    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode sizeWithoutRefMap, ValueNode referenceMap,
+                    ValueNode rememberedSet, ValueNode unaligned, ValueNode fillStartOffset, ValueNode emitMemoryBarrier) {
+        super(TYPE, StampFactory.objectNonNull());
+        this.memory = memory;
+        this.hub = hub;
+        this.sizeWithoutRefMap = sizeWithoutRefMap;
+        this.referenceMap = referenceMap;
+        this.rememberedSet = rememberedSet;
+        this.unaligned = unaligned;
+        this.fillStartOffset = fillStartOffset;
+        this.emitMemoryBarrier = emitMemoryBarrier;
+    }
+
+    public ValueNode getMemory() {
+        return memory;
+    }
+
+    public ValueNode getHub() {
+        return hub;
+    }
+
+    public ValueNode getSizeWithoutRefMap() {
+        return sizeWithoutRefMap;
+    }
+
+    public ValueNode getReferenceMap() {
+        return referenceMap;
+    }
+
+    public ValueNode getRememberedSet() {
+        return rememberedSet;
+    }
+
+    public ValueNode getUnaligned() {
+        return unaligned;
+    }
+
+    public ValueNode getFillStartOffset() {
+        return fillStartOffset;
+    }
+
+    public ValueNode getEmitMemoryBarrier() {
+        return emitMemoryBarrier;
+    }
+
+    @NodeIntrinsic
+    public static native Object formatPod(Pointer memory, Class<?> hub, int sizeWithoutRefMap, byte[] referenceMap,
+                    boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier);
+}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java
new file mode 100644
index 000000000000..893fc28b36cc
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.graal.nodes;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.core.common.type.TypeReference;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
+
+import jdk.vm.ci.meta.ResolvedJavaType;
+
+@NodeInfo
+public final class NewPodInstanceNode extends AbstractNewObjectNode {
+    public static final NodeClass<NewPodInstanceNode> TYPE = NodeClass.create(NewPodInstanceNode.class);
+
+    private final ResolvedJavaType knownInstanceType;
+    @Input ValueNode hub;
+    @Input ValueNode sizeWithoutRefMap;
+    @Input ValueNode referenceMap;
+
+    public NewPodInstanceNode(ResolvedJavaType knownInstanceType, ValueNode hub, ValueNode sizeWithoutRefMap, ValueNode referenceMap) {
+        super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(knownInstanceType)), true, null);
+        this.knownInstanceType = knownInstanceType;
+        this.hub = hub;
+        this.sizeWithoutRefMap = sizeWithoutRefMap;
+        this.referenceMap = referenceMap;
+    }
+
+    public ResolvedJavaType getKnownInstanceType() {
+        return knownInstanceType;
+    }
+
+    public ValueNode getHub() {
+        return hub;
+    }
+
+    public ValueNode getSizeWithoutRefMap() {
+        return sizeWithoutRefMap;
+    }
+
+    public ValueNode getReferenceMap() {
+        return referenceMap;
+    }
+}
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index 0a4c864e2207..51e5ded303cb 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -43,14 +43,12 @@
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.UnsignedWord;
 
-import com.oracle.svm.core.JavaMemoryUtil;
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.LayoutEncoding;
 import com.oracle.svm.core.util.ImageHeapMap;
 import com.oracle.svm.core.util.UnsignedUtils;
 
-import jdk.internal.misc.Unsafe;
 import jdk.vm.ci.meta.JavaKind;
 
 /**
@@ -84,27 +82,6 @@ public T getFactory() {
         return factory;
     }
 
-    /**
-     * Used from generated graphs: write {@link #referenceMap} to a newly allocated pod instance.
-     *
-     * {@link JavaMemoryUtil} copying is uninterruptible, so it cannot be inlined here and in our
-     * caller, and reference maps are likely too small to benefit from its optimizations. Instead,
-     * we use a simple interruptible copy loop that is safe because it does not use pointers and
-     * because a reference map decoder will not attempt to make sense of the reference map while the
-     * last two bytes (the first two it reads) remain zero, which is not a problem because all
-     * references are null at this point.
-     */
-    @SuppressWarnings("unused")
-    private void initInstanceRefMap(Object instance) {
-        Unsafe unsafe = Unsafe.getUnsafe();
-        int refMapArrayBase = unsafe.arrayBaseOffset(byte[].class);
-        int podRefMapBase = unsafe.arrayBaseOffset(podInfo.podClass) + fieldsSizeWithoutRefMap;
-        for (int offset = 0; offset < referenceMap.length; offset += 2) {
-            short entry = unsafe.getShort(referenceMap, refMapArrayBase + offset);
-            unsafe.putShort(instance, podRefMapBase + offset, entry);
-        }
-    }
-
     /**
      * A builder for constructing pods with a specific superpod or superclass and factory interface
      * and {@linkplain #addField additional fields}.
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
index 42ce8538343e..6b3be2ce9cd4 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
@@ -29,19 +29,20 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
+import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
+import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.FixedNode;
 import org.graalvm.compiler.nodes.FixedWithNextNode;
 import org.graalvm.compiler.nodes.FrameState;
 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
+import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.UnreachableBeginNode;
 import org.graalvm.compiler.nodes.UnwindNode;
 import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.nodes.calc.AddNode;
-import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 import org.graalvm.compiler.nodes.java.ExceptionObjectNode;
 
 import com.oracle.graal.pointsto.infrastructure.SubstitutionProcessor;
@@ -50,7 +51,7 @@
 import com.oracle.svm.core.graal.nodes.DeoptEntryBeginNode;
 import com.oracle.svm.core.graal.nodes.DeoptEntryNode;
 import com.oracle.svm.core.graal.nodes.LoweredDeadEndNode;
-import com.oracle.svm.core.graal.nodes.SubstrateNewHybridInstanceNode;
+import com.oracle.svm.core.graal.nodes.NewPodInstanceNode;
 import com.oracle.svm.core.graal.nodes.TestDeoptimizeNode;
 import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.Pod.RuntimeSupport.PodFactory;
@@ -108,7 +109,6 @@ public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method,
         ResolvedJavaType factoryType = method.getDeclaringClass();
         PodFactory annotation = factoryType.getAnnotation(PodFactory.class);
         ResolvedJavaType podConcreteType = kit.getMetaAccess().lookupJavaType(annotation.podClass());
-        ResolvedJavaType elementType = kit.getMetaAccess().lookupJavaType(byte.class);
         ResolvedJavaMethod targetCtor = null;
         for (ResolvedJavaMethod ctor : podConcreteType.getSuperclass().getDeclaredConstructors()) {
             if (parameterTypesMatch(method, ctor)) {
@@ -125,11 +125,10 @@ public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method,
          */
         int instanceLocal = method.getSignature().getParameterCount(true);
         int nextDeoptIndex = startMethod(kit, isDeoptTarget, 0);
-        instantiatePod(kit, factoryType, podConcreteType, elementType, instanceLocal);
+        instantiatePod(kit, providers, factoryType, podConcreteType, instanceLocal);
         if (isAnnotationPresent(DeoptTest.class)) {
             kit.append(new TestDeoptimizeNode());
         }
-        nextDeoptIndex = initInstanceRefMap(kit, method, isDeoptTarget, nextDeoptIndex, instanceLocal);
         nextDeoptIndex = invokeConstructor(kit, method, isDeoptTarget, nextDeoptIndex, targetCtor, instanceLocal);
 
         kit.createReturn(kit.loadLocal(instanceLocal, JavaKind.Object), JavaKind.Object);
@@ -144,22 +143,19 @@ private static int startMethod(HostedGraphKit kit, boolean isDeoptTarget, int ne
         return appendDeoptWithExceptionUnwind(kit, initialState, initialState.bci, nextDeoptIndex);
     }
 
-    private static void instantiatePod(HostedGraphKit kit, ResolvedJavaType factoryType, ResolvedJavaType podConcreteType, ResolvedJavaType elementType, int instanceLocal) {
+    private static void instantiatePod(HostedGraphKit kit, HostedProviders providers, ResolvedJavaType factoryType, ResolvedJavaType podConcreteType, int instanceLocal) {
         ResolvedJavaType podType = kit.getMetaAccess().lookupJavaType(Pod.class);
-        ValueNode pod = loadPod(kit, factoryType);
+        ValueNode receiver = kit.loadLocal(0, JavaKind.Object);
+        ValueNode pod = loadNonNullField(kit, receiver, findField(factoryType, "pod"));
         ValueNode sizeWithoutRefMap = kit.createLoadField(pod, findField(podType, "fieldsSizeWithoutRefMap"));
-        ValueNode refMapArray = kit.createLoadField(pod, findField(podType, "referenceMap"));
-        ValueNode refMapLength = kit.append(new ArrayLengthNode(refMapArray));
-        ValueNode hybridArrayLength = kit.append(AddNode.add(sizeWithoutRefMap, refMapLength));
-        ValueNode instance = kit.append(new SubstrateNewHybridInstanceNode(podConcreteType, elementType, hybridArrayLength));
+        ValueNode refMap = loadNonNullField(kit, pod, findField(podType, "referenceMap"));
+        ConstantNode hub = kit.createConstant(providers.getConstantReflection().asObjectHub(podConcreteType), JavaKind.Object);
+        ValueNode instance = kit.append(new NewPodInstanceNode(podConcreteType, hub, sizeWithoutRefMap, refMap));
         kit.storeLocal(instanceLocal, JavaKind.Object, instance);
     }
 
-    private static int initInstanceRefMap(HostedGraphKit kit, ResolvedJavaMethod method, boolean isDeoptTarget, int nextDeoptIndex, int instanceLocal) {
-        ValueNode instance = kit.loadLocal(instanceLocal, JavaKind.Object);
-        ResolvedJavaMethod initInstanceRefMap = kit.findMethod(Pod.class, "initInstanceRefMap", false);
-        ValueNode pod = loadPod(kit, method.getDeclaringClass());
-        return invokeWithDeoptAndExceptionUnwind(kit, isDeoptTarget, nextDeoptIndex, initInstanceRefMap, InvokeKind.Virtual, pod, instance);
+    private static ValueNode loadNonNullField(HostedGraphKit kit, ValueNode object, ResolvedJavaField field) {
+        return kit.append(PiNode.create(kit.createLoadField(object, field), StampFactory.objectNonNull()));
     }
 
     private static int invokeConstructor(HostedGraphKit kit, ResolvedJavaMethod method, boolean isDeoptTarget, int nextDeoptIndex, ResolvedJavaMethod targetCtor, int instanceLocal) {
@@ -170,11 +166,6 @@ private static int invokeConstructor(HostedGraphKit kit, ResolvedJavaMethod meth
         return invokeWithDeoptAndExceptionUnwind(kit, isDeoptTarget, nextDeoptIndex, targetCtor, InvokeKind.Special, invokeArgs);
     }
 
-    private static ValueNode loadPod(HostedGraphKit kit, ResolvedJavaType declaringClass) {
-        ValueNode receiver = kit.loadLocal(0, JavaKind.Object);
-        return kit.maybeCreateExplicitNullCheck(kit.createLoadField(receiver, findField(declaringClass, "pod")));
-    }
-
     /** @see com.oracle.svm.hosted.phases.HostedGraphBuilderPhase */
     private static int invokeWithDeoptAndExceptionUnwind(HostedGraphKit kit, boolean isDeoptTarget, int initialNextDeoptIndex, ResolvedJavaMethod target, InvokeKind invokeKind, ValueNode... args) {
         InvokeWithExceptionNode invoke = kit.startInvokeWithException(target, invokeKind, kit.getFrameState(), kit.bci(), args);

From 6caf268250e7d70efa2a2cb10e261f10e1f75e9e Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Mon, 2 May 2022 14:32:02 +0200
Subject: [PATCH 08/12] Implement Object.clone() for pods.

---
 .../genscavenge/ThreadLocalAllocation.java    |   4 +-
 .../graal/GenScavengeAllocationSnippets.java  |  28 ++--
 .../graal/nodes/FormatPodNode.java            |  12 +-
 .../graal/jdk/SubstrateObjectCloneNode.java   |   8 ++
 .../jdk/SubstrateObjectCloneSnippets.java     | 136 ++++++++++++------
 ...SubstrateObjectCloneWithExceptionNode.java |   8 ++
 .../core/graal/nodes/NewPodInstanceNode.java  |  13 +-
 .../src/com/oracle/svm/core/heap/Pod.java     |  11 +-
 .../svm/core/heap/PodReferenceMapDecoder.java |  75 +++++++++-
 .../heap/PodFactorySubstitutionMethod.java    |   4 +-
 10 files changed, 215 insertions(+), 84 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
index 61b72a603998..66020af76250 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
@@ -228,8 +228,8 @@ private static Object slowPathNewArray(Word objectHeader, int length, int fillSt
     }
 
     @SubstrateForeignCallTarget(stubCallingConvention = false)
-    private static Object slowPathNewPodInstance(Word objectHeader, int sizeWithoutRefMap, int fillStartOffset, byte[] referenceMap) {
-        return slowPathNewArrayImpl(objectHeader, sizeWithoutRefMap, fillStartOffset, referenceMap);
+    private static Object slowPathNewPodInstance(Word objectHeader, int arrayLength, int fillStartOffset, byte[] referenceMap) {
+        return slowPathNewArrayImpl(objectHeader, arrayLength, fillStartOffset, referenceMap);
     }
 
     private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fillStartOffset, byte[] podReferenceMap) {
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
index 7db796b9b53a..278b6f8bbed8 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
@@ -119,7 +119,7 @@ public Object formatArraySnippet(Word memory, DynamicHub hub, int length, boolea
     }
 
     @Snippet
-    public Object allocatePod(@NonNullParameter DynamicHub hub, int sizeWithoutRefMap, byte[] referenceMap, @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter boolean maybeUnroll,
+    public Object allocatePod(@NonNullParameter DynamicHub hub, int arrayLength, byte[] referenceMap, @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter boolean maybeUnroll,
                     @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationProfilingData profilingData) {
 
         Word thread = getTLABInfo();
@@ -128,45 +128,45 @@ public Object allocatePod(@NonNullParameter DynamicHub hub, int sizeWithoutRefMa
         ReplacementsUtil.dynamicAssert(end.subtract(top).belowOrEqual(Integer.MAX_VALUE), "TLAB is too large");
 
         int arrayBaseOffset = LayoutEncoding.getArrayBaseOffsetAsInt(hub.getLayoutEncoding());
-        UnsignedWord allocationSize = arrayAllocationSize(sizeWithoutRefMap, arrayBaseOffset, 0);
+        UnsignedWord allocationSize = arrayAllocationSize(arrayLength, arrayBaseOffset, 0);
         Word newTop = top.add(allocationSize);
 
         Object instance;
         if (useTLAB() && probability(FAST_PATH_PROBABILITY, shouldAllocateInTLAB(allocationSize, true)) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
             writeTlabTop(thread, newTop);
             emitPrefetchAllocate(newTop, true);
-            instance = formatPod(top, hub, sizeWithoutRefMap, referenceMap, false, false, afterArrayLengthOffset(),
+            instance = formatPod(top, hub, arrayLength, referenceMap, false, false, afterArrayLengthOffset(),
                             emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, profilingData.snippetCounters);
         } else {
             profilingData.snippetCounters.stub.inc();
-            instance = callSlowNewPodInstance(SLOW_NEW_POD_INSTANCE, encodeAsTLABObjectHeader(hub), sizeWithoutRefMap, afterArrayLengthOffset(), referenceMap);
+            instance = callSlowNewPodInstance(SLOW_NEW_POD_INSTANCE, encodeAsTLABObjectHeader(hub), arrayLength, afterArrayLengthOffset(), referenceMap);
         }
         profileAllocation(profilingData, allocationSize);
-        return piArrayCastToSnippetReplaceeStamp(verifyOop(instance), sizeWithoutRefMap);
+        return piArrayCastToSnippetReplaceeStamp(verifyOop(instance), arrayLength);
     }
 
     @Snippet
-    public Object formatPodSnippet(Word memory, DynamicHub hub, int sizeWithoutRefMap, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier,
+    public Object formatPodSnippet(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier,
                     @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationSnippetCounters snippetCounters) {
 
         DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
         byte[] refMapNonNull = (byte[]) PiNode.piCastNonNull(referenceMap, SnippetAnchorNode.anchor());
-        return formatPod(memory, hubNonNull, sizeWithoutRefMap, refMapNonNull, rememberedSet, unaligned, fillStartOffset,
+        return formatPod(memory, hubNonNull, arrayLength, refMapNonNull, rememberedSet, unaligned, fillStartOffset,
                         emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
     }
 
-    private Object formatPod(Word memory, DynamicHub hub, int sizeWithoutRefMap, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset,
+    private Object formatPod(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset,
                     boolean emitMemoryBarrier, boolean maybeUnroll, boolean supportsBulkZeroing, boolean supportsOptimizedFilling, AllocationSnippetCounters snippetCounters) {
 
         int layoutEncoding = hub.getLayoutEncoding();
-        UnsignedWord allocationSize = LayoutEncoding.getArraySize(layoutEncoding, sizeWithoutRefMap);
+        UnsignedWord allocationSize = LayoutEncoding.getArraySize(layoutEncoding, arrayLength);
 
         Word objectHeader = encodeAsObjectHeader(hub, rememberedSet, unaligned);
-        Object instance = formatArray(objectHeader, allocationSize, sizeWithoutRefMap, memory, FillContent.WITH_ZEROES, fillStartOffset,
+        Object instance = formatArray(objectHeader, allocationSize, arrayLength, memory, FillContent.WITH_ZEROES, fillStartOffset,
                         false, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
 
         int fromOffset = ConfigurationValues.getObjectLayout().getArrayBaseOffset(JavaKind.Byte);
-        int toOffset = LayoutEncoding.getArrayBaseOffsetAsInt(layoutEncoding) + sizeWithoutRefMap - referenceMap.length;
+        int toOffset = LayoutEncoding.getArrayBaseOffsetAsInt(layoutEncoding) + arrayLength - referenceMap.length;
         for (int i = 0; i < referenceMap.length; i++) {
             byte b = ObjectAccess.readByte(referenceMap, fromOffset + i, byteArrayIdentity());
             ObjectAccess.writeByte(instance, toOffset + i, b, LocationIdentity.INIT_LOCATION);
@@ -315,10 +315,10 @@ public void lower(NewPodInstanceNode node, LoweringTool tool) {
 
                 Arguments args = new Arguments(allocatePod, graph.getGuardsStage(), tool.getLoweringStage());
                 args.add("hub", node.getHub());
-                args.add("sizeWithoutRefMap", node.getSizeWithoutRefMap());
+                args.add("arrayLength", node.getArrayLength());
                 args.add("referenceMap", node.getReferenceMap());
                 args.addConst("emitMemoryBarrier", node.emitMemoryBarrier());
-                args.addConst("maybeUnroll", node.getSizeWithoutRefMap().isConstant());
+                args.addConst("maybeUnroll", node.getArrayLength().isConstant());
                 args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
                 args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
                 args.addConst("profilingData", getProfilingData(node, node.getKnownInstanceType()));
@@ -337,7 +337,7 @@ public void lower(FormatPodNode node, LoweringTool tool) {
                 Arguments args = new Arguments(formatPod, graph.getGuardsStage(), tool.getLoweringStage());
                 args.add("memory", node.getMemory());
                 args.add("hub", node.getHub());
-                args.add("sizeWithoutRefMap", node.getSizeWithoutRefMap());
+                args.add("arrayLength", node.getArrayLength());
                 args.add("referenceMap", node.getReferenceMap());
                 args.add("rememberedSet", node.getRememberedSet());
                 args.add("unaligned", node.getUnaligned());
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
index 2f1a6f7cac29..bf4383f1973e 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
@@ -41,19 +41,19 @@ public class FormatPodNode extends FixedWithNextNode implements Lowerable {
 
     @Input protected ValueNode memory;
     @Input protected ValueNode hub;
-    @Input protected ValueNode sizeWithoutRefMap;
+    @Input protected ValueNode arrayLength;
     @Input protected ValueNode referenceMap;
     @Input protected ValueNode rememberedSet;
     @Input protected ValueNode unaligned;
     @Input protected ValueNode fillStartOffset;
     @Input protected ValueNode emitMemoryBarrier;
 
-    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode sizeWithoutRefMap, ValueNode referenceMap,
+    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode arrayLength, ValueNode referenceMap,
                     ValueNode rememberedSet, ValueNode unaligned, ValueNode fillStartOffset, ValueNode emitMemoryBarrier) {
         super(TYPE, StampFactory.objectNonNull());
         this.memory = memory;
         this.hub = hub;
-        this.sizeWithoutRefMap = sizeWithoutRefMap;
+        this.arrayLength = arrayLength;
         this.referenceMap = referenceMap;
         this.rememberedSet = rememberedSet;
         this.unaligned = unaligned;
@@ -69,8 +69,8 @@ public ValueNode getHub() {
         return hub;
     }
 
-    public ValueNode getSizeWithoutRefMap() {
-        return sizeWithoutRefMap;
+    public ValueNode getArrayLength() {
+        return arrayLength;
     }
 
     public ValueNode getReferenceMap() {
@@ -94,6 +94,6 @@ public ValueNode getEmitMemoryBarrier() {
     }
 
     @NodeIntrinsic
-    public static native Object formatPod(Pointer memory, Class<?> hub, int sizeWithoutRefMap, byte[] referenceMap,
+    public static native Object formatPod(Pointer memory, Class<?> hub, int arrayLength, byte[] referenceMap,
                     boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier);
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneNode.java
index 3e228e47b514..529ffe7d986a 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneNode.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneNode.java
@@ -36,6 +36,7 @@
 import org.graalvm.compiler.nodes.java.LoadFieldNode;
 import org.graalvm.compiler.nodes.spi.Lowerable;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.nodes.spi.VirtualizerTool;
 import org.graalvm.compiler.replacements.nodes.BasicObjectCloneNode;
 import org.graalvm.compiler.replacements.nodes.MacroNode;
 
@@ -95,4 +96,11 @@ public void setStateBefore(FrameState f) {
         updateUsages(stateBefore, f);
         stateBefore = f;
     }
+
+    @Override
+    public void virtualize(VirtualizerTool tool) {
+        if (SubstrateObjectCloneSnippets.canVirtualize(this, tool)) {
+            super.virtualize(tool);
+        }
+    }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
index dc4726ab8350..81c04217e767 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
@@ -25,26 +25,31 @@
 package com.oracle.svm.core.graal.jdk;
 
 import static org.graalvm.compiler.nodes.PiNode.piCastToSnippetReplaceeStamp;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
 
 import java.util.Map;
 
 import org.graalvm.compiler.api.replacements.Snippet;
-import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
 import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.nodes.spi.VirtualizerTool;
+import org.graalvm.compiler.nodes.virtual.VirtualObjectNode;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.replacements.SnippetTemplate;
 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 import org.graalvm.compiler.replacements.Snippets;
+import org.graalvm.compiler.replacements.nodes.ObjectClone;
 import org.graalvm.compiler.word.BarrieredAccess;
 import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.WordFactory;
@@ -57,16 +62,21 @@
 import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
 import com.oracle.svm.core.graal.snippets.SubstrateTemplates;
 import com.oracle.svm.core.heap.InstanceReferenceMapEncoder;
+import com.oracle.svm.core.heap.PodReferenceMapDecoder;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.DynamicHubSupport;
 import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.meta.SharedType;
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.snippets.SnippetRuntime;
 import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
 import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
 import com.oracle.svm.core.util.NonmovableByteArrayReader;
+import com.oracle.svm.core.util.UnsignedUtils;
+import com.oracle.svm.core.util.VMError;
 
 import jdk.internal.misc.Unsafe;
+import jdk.vm.ci.meta.ResolvedJavaType;
 
 public final class SubstrateObjectCloneSnippets extends SubstrateTemplates implements Snippets {
     private static final SubstrateForeignCallDescriptor CLONE = SnippetRuntime.findForeignCall(SubstrateObjectCloneSnippets.class, "doClone", true, LocationIdentity.any());
@@ -86,65 +96,97 @@ private static Object doClone(Object original) throws CloneNotSupportedException
 
         DynamicHub hub = KnownIntrinsics.readHub(original);
         int layoutEncoding = hub.getLayoutEncoding();
-        if (LayoutEncoding.isArray(layoutEncoding)) {
-            int length = ArrayLengthNode.arrayLength(original);
-            Object newArray = java.lang.reflect.Array.newInstance(DynamicHub.toClass(hub.getComponentHub()), length);
-            if (LayoutEncoding.isObjectArray(layoutEncoding)) {
-                JavaMemoryUtil.copyObjectArrayForward(original, 0, newArray, 0, length, layoutEncoding);
+        boolean hasArray = LayoutEncoding.isArray(layoutEncoding);
+
+        Object result;
+        if (hasArray) {
+            // Hybrids like pods have an array encoding, but a non-array hub type.
+            if (BranchProbabilityNode.probability(FAST_PATH_PROBABILITY, hub.isArray())) {
+                int length = ArrayLengthNode.arrayLength(original);
+                Object newArray = java.lang.reflect.Array.newInstance(DynamicHub.toClass(hub.getComponentHub()), length);
+                if (LayoutEncoding.isObjectArray(layoutEncoding)) {
+                    JavaMemoryUtil.copyObjectArrayForward(original, 0, newArray, 0, length, layoutEncoding);
+                } else {
+                    JavaMemoryUtil.copyPrimitiveArrayForward(original, 0, newArray, 0, length, layoutEncoding);
+                }
+                return newArray;
+
+            } else if (hub.isPodInstanceClass()) {
+                result = PodReferenceMapDecoder.clone(original, hub, layoutEncoding);
+
             } else {
-                JavaMemoryUtil.copyPrimitiveArrayForward(original, 0, newArray, 0, length, layoutEncoding);
+                throw VMError.shouldNotReachHere("Hybrid classes do not support Object.clone().");
             }
-            return newArray;
         } else {
-            Object result = Unsafe.getUnsafe().allocateInstance(DynamicHub.toClass(hub));
-            int firstFieldOffset = ConfigurationValues.getObjectLayout().getFirstFieldOffset();
-            int curOffset = firstFieldOffset;
-
-            NonmovableArray<Byte> referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding();
-            int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
-            int referenceMapIndex = hub.getReferenceMapIndex();
-            int entryCount = NonmovableByteArrayReader.getS4(referenceMapEncoding, referenceMapIndex);
-            assert entryCount >= 0;
-
-            // The UniverseBuilder actively groups object references together. So, this loop will
-            // typically be only executed for a very small number of iterations.
-            long entryStart = referenceMapIndex + InstanceReferenceMapEncoder.MAP_HEADER_SIZE;
-            for (long idx = entryStart; idx < entryStart + entryCount * InstanceReferenceMapEncoder.MAP_ENTRY_SIZE; idx += InstanceReferenceMapEncoder.MAP_ENTRY_SIZE) {
-                int objectOffset = NonmovableByteArrayReader.getS4(referenceMapEncoding, idx);
-                long count = NonmovableByteArrayReader.getU4(referenceMapEncoding, idx + 4);
-                assert objectOffset >= firstFieldOffset : "must not overwrite the object header";
-
-                // copy non-object data
-                int primitiveDataSize = objectOffset - curOffset;
-                assert primitiveDataSize >= 0;
-                assert curOffset >= 0;
-                JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
-                curOffset += primitiveDataSize;
-
-                // copy object data
-                assert curOffset >= 0;
-                assert count >= 0;
-                JavaMemoryUtil.copyReferencesForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(count));
-                curOffset += count * referenceSize;
-            }
+            result = Unsafe.getUnsafe().allocateInstance(DynamicHub.toClass(hub));
+        }
+
+        int firstFieldOffset = ConfigurationValues.getObjectLayout().getFirstFieldOffset();
+        int curOffset = firstFieldOffset;
+
+        NonmovableArray<Byte> referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding();
+        int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
+        int referenceMapIndex = hub.getReferenceMapIndex();
+        int entryCount = NonmovableByteArrayReader.getS4(referenceMapEncoding, referenceMapIndex);
+        assert entryCount >= 0;
+
+        // The UniverseBuilder actively groups object references together. So, this loop will
+        // typically be only executed for a very small number of iterations.
+        long entryStart = referenceMapIndex + InstanceReferenceMapEncoder.MAP_HEADER_SIZE;
+        for (long idx = entryStart; idx < entryStart + entryCount * InstanceReferenceMapEncoder.MAP_ENTRY_SIZE; idx += InstanceReferenceMapEncoder.MAP_ENTRY_SIZE) {
+            int objectOffset = NonmovableByteArrayReader.getS4(referenceMapEncoding, idx);
+            long count = NonmovableByteArrayReader.getU4(referenceMapEncoding, idx + 4);
+            assert objectOffset >= firstFieldOffset : "must not overwrite the object header";
+
+            // copy non-object data
+            int primitiveDataSize = objectOffset - curOffset;
+            assert primitiveDataSize >= 0;
+            assert curOffset >= 0;
+            JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
+            curOffset += primitiveDataSize;
+
+            // copy object data
+            assert curOffset >= 0;
+            assert count >= 0;
+            JavaMemoryUtil.copyReferencesForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(count));
+            curOffset += count * referenceSize;
+        }
 
-            // copy remaining non-object data
-            int objectSize = NumUtil.safeToInt(LayoutEncoding.getInstanceSize(layoutEncoding).rawValue());
+        // copy remaining non-object data
+        if (!hasArray) {
+            int objectSize = UnsignedUtils.safeToInt(LayoutEncoding.getInstanceSize(layoutEncoding));
             int primitiveDataSize = objectSize - curOffset;
             assert primitiveDataSize >= 0;
             assert curOffset >= 0;
             JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
             curOffset += primitiveDataSize;
             assert curOffset == objectSize;
+        }
 
-            // reset monitor to uninitialized values
-            int monitorOffset = hub.getMonitorOffset();
-            if (monitorOffset != 0) {
-                BarrieredAccess.writeObject(result, monitorOffset, null);
-            }
+        // reset monitor to uninitialized values
+        int monitorOffset = hub.getMonitorOffset();
+        if (monitorOffset != 0) {
+            BarrieredAccess.writeObject(result, monitorOffset, null);
+        }
 
-            return result;
+        return result;
+    }
+
+    static boolean canVirtualize(ObjectClone node, VirtualizerTool tool) {
+        ValueNode alias = tool.getAlias(node.getObject());
+        if (alias instanceof VirtualObjectNode) {
+            return true;
+        }
+        ResolvedJavaType type = node.getConcreteType(alias.stamp(NodeView.DEFAULT));
+        if (type == null) {
+            return false;
+        }
+        if (!type.isArray() && type instanceof SharedType) {
+            // Hybrids are instances with array encoding; cloning them virtually is not implemented.
+            int encoding = ((SharedType) type).getHub().getLayoutEncoding();
+            return !LayoutEncoding.isArray(encoding);
         }
+        return true;
     }
 
     @NodeIntrinsic(value = ForeignCallNode.class)
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneWithExceptionNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneWithExceptionNode.java
index 0e74d9abb4a2..117c122d0112 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneWithExceptionNode.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneWithExceptionNode.java
@@ -39,6 +39,7 @@
 import org.graalvm.compiler.nodes.WithExceptionNode;
 import org.graalvm.compiler.nodes.memory.SingleMemoryKill;
 import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.compiler.nodes.spi.VirtualizerTool;
 import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.replacements.nodes.MacroNode.MacroParams;
 import org.graalvm.compiler.replacements.nodes.ObjectClone;
@@ -133,4 +134,11 @@ public FixedNode replaceWithNonThrowing() {
         GraphUtil.killCFG(oldException);
         return plainObjectClone;
     }
+
+    @Override
+    public void virtualize(VirtualizerTool tool) {
+        if (SubstrateObjectCloneSnippets.canVirtualize(this, tool)) {
+            ObjectClone.super.virtualize(tool);
+        }
+    }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java
index 893fc28b36cc..f7c5f0fb2a65 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/nodes/NewPodInstanceNode.java
@@ -39,14 +39,14 @@ public final class NewPodInstanceNode extends AbstractNewObjectNode {
 
     private final ResolvedJavaType knownInstanceType;
     @Input ValueNode hub;
-    @Input ValueNode sizeWithoutRefMap;
+    @Input ValueNode arrayLength;
     @Input ValueNode referenceMap;
 
-    public NewPodInstanceNode(ResolvedJavaType knownInstanceType, ValueNode hub, ValueNode sizeWithoutRefMap, ValueNode referenceMap) {
+    public NewPodInstanceNode(ResolvedJavaType knownInstanceType, ValueNode hub, ValueNode arrayLength, ValueNode referenceMap) {
         super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(knownInstanceType)), true, null);
         this.knownInstanceType = knownInstanceType;
         this.hub = hub;
-        this.sizeWithoutRefMap = sizeWithoutRefMap;
+        this.arrayLength = arrayLength;
         this.referenceMap = referenceMap;
     }
 
@@ -58,11 +58,14 @@ public ValueNode getHub() {
         return hub;
     }
 
-    public ValueNode getSizeWithoutRefMap() {
-        return sizeWithoutRefMap;
+    public ValueNode getArrayLength() {
+        return arrayLength;
     }
 
     public ValueNode getReferenceMap() {
         return referenceMap;
     }
+
+    @NodeIntrinsic
+    public static native Object newPodInstance(@ConstantNodeParameter Class<?> knownInstanceClass, Class<?> runtimeClass, int arrayLength, byte[] referenceMap);
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index 51e5ded303cb..fa2778cd3ced 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -61,11 +61,11 @@
  */
 public final class Pod<T> {
     private final RuntimeSupport.PodInfo podInfo;
-    private final int fieldsSizeWithoutRefMap;
+    private final int arrayLength;
     private final byte[] referenceMap;
     private final T factory;
 
-    private Pod(RuntimeSupport.PodInfo podInfo, int fieldsSize, byte[] referenceMap) {
+    private Pod(RuntimeSupport.PodInfo podInfo, int arrayLength, byte[] referenceMap) {
         this.podInfo = podInfo;
         try {
             @SuppressWarnings("unchecked")
@@ -74,7 +74,7 @@ private Pod(RuntimeSupport.PodInfo podInfo, int fieldsSize, byte[] referenceMap)
         } catch (ReflectiveOperationException e) {
             throw new RuntimeException(e);
         }
-        this.fieldsSizeWithoutRefMap = fieldsSize;
+        this.arrayLength = arrayLength;
         this.referenceMap = referenceMap;
     }
 
@@ -181,8 +181,8 @@ public Pod<T> build() {
             byte[] superRefMap = null;
             UnsignedWord nextOffset = baseOffset;
             if (superPod != null) {
-                nextOffset = nextOffset.add(superPod.fieldsSizeWithoutRefMap);
                 superRefMap = superPod.referenceMap;
+                nextOffset = nextOffset.add(superPod.arrayLength - superRefMap.length);
             }
             ReferenceMapEncoder refMapEncoder = new ReferenceMapEncoder(superRefMap);
             while (!fields.isEmpty()) {
@@ -209,7 +209,8 @@ public Pod<T> build() {
             }
 
             byte[] referenceMap = refMapEncoder.encode();
-            return new Pod<>(podInfo, UnsignedUtils.safeToInt(nextOffset), referenceMap);
+            int arrayLength = UnsignedUtils.safeToInt(nextOffset) + referenceMap.length;
+            return new Pod<>(podInfo, arrayLength, referenceMap);
         }
     }
 
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
index 0d41c7027029..b44a6bdabdad 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
@@ -24,13 +24,20 @@
  */
 package com.oracle.svm.core.heap;
 
+import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
+import org.graalvm.compiler.word.BarrieredAccess;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
 
+import com.oracle.svm.core.JavaMemoryUtil;
 import com.oracle.svm.core.annotate.AlwaysInline;
 import com.oracle.svm.core.config.ConfigurationValues;
+import com.oracle.svm.core.graal.nodes.NewPodInstanceNode;
+import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.util.UnsignedUtils;
 
 public final class PodReferenceMapDecoder {
     @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor")
@@ -39,14 +46,14 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco
         boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences();
 
         UnsignedWord refOffset = LayoutEncoding.getArrayBaseOffset(layoutEncoding);
-        UnsignedWord mapOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, ArrayLengthNode.arrayLength(obj) - 1);
+        UnsignedWord mapOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, ArrayLengthNode.arrayLength(obj));
 
         int nrefs;
         int gap;
         do {
-            nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset));
-            gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.subtract(1)));
             mapOffset = mapOffset.subtract(2);
+            gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset));
+            nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1)));
 
             for (int i = 0; i < nrefs; i++) {
                 if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) {
@@ -60,6 +67,68 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco
         return true;
     }
 
+    /**
+     * Implements the allocation and the copying of the reference map and data stored in the hybrid
+     * array for {@link Object#clone()}.
+     */
+    public static Object clone(Object original, DynamicHub hub, int layoutEncoding) {
+        Class<?> nonNullHub = GraalDirectives.guardingNonNull(DynamicHub.toClass(hub));
+        int length = ArrayLengthNode.arrayLength(original);
+        byte[] referenceMap = extractReferenceMap(original, layoutEncoding, length);
+        Object result = NewPodInstanceNode.newPodInstance(null, nonNullHub, length, referenceMap);
+        copyArray(original, result, layoutEncoding, length);
+        return result;
+    }
+
+    private static byte[] extractReferenceMap(Object obj, int layoutEncoding, int length) {
+        UnsignedWord mapEndOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, length);
+        UnsignedWord mapOffset = mapEndOffset;
+
+        int gap;
+        int nrefs;
+        do {
+            mapOffset = mapOffset.subtract(2);
+            gap = Byte.toUnsignedInt(BarrieredAccess.readByte(obj, mapOffset));
+            nrefs = Byte.toUnsignedInt(BarrieredAccess.readByte(obj, mapOffset.add(1)));
+        } while (gap != 0 || nrefs == 0xff);
+
+        int refMapLength = UnsignedUtils.safeToInt(mapEndOffset.subtract(mapOffset));
+        byte[] refMap = new byte[refMapLength];
+        for (int i = 0; i < refMapLength; i++) {
+            refMap[i] = BarrieredAccess.readByte(obj, mapOffset.add(i));
+        }
+        return refMap;
+    }
+
+    private static void copyArray(Object original, Object copy, int layoutEncoding, int length) {
+        int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
+
+        UnsignedWord refOffset = LayoutEncoding.getArrayBaseOffset(layoutEncoding);
+        UnsignedWord mapOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, length);
+
+        UnsignedWord gap;
+        UnsignedWord nrefs;
+        do {
+            mapOffset = mapOffset.subtract(2);
+            gap = WordFactory.unsigned(Byte.toUnsignedInt(BarrieredAccess.readByte(copy, mapOffset)));
+            nrefs = WordFactory.unsigned(Byte.toUnsignedInt(BarrieredAccess.readByte(copy, mapOffset.add(1))));
+
+            // Copy references separately with the required barriers
+            UnsignedWord refBytes = nrefs.multiply(referenceSize);
+            JavaMemoryUtil.copyReferencesForward(original, refOffset, copy, refOffset, refBytes);
+
+            // Copy primitives in between
+            UnsignedWord primOffset = refOffset.add(refBytes);
+            UnsignedWord primBytes = gap.multiply(referenceSize);
+            JavaMemoryUtil.copyForward(original, primOffset, copy, primOffset, primBytes);
+
+            refOffset = primOffset.add(primBytes);
+        } while (gap.notEqual(0) || nrefs.equal(0xff));
+
+        // The loop above could be optimized for very long sequences of references or primitive
+        // values encoded in multiple reference map entries, but those should be rare in practice.
+    }
+
     private PodReferenceMapDecoder() {
     }
 }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
index 6b3be2ce9cd4..12e96822c9a4 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
@@ -147,10 +147,10 @@ private static void instantiatePod(HostedGraphKit kit, HostedProviders providers
         ResolvedJavaType podType = kit.getMetaAccess().lookupJavaType(Pod.class);
         ValueNode receiver = kit.loadLocal(0, JavaKind.Object);
         ValueNode pod = loadNonNullField(kit, receiver, findField(factoryType, "pod"));
-        ValueNode sizeWithoutRefMap = kit.createLoadField(pod, findField(podType, "fieldsSizeWithoutRefMap"));
+        ValueNode arrayLength = kit.createLoadField(pod, findField(podType, "arrayLength"));
         ValueNode refMap = loadNonNullField(kit, pod, findField(podType, "referenceMap"));
         ConstantNode hub = kit.createConstant(providers.getConstantReflection().asObjectHub(podConcreteType), JavaKind.Object);
-        ValueNode instance = kit.append(new NewPodInstanceNode(podConcreteType, hub, sizeWithoutRefMap, refMap));
+        ValueNode instance = kit.append(new NewPodInstanceNode(podConcreteType, hub, arrayLength, refMap));
         kit.storeLocal(instanceLocal, JavaKind.Object, instance);
     }
 

From f6d9c6867739133cfaa46896e4ab09e4b4a5e0f8 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Thu, 5 May 2022 18:34:11 +0200
Subject: [PATCH 09/12] Add node source positions to pod factory methods for
 inlining in runtime compilations.

---
 .../core/graal/replacements/SubstrateGraphKit.java    |  6 +++---
 .../svm/hosted/heap/PodFactorySubstitutionMethod.java | 11 +++++++++--
 .../com/oracle/svm/hosted/phases/HostedGraphKit.java  |  6 +++++-
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java
index 3a10f6dc0af6..4b0747898c5f 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/replacements/SubstrateGraphKit.java
@@ -96,9 +96,9 @@ public class SubstrateGraphKit extends GraphKit {
     private final FrameStateBuilder frameState;
     private int nextBCI;
 
-    public SubstrateGraphKit(DebugContext debug, ResolvedJavaMethod stubMethod, Providers providers, WordTypes wordTypes, GraphBuilderConfiguration.Plugins graphBuilderPlugins,
-                    CompilationIdentifier compilationId) {
-        super(debug, stubMethod, providers, wordTypes, graphBuilderPlugins, compilationId, null, SubstrateOptions.parseOnce(), false);
+    public SubstrateGraphKit(DebugContext debug, ResolvedJavaMethod stubMethod, Providers providers, WordTypes wordTypes,
+                    GraphBuilderConfiguration.Plugins graphBuilderPlugins, CompilationIdentifier compilationId, boolean forceTrackNodeSourcePosition) {
+        super(debug, stubMethod, providers, wordTypes, graphBuilderPlugins, compilationId, null, forceTrackNodeSourcePosition || SubstrateOptions.parseOnce(), false);
         assert wordTypes != null : "Support for Word types is mandatory";
         frameState = new FrameStateBuilder(this, stubMethod, graph);
         frameState.disableKindVerification();
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
index 12e96822c9a4..2d0d13acbadf 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
@@ -32,6 +32,7 @@
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
 import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.FixedNode;
@@ -103,7 +104,11 @@ public int getModifiers() {
 
     @Override
     public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method, HostedProviders providers, Purpose purpose) {
-        HostedGraphKit kit = new HostedGraphKit(debug, providers, method);
+        // Needed to match type flows to invokes so invoked methods can be inlined in runtime
+        // compilations, see GraalFeature.processMethod() and MethodTypeFlowBuilder.uniqueKey()
+        boolean trackNodeSourcePosition = (purpose == Purpose.ANALYSIS);
+
+        HostedGraphKit kit = new HostedGraphKit(debug, providers, method, trackNodeSourcePosition);
         boolean isDeoptTarget = (method instanceof SharedMethod) && ((SharedMethod) method).isDeoptTarget();
 
         ResolvedJavaType factoryType = method.getDeclaringClass();
@@ -168,7 +173,9 @@ private static int invokeConstructor(HostedGraphKit kit, ResolvedJavaMethod meth
 
     /** @see com.oracle.svm.hosted.phases.HostedGraphBuilderPhase */
     private static int invokeWithDeoptAndExceptionUnwind(HostedGraphKit kit, boolean isDeoptTarget, int initialNextDeoptIndex, ResolvedJavaMethod target, InvokeKind invokeKind, ValueNode... args) {
-        InvokeWithExceptionNode invoke = kit.startInvokeWithException(target, invokeKind, kit.getFrameState(), kit.bci(), args);
+        int bci = kit.bci();
+        InvokeWithExceptionNode invoke = kit.startInvokeWithException(target, invokeKind, kit.getFrameState(), bci, args);
+        invoke.setNodeSourcePosition(NodeSourcePosition.placeholder(kit.getGraph().method(), bci));
         kit.exceptionPart();
         ExceptionObjectNode exception = kit.exceptionObject();
 
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java
index 7d0a227d21b6..daf7f50aaa9a 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/phases/HostedGraphKit.java
@@ -70,7 +70,11 @@
 public class HostedGraphKit extends SubstrateGraphKit {
 
     public HostedGraphKit(DebugContext debug, HostedProviders providers, ResolvedJavaMethod method) {
-        super(debug, method, providers, providers.getWordTypes(), providers.getGraphBuilderPlugins(), new SubstrateCompilationIdentifier());
+        this(debug, providers, method, false);
+    }
+
+    public HostedGraphKit(DebugContext debug, HostedProviders providers, ResolvedJavaMethod method, boolean forceTrackNodeSourcePosition) {
+        super(debug, method, providers, providers.getWordTypes(), providers.getGraphBuilderPlugins(), new SubstrateCompilationIdentifier(), forceTrackNodeSourcePosition);
     }
 
     @Override

From 8735a9d5078991b81018eac3a0dc195ef6e6ce9e Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Fri, 6 May 2022 13:16:41 +0200
Subject: [PATCH 10/12] Distinguish arrays and hybrids in layout encoding.

---
 .../genscavenge/GreyToBlackObjectVisitor.java |   2 +-
 .../genscavenge/ThreadLocalAllocation.java    |   2 +-
 .../graal/GenScavengeAllocationSnippets.java  |   2 +-
 .../src/com/oracle/svm/core/DebugHelper.java  |   2 +-
 .../com/oracle/svm/core/annotate/Hybrid.java  |   7 +-
 .../oracle/svm/core/code/CodeInfoEncoder.java |   2 +-
 .../oracle/svm/core/deopt/Deoptimizer.java    |   1 +
 .../jdk/SubstrateObjectCloneSnippets.java     |  15 ++-
 .../snippets/SubstrateAllocationSnippets.java |   8 +-
 .../svm/core/hub/InteriorObjRefWalker.java    |   2 +-
 .../oracle/svm/core/hub/LayoutEncoding.java   | 106 ++++++++++++------
 .../svm/hosted/NativeImageGenerator.java      |  15 ++-
 .../svm/hosted/image/NativeImageHeap.java     |   6 +-
 .../svm/hosted/meta/UniverseBuilder.java      |   4 +-
 14 files changed, 106 insertions(+), 68 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java
index 7ece8ff6a1cc..bd3ab89d9f3d 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java
@@ -166,7 +166,7 @@ public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLev
                         DynamicHub hub = (DynamicHub) headerHub.toObject();
                         log.string("  class: ").string(hub.getName());
                         Object entryAsObject = objectEntry.toObject();
-                        if (LayoutEncoding.isArray(entryAsObject)) {
+                        if (LayoutEncoding.isArrayLike(entryAsObject)) {
                             int length = ArrayLengthNode.arrayLength(entryAsObject);
                             log.string("  length: ").signed(length);
                         }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
index 66020af76250..92aded8b5cce 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
@@ -294,7 +294,7 @@ private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int leng
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
     private static Object allocateInstanceInNewTlab(DynamicHub hub, AlignedHeader newTlabChunk) {
-        UnsignedWord size = LayoutEncoding.getInstanceSize(hub.getLayoutEncoding());
+        UnsignedWord size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding());
         Pointer memory = allocateRawMemoryInNewTlab(size, newTlabChunk);
         return FormatObjectNode.formatObject(memory, DynamicHub.toClass(hub), false, FillContent.WITH_ZEROES, true);
     }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
index 278b6f8bbed8..3ec16b198e06 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
@@ -102,7 +102,7 @@ public Object formatObjectSnippet(Word memory, DynamicHub hub, boolean remembere
                     @ConstantParameter AllocationSnippetCounters snippetCounters) {
         DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
         int layoutEncoding = hubNonNull.getLayoutEncoding();
-        UnsignedWord size = LayoutEncoding.getInstanceSize(layoutEncoding);
+        UnsignedWord size = LayoutEncoding.getPureInstanceSize(layoutEncoding);
         Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, false);
         return formatObject(objectHeader, size, memory, fillContents, emitMemoryBarrier, false, snippetCounters);
     }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/DebugHelper.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/DebugHelper.java
index 09500eccd1e0..db20bc6b2feb 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/DebugHelper.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/DebugHelper.java
@@ -315,7 +315,7 @@ private static boolean isObjectArray(DynamicHub hub) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static boolean isInstance(DynamicHub hub) {
-        return LayoutEncoding.isInstance(hub.getLayoutEncoding());
+        return LayoutEncoding.isPureInstance(hub.getLayoutEncoding());
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
index 96a01de95af8..a9c95eb679df 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
@@ -59,9 +59,10 @@
  *
  * <p>
  * Hybrid objects have {@link HubType#Instance} but a {@link LayoutEncoding} like an array. This is
- * important to keep in mind because methods such as {@link Class#isArray()} will return
- * {@code false}, while methods such as {@link LayoutEncoding#isArray} will return {@code true} for
- * hybrid objects.
+ * important to keep in mind because methods such as {@link Class#isInstance} will return
+ * {@code true} and {@link Class#isArray()} will return {@code false}, while
+ * {@link LayoutEncoding#isPureInstance} will return {@code false} and
+ * {@link LayoutEncoding#isArrayLike} will return {@code true} and for hybrid objects.
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java
index fb8c12501536..fe74699b7ced 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoEncoder.java
@@ -646,7 +646,7 @@ private static ValueInfo findActualArrayElement(ValueInfo[] actualObject, Unsign
     private static ValueInfo findActualField(ValueInfo[] actualObject, UnsignedWord expectedOffset) {
         DynamicHub hub = (DynamicHub) SubstrateObjectConstant.asObject(actualObject[0].getValue());
         ObjectLayout objectLayout = ConfigurationValues.getObjectLayout();
-        assert LayoutEncoding.isInstance(hub.getLayoutEncoding());
+        assert LayoutEncoding.isPureInstance(hub.getLayoutEncoding());
         return findActualValue(actualObject, expectedOffset, objectLayout, WordFactory.unsigned(objectLayout.getFirstFieldOffset()), 1);
     }
 
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java
index 1488987566f9..02065dde0fc7 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/deopt/Deoptimizer.java
@@ -990,6 +990,7 @@ private Object materializeObject(int virtualObjectId, FrameInfoQueryResult sourc
             curOffset = LayoutEncoding.getArrayBaseOffset(hub.getLayoutEncoding());
             curIdx = 2;
         } else {
+            assert LayoutEncoding.isPureInstance(hub.getLayoutEncoding());
             try {
                 obj = Unsafe.getUnsafe().allocateInstance(DynamicHub.toClass(hub));
             } catch (InstantiationException ex) {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
index 81c04217e767..00857633172e 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
@@ -96,12 +96,11 @@ private static Object doClone(Object original) throws CloneNotSupportedException
 
         DynamicHub hub = KnownIntrinsics.readHub(original);
         int layoutEncoding = hub.getLayoutEncoding();
-        boolean hasArray = LayoutEncoding.isArray(layoutEncoding);
+        boolean isArrayLike = LayoutEncoding.isArrayLike(layoutEncoding);
 
         Object result;
-        if (hasArray) {
-            // Hybrids like pods have an array encoding, but a non-array hub type.
-            if (BranchProbabilityNode.probability(FAST_PATH_PROBABILITY, hub.isArray())) {
+        if (isArrayLike) {
+            if (BranchProbabilityNode.probability(FAST_PATH_PROBABILITY, LayoutEncoding.isArray(layoutEncoding))) {
                 int length = ArrayLengthNode.arrayLength(original);
                 Object newArray = java.lang.reflect.Array.newInstance(DynamicHub.toClass(hub.getComponentHub()), length);
                 if (LayoutEncoding.isObjectArray(layoutEncoding)) {
@@ -153,8 +152,8 @@ private static Object doClone(Object original) throws CloneNotSupportedException
         }
 
         // copy remaining non-object data
-        if (!hasArray) {
-            int objectSize = UnsignedUtils.safeToInt(LayoutEncoding.getInstanceSize(layoutEncoding));
+        if (!isArrayLike) {
+            int objectSize = UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceSize(layoutEncoding));
             int primitiveDataSize = objectSize - curOffset;
             assert primitiveDataSize >= 0;
             assert curOffset >= 0;
@@ -182,9 +181,9 @@ static boolean canVirtualize(ObjectClone node, VirtualizerTool tool) {
             return false;
         }
         if (!type.isArray() && type instanceof SharedType) {
-            // Hybrids are instances with array encoding; cloning them virtually is not implemented.
+            // Hybrids are instances with array-like encoding; cloning virtually is unimplemented.
             int encoding = ((SharedType) type).getHub().getLayoutEncoding();
-            return !LayoutEncoding.isArray(encoding);
+            return !LayoutEncoding.isArrayLike(encoding);
         }
         return true;
     }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
index 0e18f8343256..890b376a6f18 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
@@ -140,7 +140,7 @@ public Object allocateArray(@NonNullParameter DynamicHub hub,
     public Object allocateInstanceDynamic(@NonNullParameter DynamicHub hub, @ConstantParameter FillContent fillContents, @ConstantParameter boolean emitMemoryBarrier,
                     @ConstantParameter AllocationProfilingData profilingData) {
         // The hub was already verified by a ValidateNewInstanceClassNode.
-        UnsignedWord size = LayoutEncoding.getInstanceSize(hub.getLayoutEncoding());
+        UnsignedWord size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding());
         Object result = allocateInstanceImpl(encodeAsTLABObjectHeader(hub), size, fillContents, emitMemoryBarrier, false, profilingData);
         return piCastToSnippetReplaceeStamp(result);
     }
@@ -168,7 +168,7 @@ protected Object newmultiarray(DynamicHub hub, @ConstantParameter int rank, @Var
     private static DynamicHub validateNewInstanceClass(DynamicHub hub) {
         if (probability(EXTREMELY_FAST_PATH_PROBABILITY, hub != null)) {
             DynamicHub nonNullHub = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
-            if (probability(EXTREMELY_FAST_PATH_PROBABILITY, LayoutEncoding.isInstance(nonNullHub.getLayoutEncoding())) &&
+            if (probability(EXTREMELY_FAST_PATH_PROBABILITY, LayoutEncoding.isPureInstance(nonNullHub.getLayoutEncoding())) &&
                             probability(EXTREMELY_FAST_PATH_PROBABILITY, nonNullHub.isInstantiated())) {
                 return nonNullHub;
             }
@@ -212,7 +212,7 @@ private static Object newMultiArrayRecursion(DynamicHub hub, int rank, Word dime
     private static void instanceHubErrorStub(DynamicHub hub) throws InstantiationException {
         if (hub == null) {
             throw new NullPointerException("Allocation type is null.");
-        } else if (!LayoutEncoding.isInstance(hub.getLayoutEncoding())) {
+        } else if (!LayoutEncoding.isPureInstance(hub.getLayoutEncoding())) {
             throw new InstantiationException("Cannot allocate instance.");
         } else if (!hub.isInstantiated()) {
             throw new IllegalArgumentException("Type " + DynamicHub.toClass(hub).getTypeName() + " is instantiated reflectively but was never registered." +
@@ -443,7 +443,7 @@ public void lower(NewInstanceNode node, LoweringTool tool) {
                 DynamicHub hub = ensureMarkedAsInstantiated(type.getHub());
 
                 ConstantNode hubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(hub), providers.getMetaAccess(), graph);
-                long size = LayoutEncoding.getInstanceSize(hub.getLayoutEncoding()).rawValue();
+                long size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()).rawValue();
 
                 Arguments args = new Arguments(allocateInstance, graph.getGuardsStage(), tool.getLoweringStage());
                 args.add("hub", hubConstant);
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
index 632168282fb5..3979d35b6845 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
@@ -66,7 +66,7 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi
         final Pointer objPointer = Word.objectToUntrackedPointer(obj);
 
         // Visit each Object reference in the array part of the Object.
-        if (LayoutEncoding.isObjectArray(layoutEncoding)) {
+        if (LayoutEncoding.isArrayLikeWithObjectElements(layoutEncoding)) {
             int length = ArrayLengthNode.arrayLength(obj);
             int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
             boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences();
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
index 78bfaf239f69..81a2b263abc5 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
@@ -55,16 +55,16 @@
  * <li>Array objects: the layout encoding for arrays is a negative number with the following
  * format:<br>
  *
- * <code>[tag:2, free:10, base:12, indexShift:8]</code>
+ * <code>[tag:3, unused:9, base:12, indexShift:8]</code>
  * <ul>
- * <li>tag: 0x80 if the array is an object array, 0xC0 if it is a primitive array</li>
- * <li>free: currently unused bits</li>
+ * <li>tag: determines element type and whether the object is a true array or a {@link Hybrid}.</li>
+ * <li>unused: bits currently not used for any purpose</li>
  * <li>base: the array base offset</li>
  * <li>indexShift: the array index shift for accessing array elements or for computing the array
  * size based on the array length</li>
  * </ul>
  *
- * {@link Hybrid} objects are also encoded as arrays but are treated like instance objects in other
+ * {@link Hybrid} objects are encoded like arrays but are treated like instance objects in other
  * places (e.g. {@link HubType}). Another difference to arrays is that hybrid objects need a
  * reference map because they have fields.</li>
  * </ul>
@@ -82,10 +82,12 @@ public class LayoutEncoding {
     private static final int ARRAY_INDEX_SHIFT_MASK = 0xff;
     private static final int ARRAY_BASE_SHIFT = 8 + ARRAY_INDEX_SHIFT_SHIFT;
     private static final int ARRAY_BASE_MASK = 0xfff;
-    private static final int ARRAY_TAG_BITS = 2;
+    private static final int ARRAY_TAG_BITS = 3;
     private static final int ARRAY_TAG_SHIFT = Integer.SIZE - ARRAY_TAG_BITS;
-    private static final int ARRAY_TAG_PRIMITIVE_VALUE = ~0x00;
-    private static final int ARRAY_TAG_OBJECT_VALUE = ~0x01;
+    private static final int ARRAY_TAG_PRIMITIVE_VALUE = 0b111;
+    private static final int ARRAY_TAG_HYBRID_PRIMITIVE_VALUE = 0b110;
+    private static final int ARRAY_TAG_HYBRID_OBJECT_VALUE = 0b101;
+    private static final int ARRAY_TAG_OBJECT_VALUE = 0b100;
 
     public static int forPrimitive() {
         return PRIMITIVE_VALUE;
@@ -100,9 +102,10 @@ public static int forAbstract() {
     }
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    private static void guaranteeEncoding(ResolvedJavaType type, boolean condition, String description) {
-        if (!condition) {
-            VMError.shouldNotReachHere(description + ". This error is caused by an incorrect compact encoding of a type " +
+    private static void guaranteeEncoding(ResolvedJavaType type, boolean expected, boolean actual, String description) {
+        if (actual != expected) {
+            throw VMError.shouldNotReachHere(description + ": expected to be " + expected + ". " +
+                            "This error is caused by an incorrect compact encoding of a type " +
                             "(a class, array or a primitive). The error occurred with the following type, but also could be caused " +
                             "by characteristics of the overall type hierarchy: " + type + ". Please report this problem and the " +
                             "conditions in which it occurs and include any noteworthy characteristics of the type hierarchy and " +
@@ -111,30 +114,45 @@ private static void guaranteeEncoding(ResolvedJavaType type, boolean condition,
     }
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public static int forInstance(ResolvedJavaType type, int size) {
-        guaranteeEncoding(type, size > LAST_SPECIAL_VALUE, "Instance type size must be above special values for encoding: " + size);
+    public static int forPureInstance(ResolvedJavaType type, int size) {
+        guaranteeEncoding(type, true, size > LAST_SPECIAL_VALUE, "Instance type size must be above special values for encoding: " + size);
         int encoding = size;
-        guaranteeEncoding(type, isInstance(encoding), "Instance type encoding must denote an instance");
-        guaranteeEncoding(type, !isArray(encoding), "Instance type encoding must not denote an array");
-        guaranteeEncoding(type, !isObjectArray(encoding), "Instance type encoding must not denote an object array");
-        guaranteeEncoding(type, !isPrimitiveArray(encoding), "Instance type encoding must not denote a primitive array");
-        guaranteeEncoding(type, getInstanceSize(encoding).equal(WordFactory.unsigned(size)), "Instance type encoding size must match type size");
+        guaranteeEncoding(type, true, isPureInstance(encoding), "Instance type encoding denotes an instance");
+        guaranteeEncoding(type, false, isArray(encoding) || isArrayLike(encoding), "Instance type encoding denotes an array-like object");
+        guaranteeEncoding(type, false, isObjectArray(encoding) || isArrayLikeWithObjectElements(encoding), "Instance type encoding denotes an object array");
+        guaranteeEncoding(type, false, isPrimitiveArray(encoding) || isArrayLikeWithPrimitiveElements(encoding), "Instance type encoding denotes a primitive array");
+        guaranteeEncoding(type, true, getPureInstanceSize(encoding).equal(WordFactory.unsigned(size)), "Instance type encoding size matches type size");
         return encoding;
     }
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public static int forArray(ResolvedJavaType type, boolean isObject, int arrayBaseOffset, int arrayIndexShift) {
-        int tag = isObject ? ARRAY_TAG_OBJECT_VALUE : ARRAY_TAG_PRIMITIVE_VALUE;
+    public static int forArray(ResolvedJavaType type, boolean objectElements, int arrayBaseOffset, int arrayIndexShift) {
+        return forArrayLike(type, false, objectElements, arrayBaseOffset, arrayIndexShift);
+    }
+
+    @Platforms(Platform.HOSTED_ONLY.class)
+    public static int forHybrid(ResolvedJavaType type, boolean objectElements, int arrayBaseOffset, int arrayIndexShift) {
+        return forArrayLike(type, true, objectElements, arrayBaseOffset, arrayIndexShift);
+    }
+
+    @Platforms(Platform.HOSTED_ONLY.class)
+    private static int forArrayLike(ResolvedJavaType type, boolean isHybrid, boolean objectElements, int arrayBaseOffset, int arrayIndexShift) {
+        assert isHybrid != type.isArray();
+        int tag = isHybrid ? (objectElements ? ARRAY_TAG_HYBRID_OBJECT_VALUE : ARRAY_TAG_HYBRID_PRIMITIVE_VALUE)
+                        : (objectElements ? ARRAY_TAG_OBJECT_VALUE : ARRAY_TAG_PRIMITIVE_VALUE);
         int encoding = (tag << ARRAY_TAG_SHIFT) | (arrayBaseOffset << ARRAY_BASE_SHIFT) | (arrayIndexShift << ARRAY_INDEX_SHIFT_SHIFT);
 
-        guaranteeEncoding(type, isArray(encoding), "Array encoding must denote an array");
-        guaranteeEncoding(type, !isInstance(encoding), "Array encoding must not denote an instance type");
-        guaranteeEncoding(type, isObjectArray(encoding) == isObject, "Expected isObjectArray(encoding) == " + isObject);
-        guaranteeEncoding(type, isPrimitiveArray(encoding) != isObject, "Expected isPrimitiveArray(encoding) != " + isObject);
-        guaranteeEncoding(type, getArrayBaseOffset(encoding).equal(WordFactory.unsigned(arrayBaseOffset)),
-                        "Expected array base offset of " + arrayBaseOffset + ", but encoding gives " + getArrayBaseOffset(encoding));
-        guaranteeEncoding(type, getArrayIndexShift(encoding) == arrayIndexShift,
-                        "Expected array index shift of " + arrayIndexShift + ", but encoding gives " + getArrayIndexShift(encoding));
+        guaranteeEncoding(type, true, isArrayLike(encoding), "Array-like object encoding denotes an array-like object");
+        guaranteeEncoding(type, !isHybrid, isArray(encoding), "Encoding denotes an array");
+        guaranteeEncoding(type, false, isPureInstance(encoding), "Array-like object encoding denotes an instance type");
+        guaranteeEncoding(type, objectElements, isArrayLikeWithObjectElements(encoding), "Encoding denotes an array-like object with object elements");
+        guaranteeEncoding(type, !objectElements, isArrayLikeWithPrimitiveElements(encoding), "Encoding denotes an array-like object with primitive elements");
+        guaranteeEncoding(type, !isHybrid && objectElements, isObjectArray(encoding), "Encoding denotes an object array");
+        guaranteeEncoding(type, !isHybrid && !objectElements, isPrimitiveArray(encoding), "Encoding denotes a primitive array");
+        guaranteeEncoding(type, true, getArrayBaseOffset(encoding).equal(WordFactory.unsigned(arrayBaseOffset)),
+                        "Encoding denotes a base offset of " + arrayBaseOffset + " (actual value: " + getArrayBaseOffset(encoding) + ')');
+        guaranteeEncoding(type, true, getArrayIndexShift(encoding) == arrayIndexShift,
+                        "Encoding denotes an index shift of " + arrayIndexShift + " (actual value: " + getArrayIndexShift(encoding) + ')');
         return encoding;
     }
 
@@ -150,19 +168,27 @@ public static boolean isAbstract(int encoding) {
         return encoding == ABSTRACT_VALUE;
     }
 
+    /** Tests if an encoding denotes a pure instance object, i.e. not a hybrid object or array. */
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static boolean isInstance(int encoding) {
+    public static boolean isPureInstance(int encoding) {
         return encoding > LAST_SPECIAL_VALUE;
     }
 
+    /** Determines the size of a pure instance object (i.e. not a hybrid object or array). */
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static UnsignedWord getInstanceSize(int encoding) {
+    public static UnsignedWord getPureInstanceSize(int encoding) {
         return WordFactory.unsigned(encoding);
     }
 
     // May be inlined because it does not deal in Pointers.
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArray(int encoding) {
+        return isPrimitiveArray(encoding) || isObjectArray(encoding);
+    }
+
+    // May be inlined because it does not deal in Pointers.
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean isArrayLike(int encoding) {
         return encoding < NEUTRAL_VALUE;
     }
 
@@ -173,7 +199,19 @@ public static boolean isPrimitiveArray(int encoding) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isObjectArray(int encoding) {
-        return encoding < (ARRAY_TAG_PRIMITIVE_VALUE << ARRAY_TAG_SHIFT);
+        return encoding <= ~(~ARRAY_TAG_OBJECT_VALUE << ARRAY_TAG_SHIFT);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean isArrayLikeWithPrimitiveElements(int encoding) {
+        int tag = (encoding >>> ARRAY_TAG_SHIFT);
+        return (tag == ARRAY_TAG_PRIMITIVE_VALUE || tag == ARRAY_TAG_HYBRID_PRIMITIVE_VALUE);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean isArrayLikeWithObjectElements(int encoding) {
+        int tag = (encoding >>> ARRAY_TAG_SHIFT);
+        return (tag == ARRAY_TAG_OBJECT_VALUE || tag == ARRAY_TAG_HYBRID_OBJECT_VALUE);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
@@ -217,10 +255,10 @@ public static UnsignedWord getSizeFromObject(Object obj) {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static UnsignedWord getSizeFromObjectInline(Object obj) {
         int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding();
-        if (isArray(encoding)) {
+        if (isArrayLike(encoding)) {
             return getArraySize(encoding, ArrayLengthNode.arrayLength(obj));
         } else {
-            return getInstanceSize(encoding);
+            return getPureInstanceSize(encoding);
         }
     }
 
@@ -241,9 +279,9 @@ public static boolean isArray(Object obj) {
         return isArray(encoding);
     }
 
-    public static boolean isInstance(Object obj) {
+    public static boolean isArrayLike(Object obj) {
         final int encoding = KnownIntrinsics.readHub(obj).getLayoutEncoding();
-        return isInstance(encoding);
+        return isArrayLike(encoding);
     }
 
     @Fold
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
index f33fd3c0cd08..f588c19ec399 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
@@ -1668,14 +1668,13 @@ private void printTypes() {
                 System.out.print("interface  ");
             } else if (LayoutEncoding.isAbstract(le)) {
                 System.out.print("abstract  ");
-            } else if (LayoutEncoding.isInstance(le)) {
-                System.out.format("instance size %d  ", LayoutEncoding.getInstanceSize(le).rawValue());
-            } else if (LayoutEncoding.isObjectArray(le)) {
-                System.out.format("object array base %d shift %d scale %d  ", LayoutEncoding.getArrayBaseOffset(le).rawValue(), LayoutEncoding.getArrayIndexShift(le),
-                                LayoutEncoding.getArrayIndexScale(le));
-            } else if (LayoutEncoding.isPrimitiveArray(le)) {
-                System.out.format("primitive array base %d shift %d scale %d  ", LayoutEncoding.getArrayBaseOffset(le).rawValue(), LayoutEncoding.getArrayIndexShift(le),
-                                LayoutEncoding.getArrayIndexScale(le));
+            } else if (LayoutEncoding.isPureInstance(le)) {
+                System.out.format("instance size %d  ", LayoutEncoding.getPureInstanceSize(le).rawValue());
+            } else if (LayoutEncoding.isArrayLike(le)) {
+                String arrayType = LayoutEncoding.isArray(le) ? "array" : "array-like object";
+                String elements = LayoutEncoding.isArrayLikeWithPrimitiveElements(le) ? "primitives" : "objects";
+                System.out.format("%s containing %s, base %d shift %d scale %d  ", arrayType, elements, LayoutEncoding.getArrayBaseOffset(le).rawValue(),
+                                LayoutEncoding.getArrayIndexShift(le), LayoutEncoding.getArrayIndexScale(le));
             } else {
                 throw VMError.shouldNotReachHere();
             }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java
index 054e701d5fd5..64342c48756a 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImageHeap.java
@@ -332,7 +332,7 @@ private boolean assertFillerObjectSizes() {
         assert minArraySize == objectLayout.getArraySize(JavaKind.Int, 0);
 
         HostedType filler = metaAccess.lookupJavaType(FillerObject.class);
-        UnsignedWord fillerSize = LayoutEncoding.getInstanceSize(filler.getHub().getLayoutEncoding());
+        UnsignedWord fillerSize = LayoutEncoding.getPureInstanceSize(filler.getHub().getLayoutEncoding());
         assert fillerSize.equal(minInstanceSize);
 
         assert minInstanceSize * 2 >= minArraySize : "otherwise, we might need more than one non-array object";
@@ -421,7 +421,7 @@ private void addObjectToImageHeap(final Object object, boolean immutableFromPare
                 assert hybridArray != null : "Cannot read value for field " + hybridArrayField.format("%H.%n");
                 size = hybridLayout.getTotalSize(Array.getLength(hybridArray));
             } else {
-                size = LayoutEncoding.getInstanceSize(hub.getLayoutEncoding()).rawValue();
+                size = LayoutEncoding.getPureInstanceSize(hub.getLayoutEncoding()).rawValue();
             }
 
             info = addToImageHeap(object, clazz, size, identityHashCode, reason);
@@ -561,7 +561,7 @@ private long getSize(Object object, HostedType type) {
         if (type.isInstanceClass()) {
             HostedInstanceClass clazz = (HostedInstanceClass) type;
             assert !HybridLayout.isHybrid(clazz);
-            return LayoutEncoding.getInstanceSize(clazz.getHub().getLayoutEncoding()).rawValue();
+            return LayoutEncoding.getPureInstanceSize(clazz.getHub().getLayoutEncoding()).rawValue();
         } else if (type.isArray()) {
             return objectLayout.getArraySize(type.getComponentType().getStorageKind(), Array.getLength(object));
         } else {
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
index f7d6e13ffd41..6df4208330c7 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
@@ -902,9 +902,9 @@ private void buildHubs() {
                     HybridLayout<?> hybridLayout = new HybridLayout<>(instanceClass, ol, hMetaAccess);
                     JavaKind storageKind = hybridLayout.getArrayElementStorageKind();
                     boolean isObject = (storageKind == JavaKind.Object);
-                    layoutHelper = LayoutEncoding.forArray(type, isObject, hybridLayout.getArrayBaseOffset(), ol.getArrayIndexShift(storageKind));
+                    layoutHelper = LayoutEncoding.forHybrid(type, isObject, hybridLayout.getArrayBaseOffset(), ol.getArrayIndexShift(storageKind));
                 } else {
-                    layoutHelper = LayoutEncoding.forInstance(type, ConfigurationValues.getObjectLayout().alignUp(instanceClass.getInstanceSize()));
+                    layoutHelper = LayoutEncoding.forPureInstance(type, ConfigurationValues.getObjectLayout().alignUp(instanceClass.getInstanceSize()));
                 }
                 monitorOffset = instanceClass.getMonitorFieldOffset();
             } else if (type.isArray()) {

From 5f7626d848b413b2ebd60c50e8b88cd57bd53257 Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Mon, 9 May 2022 12:09:48 +0200
Subject: [PATCH 11/12] Make pod support conditional on Serial GC (including
 Epsilon GC).

---
 .../jdk/SubstrateObjectCloneSnippets.java     |  3 ++-
 .../src/com/oracle/svm/core/heap/Pod.java     | 16 +++++++++++++++-
 .../svm/core/hub/InteriorObjRefWalker.java    |  3 ++-
 .../src/com/oracle/svm/hosted/SVMHost.java    |  2 +-
 .../oracle/svm/hosted/heap/PodSupport.java    | 19 ++++++++++++++-----
 .../svm/hosted/meta/UniverseBuilder.java      | 17 +++++++++++++----
 6 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
index 00857633172e..e9aadeb6a1dd 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
@@ -62,6 +62,7 @@
 import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
 import com.oracle.svm.core.graal.snippets.SubstrateTemplates;
 import com.oracle.svm.core.heap.InstanceReferenceMapEncoder;
+import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.PodReferenceMapDecoder;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.hub.DynamicHubSupport;
@@ -110,7 +111,7 @@ private static Object doClone(Object original) throws CloneNotSupportedException
                 }
                 return newArray;
 
-            } else if (hub.isPodInstanceClass()) {
+            } else if (Pod.RuntimeSupport.isPresent() && hub.isPodInstanceClass()) {
                 result = PodReferenceMapDecoder.clone(original, hub, layoutEncoding);
 
             } else {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index fa2778cd3ced..d232c1c33e5c 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -38,6 +38,7 @@
 import java.util.function.Supplier;
 
 import org.graalvm.collections.EconomicMap;
+import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -124,11 +125,14 @@ public static <T> Builder<T> createExtending(Class<?> superClass, Class<T> facto
 
         private Builder(Class<?> superClass, Class<?> factoryInterface, Pod<T> superPod) {
             assert superPod == null || (superClass == null && factoryInterface == null);
+            if (!RuntimeSupport.isPresent()) {
+                throw new UnsupportedOperationException("Pods are not available in this native image. Only SerialGC currently supports pods.");
+            }
             if (superPod != null) {
                 this.podInfo = superPod.podInfo;
             } else if (superClass != null && factoryInterface != null) {
                 RuntimeSupport.PodSpec spec = new RuntimeSupport.PodSpec(superClass, factoryInterface);
-                this.podInfo = ImageSingletons.lookup(RuntimeSupport.class).getInfo(spec);
+                this.podInfo = RuntimeSupport.singleton().getInfo(spec);
                 if (this.podInfo == null) {
                     throw new IllegalArgumentException("Pod superclass/factory interface pair was not registered during image build: " + superClass + ", " + factoryInterface);
                 }
@@ -256,6 +260,16 @@ public int compareTo(Field f) {
     }
 
     public static final class RuntimeSupport {
+        @Fold
+        public static boolean isPresent() {
+            return ImageSingletons.contains(RuntimeSupport.class);
+        }
+
+        @Fold
+        public static RuntimeSupport singleton() {
+            return ImageSingletons.lookup(RuntimeSupport.class);
+        }
+
         @Retention(RetentionPolicy.RUNTIME)
         @Target(ElementType.TYPE)
         public @interface PodFactory {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
index 3979d35b6845..0d4ae77e5d2f 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java
@@ -36,6 +36,7 @@
 import com.oracle.svm.core.heap.InstanceReferenceMapDecoder;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
+import com.oracle.svm.core.heap.Pod;
 import com.oracle.svm.core.heap.PodReferenceMapDecoder;
 import com.oracle.svm.core.heap.ReferenceAccess;
 
@@ -80,7 +81,7 @@ public static boolean walkObjectInline(final Object obj, final ObjectReferenceVi
                 }
                 pos = pos.add(referenceSize);
             }
-        } else if (objHub.isPodInstanceClass()) {
+        } else if (Pod.RuntimeSupport.isPresent() && objHub.isPodInstanceClass()) {
             if (!PodReferenceMapDecoder.walkOffsetsFromPointer(objPointer, layoutEncoding, visitor, obj)) {
                 return false;
             }
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
index 773866199635..9bb78fba8d62 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/SVMHost.java
@@ -462,7 +462,7 @@ private static HubType computeHubType(AnalysisType type) {
         } else if (type.isInstanceClass()) {
             if (Reference.class.isAssignableFrom(type.getJavaClass())) {
                 return HubType.InstanceReference;
-            } else if (PodSupport.singleton().isPodClass(type.getJavaClass())) {
+            } else if (PodSupport.isPresent() && PodSupport.singleton().isPodClass(type.getJavaClass())) {
                 return HubType.PodInstance;
             }
             assert !Target_java_lang_ref_Reference.class.isAssignableFrom(type.getJavaClass()) : "should not see substitution type here";
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
index ff42cc993da9..2607b4db88b1 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
@@ -48,6 +48,7 @@
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.hosted.Feature;
 
+import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.annotate.AutomaticFeature;
 import com.oracle.svm.core.annotate.DeoptTest;
 import com.oracle.svm.core.annotate.Hybrid;
@@ -67,7 +68,14 @@
 
 /** Support for preparing the creation of {@link Pod} objects during the image build. */
 public interface PodSupport {
+    static boolean isPresent() {
+        return ImageSingletons.contains(PodSupport.class);
+    }
+
     static PodSupport singleton() {
+        if (!ImageSingletons.contains(PodSupport.class)) {
+            throw UserError.abort("Pods are not available in this native image build. Only SerialGC currently supports pods.");
+        }
         return ImageSingletons.lookup(PodSupport.class);
     }
 
@@ -107,6 +115,11 @@ final class PodFeature implements PodSupport, Feature {
     private volatile boolean instantiated = false;
     private boolean sealed = false;
 
+    @Override
+    public boolean isInConfiguration(IsInConfigurationAccess access) {
+        return SubstrateOptions.UseSerialGC.getValue();
+    }
+
     @Override
     public void afterRegistration(AfterRegistrationAccess access) {
         ImageSingletons.add(PodSupport.class, this);
@@ -133,11 +146,7 @@ private void registerAsInstantiated(DuringAnalysisAccess access) {
 
     private static void registerPodAsInstantiated(BeforeAnalysisAccess access, PodSpec spec, PodInfo info) {
         access.registerAsInHeap(info.podClass);
-        runtimeSupport().registerPod(spec, info);
-    }
-
-    private static Pod.RuntimeSupport runtimeSupport() {
-        return ImageSingletons.lookup(Pod.RuntimeSupport.class);
+        Pod.RuntimeSupport.singleton().registerPod(spec, info);
     }
 
     @Override
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
index 6df4208330c7..e789036cd657 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/meta/UniverseBuilder.java
@@ -401,6 +401,18 @@ private void layoutInstanceFields() {
         layoutInstanceFields(hUniverse.getObjectClass(), ConfigurationValues.getObjectLayout().getFirstFieldOffset(), new HostedField[0]);
     }
 
+    private static boolean mustReserveLengthField(HostedInstanceClass clazz) {
+        if (PodSupport.isPresent() && PodSupport.singleton().mustReserveLengthField(clazz.getJavaClass())) {
+            return true;
+        }
+        if (HybridLayout.isHybrid(clazz)) {
+            // A pod ancestor subclassing Object must have already reserved a length field, unless
+            // the pod subclasses Object itself, in which case we would have returned true earlier.
+            return !PodSupport.isPresent() || !PodSupport.singleton().isPodClass(clazz.getJavaClass());
+        }
+        return false;
+    }
+
     private void layoutInstanceFields(HostedInstanceClass clazz, int superSize, HostedField[] superFields) {
         ArrayList<HostedField> rawFields = new ArrayList<>();
         ArrayList<HostedField> orderedFields = new ArrayList<>();
@@ -414,10 +426,7 @@ private void layoutInstanceFields(HostedInstanceClass clazz, int superSize, Host
             startSize = DeoptimizedFrame.getScratchSpaceOffset() + layout.getDeoptScratchSpace();
         }
 
-        PodSupport pods = PodSupport.singleton();
-        if ((HybridLayout.isHybrid(clazz) && !pods.isPodClass(clazz.getJavaClass())) || pods.mustReserveLengthField(clazz.getJavaClass())) {
-            // Reserve space for the array length field. For pods, the ancestor subclassing Object
-            // must have already reserved this space (unless the pod subclasses Object itself).
+        if (mustReserveLengthField(clazz)) {
             VMError.guarantee(startSize == layout.getArrayLengthOffset());
             int fieldSize = layout.sizeInBytes(JavaKind.Int);
             startSize += fieldSize;

From 6f2e88bd09023e66a4e7ee008d651f8a0c5a323a Mon Sep 17 00:00:00 2001
From: Peter Hofer <peter.hofer@oracle.com>
Date: Thu, 12 May 2022 14:20:14 +0200
Subject: [PATCH 12/12] Fixes and improvements.

---
 .../genscavenge/ThreadLocalAllocation.java    | 28 ++++++++--------
 .../graal/GenScavengeAllocationSnippets.java  | 16 +++++----
 .../graal/nodes/FormatPodNode.java            | 19 +++++++----
 .../com/oracle/svm/core/annotate/Hybrid.java  | 10 +++---
 .../jdk/SubstrateObjectCloneSnippets.java     | 29 ++++++++--------
 .../src/com/oracle/svm/core/heap/Pod.java     | 29 +++++++++++++++-
 .../svm/core/heap/PodReferenceMapDecoder.java | 16 +++++++--
 .../svm/core/heap/StoredContinuation.java     |  2 +-
 .../oracle/svm/core/hub/LayoutEncoding.java   | 33 ++++++++++++-------
 .../svm/hosted/NativeImageGenerator.java      |  2 +-
 .../hosted/config/HybridLayoutSupport.java    | 10 +++---
 .../heap/PodFactorySubstitutionMethod.java    |  2 +-
 .../oracle/svm/hosted/heap/PodSupport.java    |  9 +++--
 .../svm/jni/JNIThreadLocalPinnedObjects.java  |  2 +-
 14 files changed, 135 insertions(+), 72 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
index 92aded8b5cce..f575a886c673 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
@@ -224,15 +224,15 @@ private static Object slowPathNewInstanceWithoutAllocating(DynamicHub hub) {
 
     @SubstrateForeignCallTarget(stubCallingConvention = false)
     private static Object slowPathNewArray(Word objectHeader, int length, int fillStartOffset) {
-        return slowPathNewArrayImpl(objectHeader, length, fillStartOffset, null);
+        return slowPathNewArrayOrPodImpl(objectHeader, length, fillStartOffset, null);
     }
 
     @SubstrateForeignCallTarget(stubCallingConvention = false)
     private static Object slowPathNewPodInstance(Word objectHeader, int arrayLength, int fillStartOffset, byte[] referenceMap) {
-        return slowPathNewArrayImpl(objectHeader, arrayLength, fillStartOffset, referenceMap);
+        return slowPathNewArrayOrPodImpl(objectHeader, arrayLength, fillStartOffset, referenceMap);
     }
 
-    private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fillStartOffset, byte[] podReferenceMap) {
+    private static Object slowPathNewArrayOrPodImpl(Word objectHeader, int length, int fillStartOffset, byte[] podReferenceMap) {
         /*
          * Avoid stack overflow errors while producing memory chunks, because that could leave the
          * heap in an inconsistent state.
@@ -256,7 +256,7 @@ private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fi
                 throw OutOfMemoryUtil.reportOutOfMemoryError(outOfMemoryError);
             }
 
-            Object result = slowPathNewArrayWithoutAllocating(hub, length, size, fillStartOffset, podReferenceMap);
+            Object result = slowPathNewArrayOrPodWithoutAllocating(hub, length, size, fillStartOffset, podReferenceMap);
             runSlowPathHooks();
             return result;
         } finally {
@@ -265,26 +265,26 @@ private static Object slowPathNewArrayImpl(Word objectHeader, int length, int fi
     }
 
     @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in the implementation of allocation.")
-    private static Object slowPathNewArrayWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
+    private static Object slowPathNewArrayOrPodWithoutAllocating(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
         DeoptTester.disableDeoptTesting();
         try {
-            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayWithoutAllocating", DynamicHub.toClass(hub).getName());
+            HeapImpl.exitIfAllocationDisallowed("ThreadLocalAllocation.slowPathNewArrayOrPodWithoutAllocating", DynamicHub.toClass(hub).getName());
             GCImpl.getGCImpl().maybeCollectOnAllocation();
 
             if (size.aboveOrEqual(HeapParameters.getLargeArrayThreshold())) {
                 /* Large arrays go into their own unaligned chunk. */
                 boolean needsZeroing = !HeapChunkProvider.areUnalignedChunksZeroed();
                 UnalignedHeapChunk.UnalignedHeader newTlabChunk = HeapImpl.getChunkProvider().produceUnalignedChunk(size);
-                return allocateLargeArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, needsZeroing, podReferenceMap);
+                return allocateLargeArrayOrPodInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, needsZeroing, podReferenceMap);
             }
             /* Small arrays go into the regular aligned chunk. */
 
             // We might have allocated in the caller and acquired a TLAB with enough space already
             // (but we need to check in an uninterruptible method to be safe)
-            Object array = allocateSmallArrayInCurrentTlab(hub, length, size, fillStartOffset, podReferenceMap);
+            Object array = allocateSmallArrayOrPodInCurrentTlab(hub, length, size, fillStartOffset, podReferenceMap);
             if (array == null) { // We need a new chunk.
                 AlignedHeader newTlabChunk = HeapImpl.getChunkProvider().produceAlignedChunk();
-                array = allocateSmallArrayInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, podReferenceMap);
+                array = allocateSmallArrayOrPodInNewTlab(hub, length, size, fillStartOffset, newTlabChunk, podReferenceMap);
             }
             return array;
         } finally {
@@ -300,7 +300,7 @@ private static Object allocateInstanceInNewTlab(DynamicHub hub, AlignedHeader ne
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
-    private static Object allocateSmallArrayInCurrentTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
+    private static Object allocateSmallArrayOrPodInCurrentTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, byte[] podReferenceMap) {
         if (size.aboveThan(availableTlabMemory(getTlab()))) {
             return null;
         }
@@ -309,13 +309,15 @@ private static Object allocateSmallArrayInCurrentTlab(DynamicHub hub, int length
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory.")
-    private static Object allocateSmallArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, AlignedHeader newTlabChunk, byte[] podReferenceMap) {
+    private static Object allocateSmallArrayOrPodInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, AlignedHeader newTlabChunk, byte[] podReferenceMap) {
         Pointer memory = allocateRawMemoryInNewTlab(size, newTlabChunk);
         return formatArrayOrPod(memory, hub, length, false, FillContent.WITH_ZEROES, fillStartOffset, podReferenceMap);
     }
 
     @Uninterruptible(reason = "Holds uninitialized memory, modifies TLAB")
-    private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset, UnalignedHeader newTlabChunk, boolean needsZeroing, byte[] podReferenceMap) {
+    private static Object allocateLargeArrayOrPodInNewTlab(DynamicHub hub, int length, UnsignedWord size, int fillStartOffset,
+                    UnalignedHeader newTlabChunk, boolean needsZeroing, byte[] podReferenceMap) {
+
         ThreadLocalAllocation.Descriptor tlab = getTlab();
 
         HeapChunk.setNext(newTlabChunk, tlab.getUnalignedChunk());
@@ -344,7 +346,7 @@ private static Object allocateLargeArrayInNewTlab(DynamicHub hub, int length, Un
     private static Object formatArrayOrPod(Pointer memory, DynamicHub hub, int length, boolean unaligned, FillContent fillContent, int fillStartOffset, byte[] podReferenceMap) {
         Class<?> clazz = DynamicHub.toClass(hub);
         if (podReferenceMap != null) {
-            return FormatPodNode.formatPod(memory, clazz, length, podReferenceMap, false, unaligned, fillStartOffset, true);
+            return FormatPodNode.formatPod(memory, clazz, length, podReferenceMap, false, unaligned, fillContent, fillStartOffset, true);
         }
         return FormatArrayNode.formatArray(memory, clazz, length, false, unaligned, fillContent, fillStartOffset, true);
     }
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
index 3ec16b198e06..b688588ea0b6 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSnippets.java
@@ -135,7 +135,7 @@ public Object allocatePod(@NonNullParameter DynamicHub hub, int arrayLength, byt
         if (useTLAB() && probability(FAST_PATH_PROBABILITY, shouldAllocateInTLAB(allocationSize, true)) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
             writeTlabTop(thread, newTop);
             emitPrefetchAllocate(newTop, true);
-            instance = formatPod(top, hub, arrayLength, referenceMap, false, false, afterArrayLengthOffset(),
+            instance = formatPod(top, hub, arrayLength, referenceMap, false, false, FillContent.WITH_ZEROES, afterArrayLengthOffset(),
                             emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, profilingData.snippetCounters);
         } else {
             profilingData.snippetCounters.stub.inc();
@@ -146,23 +146,24 @@ public Object allocatePod(@NonNullParameter DynamicHub hub, int arrayLength, byt
     }
 
     @Snippet
-    public Object formatPodSnippet(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier,
-                    @ConstantParameter boolean supportsBulkZeroing, @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationSnippetCounters snippetCounters) {
+    public Object formatPodSnippet(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned,
+                    FillContent fillContents, int fillStartOffset, @ConstantParameter boolean emitMemoryBarrier, @ConstantParameter boolean supportsBulkZeroing,
+                    @ConstantParameter boolean supportsOptimizedFilling, @ConstantParameter AllocationSnippetCounters snippetCounters) {
 
         DynamicHub hubNonNull = (DynamicHub) PiNode.piCastNonNull(hub, SnippetAnchorNode.anchor());
         byte[] refMapNonNull = (byte[]) PiNode.piCastNonNull(referenceMap, SnippetAnchorNode.anchor());
-        return formatPod(memory, hubNonNull, arrayLength, refMapNonNull, rememberedSet, unaligned, fillStartOffset,
+        return formatPod(memory, hubNonNull, arrayLength, refMapNonNull, rememberedSet, unaligned, fillContents, fillStartOffset,
                         emitMemoryBarrier, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
     }
 
-    private Object formatPod(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, int fillStartOffset,
+    private Object formatPod(Word memory, DynamicHub hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned, FillContent fillContents, int fillStartOffset,
                     boolean emitMemoryBarrier, boolean maybeUnroll, boolean supportsBulkZeroing, boolean supportsOptimizedFilling, AllocationSnippetCounters snippetCounters) {
 
         int layoutEncoding = hub.getLayoutEncoding();
         UnsignedWord allocationSize = LayoutEncoding.getArraySize(layoutEncoding, arrayLength);
 
         Word objectHeader = encodeAsObjectHeader(hub, rememberedSet, unaligned);
-        Object instance = formatArray(objectHeader, allocationSize, arrayLength, memory, FillContent.WITH_ZEROES, fillStartOffset,
+        Object instance = formatArray(objectHeader, allocationSize, arrayLength, memory, fillContents, fillStartOffset,
                         false, maybeUnroll, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
 
         int fromOffset = ConfigurationValues.getObjectLayout().getArrayBaseOffset(JavaKind.Byte);
@@ -341,8 +342,9 @@ public void lower(FormatPodNode node, LoweringTool tool) {
                 args.add("referenceMap", node.getReferenceMap());
                 args.add("rememberedSet", node.getRememberedSet());
                 args.add("unaligned", node.getUnaligned());
+                args.add("fillContents", node.getFillContents());
                 args.add("fillStartOffset", node.getFillStartOffset());
-                args.add("emitMemoryBarrier", node.getEmitMemoryBarrier());
+                args.addConst("emitMemoryBarrier", node.getEmitMemoryBarrier());
                 args.addConst("supportsBulkZeroing", tool.getLowerer().supportsBulkZeroing());
                 args.addConst("supportsOptimizedFilling", tool.getLowerer().supportsOptimizedFilling(graph.getOptions()));
                 args.addConst("snippetCounters", snippetCounters);
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
index bf4383f1973e..3d3acb2d6fa9 100644
--- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
+++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/nodes/FormatPodNode.java
@@ -33,6 +33,7 @@
 import org.graalvm.compiler.nodes.FixedWithNextNode;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.compiler.replacements.AllocationSnippets;
 import org.graalvm.word.Pointer;
 
 @NodeInfo(cycles = CYCLES_64, size = SIZE_64)
@@ -45,11 +46,12 @@ public class FormatPodNode extends FixedWithNextNode implements Lowerable {
     @Input protected ValueNode referenceMap;
     @Input protected ValueNode rememberedSet;
     @Input protected ValueNode unaligned;
+    @Input protected ValueNode fillContents;
     @Input protected ValueNode fillStartOffset;
-    @Input protected ValueNode emitMemoryBarrier;
+    private final boolean emitMemoryBarrier;
 
-    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode arrayLength, ValueNode referenceMap,
-                    ValueNode rememberedSet, ValueNode unaligned, ValueNode fillStartOffset, ValueNode emitMemoryBarrier) {
+    public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode arrayLength, ValueNode referenceMap, ValueNode rememberedSet,
+                    ValueNode unaligned, ValueNode fillContents, ValueNode fillStartOffset, boolean emitMemoryBarrier) {
         super(TYPE, StampFactory.objectNonNull());
         this.memory = memory;
         this.hub = hub;
@@ -57,6 +59,7 @@ public FormatPodNode(ValueNode memory, ValueNode hub, ValueNode arrayLength, Val
         this.referenceMap = referenceMap;
         this.rememberedSet = rememberedSet;
         this.unaligned = unaligned;
+        this.fillContents = fillContents;
         this.fillStartOffset = fillStartOffset;
         this.emitMemoryBarrier = emitMemoryBarrier;
     }
@@ -85,15 +88,19 @@ public ValueNode getUnaligned() {
         return unaligned;
     }
 
+    public ValueNode getFillContents() {
+        return fillContents;
+    }
+
     public ValueNode getFillStartOffset() {
         return fillStartOffset;
     }
 
-    public ValueNode getEmitMemoryBarrier() {
+    public boolean getEmitMemoryBarrier() {
         return emitMemoryBarrier;
     }
 
     @NodeIntrinsic
-    public static native Object formatPod(Pointer memory, Class<?> hub, int arrayLength, byte[] referenceMap,
-                    boolean rememberedSet, boolean unaligned, int fillStartOffset, boolean emitMemoryBarrier);
+    public static native Object formatPod(Pointer memory, Class<?> hub, int arrayLength, byte[] referenceMap, boolean rememberedSet, boolean unaligned,
+                    AllocationSnippets.FillContent fillContents, int fillStartOffset, @ConstantNodeParameter boolean emitMemoryBarrier);
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
index a9c95eb679df..dfc4ec629be9 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/annotate/Hybrid.java
@@ -62,18 +62,18 @@
  * important to keep in mind because methods such as {@link Class#isInstance} will return
  * {@code true} and {@link Class#isArray()} will return {@code false}, while
  * {@link LayoutEncoding#isPureInstance} will return {@code false} and
- * {@link LayoutEncoding#isArrayLike} will return {@code true} and for hybrid objects.
+ * {@link LayoutEncoding#isArrayLike} will return {@code true} for hybrid objects.
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface Hybrid {
 
     /**
-     * The type of the array part of the hybrid class. Must be specified if no field annotated
-     * with @{@link Hybrid.Array} is declared, otherwise that field's type determines the type of
-     * the array part.
+     * The component type of the array part of the hybrid class. Must be specified if no field
+     * annotated with @{@link Hybrid.Array} is declared, otherwise that field's type determines the
+     * type of the array part.
      */
-    Class<?> arrayType() default Array.class;
+    Class<?> componentType() default void.class;
 
     /**
      * If {@code true}, allow the data in the hybrid fields to be duplicated between the hybrid
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
index e9aadeb6a1dd..2fea85c8c82c 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java
@@ -153,15 +153,14 @@ private static Object doClone(Object original) throws CloneNotSupportedException
         }
 
         // copy remaining non-object data
-        if (!isArrayLike) {
-            int objectSize = UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceSize(layoutEncoding));
-            int primitiveDataSize = objectSize - curOffset;
-            assert primitiveDataSize >= 0;
-            assert curOffset >= 0;
-            JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
-            curOffset += primitiveDataSize;
-            assert curOffset == objectSize;
-        }
+        int endOffset = isArrayLike ? LayoutEncoding.getArrayBaseOffsetAsInt(layoutEncoding)
+                        : UnsignedUtils.safeToInt(LayoutEncoding.getPureInstanceSize(layoutEncoding));
+        int primitiveDataSize = endOffset - curOffset;
+        assert primitiveDataSize >= 0;
+        assert curOffset >= 0;
+        JavaMemoryUtil.copyForward(original, WordFactory.unsigned(curOffset), result, WordFactory.unsigned(curOffset), WordFactory.unsigned(primitiveDataSize));
+        curOffset += primitiveDataSize;
+        assert curOffset == endOffset;
 
         // reset monitor to uninitialized values
         int monitorOffset = hub.getMonitorOffset();
@@ -178,15 +177,15 @@ static boolean canVirtualize(ObjectClone node, VirtualizerTool tool) {
             return true;
         }
         ResolvedJavaType type = node.getConcreteType(alias.stamp(NodeView.DEFAULT));
-        if (type == null) {
-            return false;
-        }
-        if (!type.isArray() && type instanceof SharedType) {
+        if (type instanceof SharedType) {
             // Hybrids are instances with array-like encoding; cloning virtually is unimplemented.
             int encoding = ((SharedType) type).getHub().getLayoutEncoding();
-            return !LayoutEncoding.isArrayLike(encoding);
+            return !LayoutEncoding.isHybrid(encoding);
+        }
+        if (type != null && type.isArray()) {
+            return true; // cannot be a hybrid
         }
-        return true;
+        return false;
     }
 
     @NodeIntrinsic(value = ForeignCallNode.class)
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
index d232c1c33e5c..a92cda205ad8 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Pod.java
@@ -177,6 +177,19 @@ public Pod<T> build() {
             guaranteeUnbuilt();
             built = true;
 
+            /*
+             * We layout the requested fields in the hybrid object's array part in a similar fashion
+             * as UniverseBuilder does for regular instance fields, putting reference fields at the
+             * beginning and trying to put narrow fields in alignment gaps between wider fields. The
+             * entire layout of a pod might look as follows:
+             *
+             * [ hub pointer | identity hashcode | array length | superclass instance fields |
+             * instance fields | monitor | pod fields (ref, short, byte, long) | pod reference map ]
+             *
+             * The array length part would provide the combined length of the pod fields and pod
+             * reference map excluding any alignment padding at their beginning or the object's end.
+             */
+
             Collections.sort(fields);
 
             UnsignedWord baseOffset = LayoutEncoding.getArrayBaseOffset(
@@ -346,6 +359,20 @@ void add(int offset, int size) {
             bitset.set(index);
         }
 
+        /**
+         * Generates a reference map composed of unsigned byte pairs of {@code nrefs} (sequence of n
+         * references) and {@code gaps} (number of sequential reference-sized primitives). When a
+         * single lengthy sequence of references or primitives cannot be encoded in an unsigned
+         * byte, it is encoded in multiple pairs with either gaps or nrefs being 0. The reference
+         * map covers only the part of the array from the beginning until the last reference. The
+         * end of the reference map is indicated by a pair with {@code gaps} == 0, unless
+         * {@code nrefs} is 0xff which could also mean that the pair is part of an ongoing sequence,
+         * in which case an extra zero pair is added at the end.
+         *
+         * We try to place reference fields at the beginning, so the reference map begins with
+         * {@code nrefs} indicating the number of references right at the start of the array. If
+         * there are primitive fields at the beginning, this first value will be zero.
+         */
         byte[] encode() {
             if (bitset.isEmpty()) {
                 return new byte[]{0, 0};
@@ -354,7 +381,7 @@ byte[] encode() {
             ByteArrayOutputStream buffer = new ByteArrayOutputStream();
             int previous;
             int index = 0;
-            for (;;) { // assume references are placed first
+            for (;;) {
                 previous = index;
                 index = bitset.nextClearBit(previous);
                 int nrefs = index - previous;
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
index b44a6bdabdad..70a035590350 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java
@@ -27,6 +27,7 @@
 import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 import org.graalvm.compiler.word.BarrieredAccess;
+import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
@@ -80,6 +81,12 @@ public static Object clone(Object original, DynamicHub hub, int layoutEncoding)
         return result;
     }
 
+    /**
+     * We could optimize cloning if it turns out to be a bottleneck by avoiding this and passing the
+     * existing pod to {@link NewPodInstanceNode} (and its slow path), but this needs some
+     * duplication of nodes and snippets to avoid touching the {@link LocationIdentity} of extra
+     * objects in situations in which we don't need to.
+     */
     private static byte[] extractReferenceMap(Object obj, int layoutEncoding, int length) {
         UnsignedWord mapEndOffset = LayoutEncoding.getArrayElementOffset(layoutEncoding, length);
         UnsignedWord mapOffset = mapEndOffset;
@@ -114,11 +121,10 @@ private static void copyArray(Object original, Object copy, int layoutEncoding,
             nrefs = WordFactory.unsigned(Byte.toUnsignedInt(BarrieredAccess.readByte(copy, mapOffset.add(1))));
 
             // Copy references separately with the required barriers
-            UnsignedWord refBytes = nrefs.multiply(referenceSize);
-            JavaMemoryUtil.copyReferencesForward(original, refOffset, copy, refOffset, refBytes);
+            JavaMemoryUtil.copyReferencesForward(original, refOffset, copy, refOffset, nrefs);
 
             // Copy primitives in between
-            UnsignedWord primOffset = refOffset.add(refBytes);
+            UnsignedWord primOffset = refOffset.add(nrefs.multiply(referenceSize));
             UnsignedWord primBytes = gap.multiply(referenceSize);
             JavaMemoryUtil.copyForward(original, primOffset, copy, primOffset, primBytes);
 
@@ -127,6 +133,10 @@ private static void copyArray(Object original, Object copy, int layoutEncoding,
 
         // The loop above could be optimized for very long sequences of references or primitive
         // values encoded in multiple reference map entries, but those should be rare in practice.
+
+        // Copy primitives between last reference and reference map
+        UnsignedWord primBytes = mapOffset.subtract(refOffset);
+        JavaMemoryUtil.copyForward(original, refOffset, copy, refOffset, primBytes);
     }
 
     private PodReferenceMapDecoder() {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
index 2e4b9159d522..4b05c86b2531 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/StoredContinuation.java
@@ -31,7 +31,7 @@
  *
  * For object layout and other implementation details, see {@link StoredContinuationImpl}.
  */
-@Hybrid(arrayType = byte[].class)
+@Hybrid(componentType = byte.class)
 public final class StoredContinuation {
     /** Must be allocated via {@link StoredContinuationImpl}. */
     private StoredContinuation() {
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
index 81a2b263abc5..88e9bd330d95 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java
@@ -84,10 +84,13 @@ public class LayoutEncoding {
     private static final int ARRAY_BASE_MASK = 0xfff;
     private static final int ARRAY_TAG_BITS = 3;
     private static final int ARRAY_TAG_SHIFT = Integer.SIZE - ARRAY_TAG_BITS;
-    private static final int ARRAY_TAG_PRIMITIVE_VALUE = 0b111;
-    private static final int ARRAY_TAG_HYBRID_PRIMITIVE_VALUE = 0b110;
-    private static final int ARRAY_TAG_HYBRID_OBJECT_VALUE = 0b101;
-    private static final int ARRAY_TAG_OBJECT_VALUE = 0b100;
+    private static final int ARRAY_TAG_IDENTITY_BIT = 0b100;
+    private static final int ARRAY_TAG_PRIMITIVE_BIT = 0b010;
+    private static final int ARRAY_TAG_PURE_BIT = 0b001; // means non-hybrid
+    private static final int ARRAY_TAG_PRIMITIVE_VALUE = ARRAY_TAG_IDENTITY_BIT | ARRAY_TAG_PRIMITIVE_BIT | ARRAY_TAG_PURE_BIT; // 0b111
+    private static final int ARRAY_TAG_HYBRID_PRIMITIVE_VALUE = ARRAY_TAG_IDENTITY_BIT | ARRAY_TAG_PRIMITIVE_BIT;               // 0b110
+    private static final int ARRAY_TAG_OBJECT_VALUE = ARRAY_TAG_IDENTITY_BIT | ARRAY_TAG_PURE_BIT;                              // 0b101
+    private static final int ARRAY_TAG_HYBRID_OBJECT_VALUE = ARRAY_TAG_IDENTITY_BIT;                                            // 0b100
 
     public static int forPrimitive() {
         return PRIMITIVE_VALUE;
@@ -119,6 +122,7 @@ public static int forPureInstance(ResolvedJavaType type, int size) {
         int encoding = size;
         guaranteeEncoding(type, true, isPureInstance(encoding), "Instance type encoding denotes an instance");
         guaranteeEncoding(type, false, isArray(encoding) || isArrayLike(encoding), "Instance type encoding denotes an array-like object");
+        guaranteeEncoding(type, false, isHybrid(encoding), "Instance type encoding denotes a hybrid");
         guaranteeEncoding(type, false, isObjectArray(encoding) || isArrayLikeWithObjectElements(encoding), "Instance type encoding denotes an object array");
         guaranteeEncoding(type, false, isPrimitiveArray(encoding) || isArrayLikeWithPrimitiveElements(encoding), "Instance type encoding denotes a primitive array");
         guaranteeEncoding(type, true, getPureInstanceSize(encoding).equal(WordFactory.unsigned(size)), "Instance type encoding size matches type size");
@@ -144,6 +148,7 @@ private static int forArrayLike(ResolvedJavaType type, boolean isHybrid, boolean
 
         guaranteeEncoding(type, true, isArrayLike(encoding), "Array-like object encoding denotes an array-like object");
         guaranteeEncoding(type, !isHybrid, isArray(encoding), "Encoding denotes an array");
+        guaranteeEncoding(type, isHybrid, isHybrid(encoding), "Encoding denotes a hybrid");
         guaranteeEncoding(type, false, isPureInstance(encoding), "Array-like object encoding denotes an instance type");
         guaranteeEncoding(type, objectElements, isArrayLikeWithObjectElements(encoding), "Encoding denotes an array-like object with object elements");
         guaranteeEncoding(type, !objectElements, isArrayLikeWithPrimitiveElements(encoding), "Encoding denotes an array-like object with primitive elements");
@@ -180,18 +185,24 @@ public static UnsignedWord getPureInstanceSize(int encoding) {
         return WordFactory.unsigned(encoding);
     }
 
-    // May be inlined because it does not deal in Pointers.
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArray(int encoding) {
-        return isPrimitiveArray(encoding) || isObjectArray(encoding);
+        int mask = (ARRAY_TAG_IDENTITY_BIT | ARRAY_TAG_PURE_BIT) << ARRAY_TAG_SHIFT;
+        return (encoding & mask) == mask;
     }
 
-    // May be inlined because it does not deal in Pointers.
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArrayLike(int encoding) {
         return encoding < NEUTRAL_VALUE;
     }
 
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean isHybrid(int encoding) {
+        int setBits = ARRAY_TAG_IDENTITY_BIT << ARRAY_TAG_SHIFT;
+        int mask = setBits | (ARRAY_TAG_PURE_BIT << ARRAY_TAG_SHIFT);
+        return (encoding & mask) == setBits;
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isPrimitiveArray(int encoding) {
         return UnsignedMath.aboveOrEqual(encoding, ARRAY_TAG_PRIMITIVE_VALUE << ARRAY_TAG_SHIFT);
@@ -199,19 +210,17 @@ public static boolean isPrimitiveArray(int encoding) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isObjectArray(int encoding) {
-        return encoding <= ~(~ARRAY_TAG_OBJECT_VALUE << ARRAY_TAG_SHIFT);
+        return (encoding >>> ARRAY_TAG_SHIFT) == ARRAY_TAG_OBJECT_VALUE;
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArrayLikeWithPrimitiveElements(int encoding) {
-        int tag = (encoding >>> ARRAY_TAG_SHIFT);
-        return (tag == ARRAY_TAG_PRIMITIVE_VALUE || tag == ARRAY_TAG_HYBRID_PRIMITIVE_VALUE);
+        return UnsignedMath.aboveOrEqual(encoding, ARRAY_TAG_HYBRID_PRIMITIVE_VALUE << ARRAY_TAG_SHIFT);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isArrayLikeWithObjectElements(int encoding) {
-        int tag = (encoding >>> ARRAY_TAG_SHIFT);
-        return (tag == ARRAY_TAG_OBJECT_VALUE || tag == ARRAY_TAG_HYBRID_OBJECT_VALUE);
+        return encoding <= ~(~ARRAY_TAG_OBJECT_VALUE << ARRAY_TAG_SHIFT);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
index f588c19ec399..fd84fa886a45 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageGenerator.java
@@ -1671,7 +1671,7 @@ private void printTypes() {
             } else if (LayoutEncoding.isPureInstance(le)) {
                 System.out.format("instance size %d  ", LayoutEncoding.getPureInstanceSize(le).rawValue());
             } else if (LayoutEncoding.isArrayLike(le)) {
-                String arrayType = LayoutEncoding.isArray(le) ? "array" : "array-like object";
+                String arrayType = LayoutEncoding.isHybrid(le) ? "hybrid" : "array";
                 String elements = LayoutEncoding.isArrayLikeWithPrimitiveElements(le) ? "primitives" : "objects";
                 System.out.format("%s containing %s, base %d shift %d scale %d  ", arrayType, elements, LayoutEncoding.getArrayBaseOffset(le).rawValue(),
                                 LayoutEncoding.getArrayIndexShift(le), LayoutEncoding.getArrayIndexScale(le));
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
index cab39a4fe50b..50cb9ebf661e 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/config/HybridLayoutSupport.java
@@ -78,13 +78,15 @@ protected HybridInfo inspectHybrid(HostedInstanceClass hybridClass, MetaAccessPr
         }
 
         HostedType arrayType;
-        boolean arrayTypeIsSet = (annotation.arrayType() != Hybrid.Array.class);
+        boolean arrayTypeIsSet = (annotation.componentType() != void.class);
         if (foundArrayField != null) {
             arrayType = foundArrayField.getType();
-            assert !arrayTypeIsSet || arrayType.equals(metaAccess.lookupJavaType(annotation.arrayType())) : "@Hybrid.arrayType must match the type of a @Hybrid.Array field when both are present";
+
+            assert !arrayTypeIsSet || arrayType.equals(metaAccess.lookupJavaType(annotation.componentType()).getArrayClass()) : //
+            "@Hybrid.componentType must match the type of a @Hybrid.Array field when both are present";
         } else {
-            assert arrayTypeIsSet : "@Hybrid.arrayType must be set when no @Hybrid.Array field is present (if present, ensure it is reachable)";
-            arrayType = (HostedArrayClass) metaAccess.lookupJavaType(annotation.arrayType());
+            assert arrayTypeIsSet : "@Hybrid.componentType must be set when no @Hybrid.Array field is present (if present, ensure it is reachable)";
+            arrayType = (HostedArrayClass) metaAccess.lookupJavaType(annotation.componentType()).getArrayClass();
         }
         assert arrayType.isArray();
         return new HybridInfo(arrayType, foundArrayField, foundTypeIDSlotsField);
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
index 2d0d13acbadf..13190c463868 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodFactorySubstitutionMethod.java
@@ -128,7 +128,7 @@ public StructuredGraph buildGraph(DebugContext debug, ResolvedJavaMethod method,
          * target. We must be careful to use values only from the frame state, so we keep the
          * allocated instance in a local and load it after each step during which a deopt can occur.
          */
-        int instanceLocal = method.getSignature().getParameterCount(true);
+        int instanceLocal = kit.getFrameState().localsSize() - 1; // reserved when generating class
         int nextDeoptIndex = startMethod(kit, isDeoptTarget, 0);
         instantiatePod(kit, providers, factoryType, podConcreteType, instanceLocal);
         if (isAnnotationPresent(DeoptTest.class)) {
diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
index 2607b4db88b1..d51641233e4e 100644
--- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
+++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/heap/PodSupport.java
@@ -65,6 +65,7 @@
 
 import jdk.internal.org.objectweb.asm.ClassWriter;
 import jdk.internal.org.objectweb.asm.Type;
+import jdk.vm.ci.meta.JavaKind;
 
 /** Support for preparing the creation of {@link Pod} objects during the image build. */
 public interface PodSupport {
@@ -204,7 +205,7 @@ private static Class<?> generatePodClass(Class<?> superClass) {
         int access = ACC_PUBLIC | ACC_SUPER | ACC_SYNTHETIC | ACC_FINAL;
         writer.visit(V11, access, className.replace('.', '/'), null, Type.getInternalName(superClass), null);
         var annotation = writer.visitAnnotation(Type.getDescriptor(Hybrid.class), true);
-        annotation.visit("arrayType", Type.getType(byte[].class));
+        annotation.visit("componentType", Type.getType(byte.class));
         annotation.visitEnd();
         writer.visitEnd();
         byte[] data = writer.toByteArray();
@@ -250,7 +251,11 @@ private static Constructor<?> generatePodFactory(Class<?> podClass, Class<?> fac
             impl.visitCode(); // substituted with custom graph
             impl.visitInsn(ACONST_NULL);
             impl.visitInsn(ARETURN);
-            impl.visitMaxs(1, method.getParameterCount() + 1);
+            int localSlots = 1; // spare slot needed by generated graph
+            for (Class<?> clazz : method.getParameterTypes()) {
+                localSlots += JavaKind.fromJavaClass(clazz).getSlotCount();
+            }
+            impl.visitMaxs(1, localSlots);
             impl.visitEnd();
         }
         writer.visitEnd();
diff --git a/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/JNIThreadLocalPinnedObjects.java b/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/JNIThreadLocalPinnedObjects.java
index b064820a41e9..72957b3e41e6 100644
--- a/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/JNIThreadLocalPinnedObjects.java
+++ b/substratevm/src/com.oracle.svm.jni/src/com/oracle/svm/jni/JNIThreadLocalPinnedObjects.java
@@ -87,7 +87,7 @@ public static boolean unpinObject(Object object) {
     }
 
     public static boolean unpinArrayByAddress(PointerBase address) {
-        return unpinFirst(n -> LayoutEncoding.isArray(n.object.getObject()) && n.object.addressOfArrayElement(0) == address);
+        return unpinFirst(n -> LayoutEncoding.isArrayLike(n.object.getObject()) && n.object.addressOfArrayElement(0) == address);
     }
 
     static int pinnedObjectCount() {