diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java new file mode 100644 index 000000000000..50541c51b511 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.posix; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.VoidPointer; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.RestrictHeapAccess; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.posix.headers.Pthread; +import com.oracle.svm.core.posix.headers.Signal; +import com.oracle.svm.core.posix.headers.Time; +import com.oracle.svm.core.sampler.SubstrateSigprofHandler; + +@AutomaticFeature +@SuppressWarnings("unused") +class PosixSubstrateSigprofHandlerFeature implements Feature { + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(SubstrateSigprofHandler.class, new PosixSubstrateSigprofHandler()); + } +} + +public class PosixSubstrateSigprofHandler extends SubstrateSigprofHandler { + + public static final long INTERVAL_S = 0; + public static final long INTERVAL_uS = 20_000; + + @Platforms(Platform.HOSTED_ONLY.class) + public PosixSubstrateSigprofHandler() { + } + + /** The address of the signal handler for signals handled by Java code, below. */ + private static final CEntryPointLiteral advancedSignalDispatcher = CEntryPointLiteral.create(PosixSubstrateSigprofHandler.class, + "dispatch", int.class, Signal.siginfo_t.class, Signal.ucontext_t.class); + + @SuppressWarnings("unused") + @CEntryPoint(include = CEntryPoint.NotIncludedAutomatically.class, publishAs = CEntryPoint.Publish.NotPublished) + @CEntryPointOptions(prologue = CEntryPointOptions.NoPrologue.class, epilogue = CEntryPointOptions.NoEpilogue.class) + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in sigprof signal handler.") + @Uninterruptible(reason = "Signal handler may only execute uninterruptible code.") + private static void dispatch(@SuppressWarnings("unused") int signalNumber, @SuppressWarnings("unused") Signal.siginfo_t sigInfo, Signal.ucontext_t uContext) { + tryEnterIsolateAndDoWalk(uContext); + } + + private static void registerSigprofSignal() { + int structSigActionSize = SizeOf.get(Signal.sigaction.class); + Signal.sigaction structSigAction = StackValue.get(structSigActionSize); + LibC.memset(structSigAction, WordFactory.signed(0), WordFactory.unsigned(structSigActionSize)); + + /* Register sa_sigaction signal handler */ + structSigAction.sa_flags(Signal.SA_SIGINFO() | Signal.SA_NODEFER()); + structSigAction.sa_sigaction(advancedSignalDispatcher.getFunctionPointer()); + Signal.sigaction(Signal.SignalEnum.SIGPROF.getCValue(), structSigAction, WordFactory.nullPointer()); + } + + private static int callSetitimer() { + /* Call setitimer to start profiling. */ + Time.itimerval newValue = StackValue.get(Time.itimerval.class); + Time.itimerval oldValue = StackValue.get(Time.itimerval.class); + + newValue.it_value().set_tv_sec(INTERVAL_S); + newValue.it_value().set_tv_usec(INTERVAL_uS); + newValue.it_interval().set_tv_sec(INTERVAL_S); + newValue.it_interval().set_tv_usec(INTERVAL_uS); + + return Time.NoTransitions.setitimer(Time.TimerTypeEnum.ITIMER_PROF, newValue, oldValue); + } + + @Override + protected void install0() { + registerSigprofSignal(); + PosixUtils.checkStatusIs0(callSetitimer(), "setitimer(which, newValue, oldValue): wrong arguments."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected UnsignedWord createThreadLocalKey() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected void deleteThreadLocalKey(UnsignedWord key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected void setThreadLocalKeyValue(UnsignedWord key, IsolateThread value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected IsolateThread getThreadLocalKeyValue(UnsignedWord key) { + /* + * Although this method is not async-signal-safe in general we rely on + * implementation-specific behavior here. + */ + return (IsolateThread) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } +} diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java index fb447ac4ad57..c3163b6c2540 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java @@ -177,6 +177,9 @@ public interface pthread_key_tPointer extends PointerBase { @CFunction(transition = Transition.NO_TRANSITION) public static native int pthread_key_create(pthread_key_tPointer key, PointerBase keyDestructor); + @CFunction(transition = Transition.NO_TRANSITION) + public static native int pthread_key_delete(pthread_key_t key); + @CFunction(transition = Transition.NO_TRANSITION) public static native int pthread_setspecific(pthread_key_t key, VoidPointer value); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java index ed4c2d76447b..c2bf212c4c8a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java @@ -62,8 +62,19 @@ public void afterCreateIsolate(Isolate isolate) { } } + @Uninterruptible(reason = "The isolate teardown is in progress.") + public void onIsolateTeardown() { + for (int i = 0; i < listeners.length; i++) { + listeners[i].onIsolateTeardown(); + } + } + public interface IsolateListener { @Uninterruptible(reason = "Thread state not yet set up.") void afterCreateIsolate(Isolate isolate); + + @Uninterruptible(reason = "The isolate teardown is in progress.") + default void onIsolateTeardown() { + } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java index ecc54763cd03..322a5487b911 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java @@ -115,6 +115,7 @@ public boolean isCurrentThreadCpuTimeSupported() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getThreadCount() { return threadCount.get(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java index 5d13e056578c..a67410ee653f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java @@ -73,14 +73,13 @@ public interface JfrNativeEventWriterData extends PointerBase { void setCurrentPos(Pointer value); /** - * Returns the end position for the current event write. Writing of data cannot exceed this - * position. + * Returns the position where the buffer ends. Writing of data cannot exceed this position. */ @RawField Pointer getEndPos(); /** - * Sets the end position for the current event write. + * Sets the position where the buffer ends. */ @RawField void setEndPos(Pointer value); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java index 7fe7efb266bb..2c6d918d12c6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java @@ -261,7 +261,9 @@ public long getThreadId(Thread thread) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long getThreadId(IsolateThread isolateThread) { - return threadLocal.getTraceId(isolateThread); + long threadId = threadLocal.getTraceId(isolateThread); + VMError.guarantee(threadId > 0); + return threadId; } /** See {@link JVM#storeMetadataDescriptor}. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java new file mode 100644 index 000000000000..5f1f57f15cbc --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +/** + * A {@link SamplerBuffer} is a block of native memory into which the results of stack walks are + * written. + */ +@RawStructure +interface SamplerBuffer extends PointerBase { + + /** + * Returns the buffer that is next in the {@link SamplerBufferStack}, otherwise null. + */ + @RawField + SamplerBuffer getNext(); + + /** + * Sets the successor to this buffer in the {@link SamplerBufferStack}. + */ + @RawField + void setNext(SamplerBuffer buffer); + + /** + * Returns the JFR id of the thread that owns this buffer. + */ + @RawField + long getOwner(); + + /** + * Sets the JFR id of the thread that owns this buffer. + */ + @RawField + void setOwner(long threadId); + + /** + * Returns the current position. Any data before this position is valid sample data. + */ + @RawField + Pointer getPos(); + + /** + * Sets the current position. + */ + @RawField + void setPos(Pointer pos); + + /** + * Returns the size of the buffer. This excludes the header of the buffer. + */ + @RawField + UnsignedWord getSize(); + + /** + * Sets the size of the buffer. + */ + @RawField + void setSize(UnsignedWord value); + + /** + * Should this buffer be freed after processing the data in it. + */ + @RawField + boolean getFreeable(); + + /** + * Sets the freeable status of the buffer. + */ + @RawField + void setFreeable(boolean freeable); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java new file mode 100644 index 000000000000..f8021d4ee885 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.UnsignedUtils; + +/** + * Used to access the raw memory of a {@link SamplerBufferAccess}. + */ +final class SamplerBufferAccess { + + private SamplerBufferAccess() { + } + + @Fold + public static UnsignedWord getHeaderSize() { + return UnsignedUtils.roundUp(SizeOf.unsigned(SamplerBuffer.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static SamplerBuffer allocate(UnsignedWord dataSize) { + UnsignedWord headerSize = SamplerBufferAccess.getHeaderSize(); + SamplerBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + if (result.isNonNull()) { + result.setSize(dataSize); + result.setFreeable(false); + reinitialize(result); + } + return result; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void free(SamplerBuffer buffer) { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void reinitialize(SamplerBuffer buffer) { + Pointer dataStart = getDataStart(buffer); + buffer.setPos(dataStart); + buffer.setOwner(0L); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer getDataStart(SamplerBuffer buffer) { + return ((Pointer) buffer).add(getHeaderSize()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean isEmpty(SamplerBuffer buffer) { + return getDataStart(buffer).equal(buffer.getPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer getDataEnd(SamplerBuffer buffer) { + return getDataStart(buffer).add(buffer.getSize()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java new file mode 100644 index 000000000000..96af063d0e6b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.util.VMError; + +import jdk.jfr.internal.Options; + +/** + * The pool that maintains the desirable number of buffers in the system by allocating/releasing + * extra buffers. + */ +class SamplerBufferPool { + + private static final long THREAD_BUFFER_SIZE = Options.getThreadBufferSize(); + + private static final VMMutex mutex = new VMMutex("SamplerBufferPool"); + + private static long bufferCount; + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + public static void releaseBufferAndAdjustCount(SamplerBuffer threadLocalBuffer) { + adjustBufferCount0(threadLocalBuffer); + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + public static void adjustBufferCount() { + adjustBufferCount0(WordFactory.nullPointer()); + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + private static void adjustBufferCount0(SamplerBuffer threadLocalBuffer) { + mutex.lockNoTransition(); + try { + releaseThreadLocalBuffer(threadLocalBuffer); + long diff = diff(); + if (diff > 0) { + for (int i = 0; i < diff; i++) { + if (!allocateAndPush()) { + break; + } + } + } else { + for (long i = diff; i < 0; i++) { + if (!popAndFree()) { + break; + } + } + } + } finally { + mutex.unlock(); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + private static void releaseThreadLocalBuffer(SamplerBuffer buffer) { + /* + * buffer is null if the thread is not running yet, or we did not perform the stack walk for + * this thread during the run. + */ + if (buffer.isNonNull()) { + if (SamplerBufferAccess.isEmpty(buffer)) { + /* We can free it right away. */ + SamplerBufferAccess.free(buffer); + } else { + /* Put it in the stack with other unprocessed buffers. */ + buffer.setFreeable(true); + SubstrateSigprofHandler.singleton().fullBuffers().pushBuffer(buffer); + } + VMError.guarantee(bufferCount > 0); + bufferCount--; + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean allocateAndPush() { + VMError.guarantee(bufferCount >= 0); + SamplerBuffer buffer = SamplerBufferAccess.allocate(WordFactory.unsigned(THREAD_BUFFER_SIZE)); + if (buffer.isNonNull()) { + SubstrateSigprofHandler.singleton().availableBuffers().pushBuffer(buffer); + bufferCount++; + return true; + } else { + return false; + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean popAndFree() { + VMError.guarantee(bufferCount > 0); + SamplerBuffer buffer = SubstrateSigprofHandler.singleton().availableBuffers().popBuffer(); + if (buffer.isNonNull()) { + SamplerBufferAccess.free(buffer); + bufferCount--; + return true; + } else { + return false; + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static long diff() { + double diffD = SubstrateSigprofHandler.singleton().substrateThreadMXBean().getThreadCount() * 1.5 - bufferCount; + return (long) (diffD + 0.5); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java new file mode 100644 index 000000000000..229e942b6892 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; + +/** + * The linked-list implementation of the stack that holds a sequence of native memory buffers. + * + * The stack uses spin-lock to protect itself from races with competing pop operations (ABA + * problem). + * + * @see SamplerSpinLock + */ +class SamplerBufferStack { + + private SamplerBuffer head; + private final SamplerSpinLock spinLock; + + @Platforms(Platform.HOSTED_ONLY.class) + SamplerBufferStack() { + this.spinLock = new SamplerSpinLock(); + } + + /** + * Push the buffer into the linked-list. + */ + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + public void pushBuffer(SamplerBuffer buffer) { + spinLock.lock(); + try { + buffer.setNext(head); + head = buffer; + } finally { + spinLock.unlock(); + } + } + + /** + * Pop the buffer from the linked-list. Returns {@code null} if the list is empty. + */ + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + public SamplerBuffer popBuffer() { + spinLock.lock(); + try { + SamplerBuffer result = head; + if (result.isNonNull()) { + head = head.getNext(); + result.setNext(WordFactory.nullPointer()); + } + return result; + } finally { + spinLock.unlock(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean isLockedByCurrentThread() { + return spinLock.isOwner(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java new file mode 100644 index 000000000000..bd25d3101b71 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.word.LocationIdentity; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.IsolateListenerSupport; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.c.CGlobalData; +import com.oracle.svm.core.c.CGlobalDataFactory; + +class SamplerIsolateLocal implements IsolateListenerSupport.IsolateListener { + + /** Stores the address of the first isolate created. */ + private static final CGlobalData firstIsolate = CGlobalDataFactory.createWord(); + + /** Stores the isolate-specific key. */ + private static UnsignedWord key = WordFactory.unsigned(0); + + @Override + @Uninterruptible(reason = "Thread state not yet set up.") + public void afterCreateIsolate(Isolate isolate) { + if (SubstrateSigprofHandler.isProfilingSupported()) { + if (firstIsolate.get().logicCompareAndSwapWord(0, WordFactory.zero(), isolate, LocationIdentity.ANY_LOCATION)) { + key = SubstrateSigprofHandler.singleton().createThreadLocalKey(); + } + } + } + + @Override + @Uninterruptible(reason = "The isolate teardown is in progress.") + public void onIsolateTeardown() { + if (SubstrateSigprofHandler.singleton().isProfilingEnabled() && isKeySet()) { + /* Invalidate the isolate-specific key. */ + UnsignedWord oldKey = key; + key = WordFactory.unsigned(0); + + /* Manually disable sampling for the current thread (no other threads are remaining). */ + SamplerThreadLocal.teardown(CurrentIsolate.getCurrentThread()); + + /* Now, it's safe to delete the isolate-specific key. */ + SubstrateSigprofHandler.singleton().deleteThreadLocalKey(oldKey); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Isolate getIsolate() { + return firstIsolate.get().readWord(0); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getKey() { + return key; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean isKeySet() { + return key.aboveThan(0); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java new file mode 100644 index 000000000000..7290fea6c6c6 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.util.VMError; + +final class SamplerSampleWriter { + + private static final int END_MARKER_SIZE = Long.BYTES; + private static final long END_MARKER = -1; + + private SamplerSampleWriter() { + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean putLong(SamplerSampleWriterData data, long value) { + if (ensureSize(data, Long.BYTES)) { + data.getCurrentPos().writeLong(0, value); + increaseCurrentPos(data, WordFactory.unsigned(Long.BYTES)); + return true; + } else { + return false; + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void commit(SamplerSampleWriterData data) { + SamplerBuffer buffer = data.getSamplerBuffer(); + /* + * put END_MARKER should not fail as ensureSize takes end marker size in consideration. + */ + VMError.guarantee(getAvailableSize(data).aboveOrEqual(END_MARKER_SIZE)); + data.getCurrentPos().writeLong(0, END_MARKER); + increaseCurrentPos(data, WordFactory.unsigned(Long.BYTES)); + + buffer.setPos(data.getCurrentPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean ensureSize(SamplerSampleWriterData data, int requested) { + assert requested > 0; + int totalRequested = requested + END_MARKER_SIZE; + if (getAvailableSize(data).belowThan(totalRequested)) { + if (!accommodate(data, getUncommittedSize(data))) { + return false; + } + } + assert getAvailableSize(data).aboveOrEqual(totalRequested); + return true; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean accommodate(SamplerSampleWriterData data, UnsignedWord uncommitted) { + if (SamplerBufferAccess.isEmpty(data.getSamplerBuffer())) { + /* + * Sample is too big to fit into the size of one buffer i.e. we want to do + * accommodations while nothing is committed into buffer. + */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + + /* Pop first free buffer from the pool. */ + SamplerBuffer newBuffer = SubstrateSigprofHandler.singleton().availableBuffers().popBuffer(); + if (newBuffer.isNull()) { + /* No available buffers on the pool. Fallback! */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + SamplerThreadLocal.setThreadLocalBuffer(newBuffer); + + /* Copy the uncommitted content of old buffer into new one. */ + UnmanagedMemoryUtil.copy(data.getStartPos(), SamplerBufferAccess.getDataStart(newBuffer), uncommitted); + + /* Put in the stack with other unprocessed buffers. */ + SamplerBuffer oldBuffer = data.getSamplerBuffer(); + SubstrateSigprofHandler.singleton().fullBuffers().pushBuffer(oldBuffer); + + /* Reinitialize data structure. */ + data.setSamplerBuffer(newBuffer); + reset(data); + increaseCurrentPos(data, uncommitted); + return true; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static UnsignedWord getAvailableSize(SamplerSampleWriterData data) { + return data.getEndPos().subtract(data.getCurrentPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static UnsignedWord getUncommittedSize(SamplerSampleWriterData data) { + return data.getCurrentPos().subtract(data.getStartPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void increaseCurrentPos(SamplerSampleWriterData data, UnsignedWord delta) { + data.setCurrentPos(data.getCurrentPos().add(delta)); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void reset(SamplerSampleWriterData data) { + SamplerBuffer buffer = data.getSamplerBuffer(); + data.setStartPos(buffer.getPos()); + data.setCurrentPos(buffer.getPos()); + data.setEndPos(SamplerBufferAccess.getDataEnd(buffer)); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java new file mode 100644 index 000000000000..49cdaaf18635 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; + +/** + * A data structure that holds the mutable state of a {@link SamplerSampleWriter}. Typically, it is + * allocated on the stack. + */ +@RawStructure +interface SamplerSampleWriterData extends PointerBase { + /** + * Gets the buffer that data will be written to. + */ + @RawField + SamplerBuffer getSamplerBuffer(); + + /** + * Sets the buffer that data will be written to. + */ + @RawField + void setSamplerBuffer(SamplerBuffer buffer); + + /** + * Gets the start position for the current sample write. + */ + @RawField + Pointer getStartPos(); + + /** + * Sets the start position for the current sample write. + */ + @RawField + void setStartPos(Pointer value); + + /** + * Gets the current position of the sample write. This position is moved forward as data is + * written for a sample. + */ + @RawField + Pointer getCurrentPos(); + + /** + * Sets the current position of the sample write. + */ + @RawField + void setCurrentPos(Pointer value); + + /** + * Returns the position where the buffer ends. Writing of data cannot exceed this position. + */ + @RawField + Pointer getEndPos(); + + /** + * Sets the position where the buffer ends. + */ + @RawField + void setEndPos(Pointer value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java new file mode 100644 index 000000000000..85dc89e26577 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.compiler.nodes.PauseNode; +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.util.VMError; + +/** + * The custom implementation of spin lock that is async signal safe. + * + * In some specific situations, the signal handler can interrupt execution while the same thread + * already has the lock. This implementation will check and fatally fail while other spin locks + * implementations can deadlock in this case. So it is essential to check if the current thread is + * the owner of the lock, before acquiring it. + */ +class SamplerSpinLock { + private final UninterruptibleUtils.AtomicPointer owner; + + @Platforms(Platform.HOSTED_ONLY.class) + SamplerSpinLock() { + this.owner = new UninterruptibleUtils.AtomicPointer<>(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean isOwner() { + return owner.get().equal(CurrentIsolate.getCurrentThread()); + } + + @Uninterruptible(reason = "This method does not do a transition, so the whole critical section must be uninterruptible.", callerMustBe = true) + public void lock() { + VMError.guarantee(!isOwner(), "The current thread already has the lock!"); + IsolateThread currentThread = CurrentIsolate.getCurrentThread(); + while (!owner.compareAndSet(WordFactory.nullPointer(), currentThread)) { + PauseNode.pause(); + } + } + + @Uninterruptible(reason = "The whole critical section must be uninterruptible.", callerMustBe = true) + public void unlock() { + VMError.guarantee(isOwner(), "The current thread doesn't have the lock!"); + owner.compareAndSet(CurrentIsolate.getCurrentThread(), WordFactory.nullPointer()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java new file mode 100644 index 000000000000..3998a5b550d4 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.word.Pointer; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.deopt.DeoptimizedFrame; +import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; + +final class SamplerStackWalkVisitor extends ParameterizedStackFrameVisitor { + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame, Void voidData) { + return SamplerSampleWriter.putLong(SamplerThreadLocal.getWriterData(), ip.rawValue()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Void data) { + SamplerThreadLocal.increaseUnparseableStacks(); + return false; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java new file mode 100644 index 000000000000..6cd132cda1fe --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.jfr.SubstrateJVM; +import com.oracle.svm.core.thread.ThreadListener; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalLong; +import com.oracle.svm.core.threadlocal.FastThreadLocalWord; + +class SamplerThreadLocal implements ThreadListener { + + private static final FastThreadLocalWord localBuffer = FastThreadLocalFactory.createWord("SamplerThreadLocal.localBuffer"); + private static final FastThreadLocalLong missedSamples = FastThreadLocalFactory.createLong("SamplerThreadLocal.missedSamples"); + private static final FastThreadLocalLong unparseableStacks = FastThreadLocalFactory.createLong("SamplerThreadLocal.unparseableStacks"); + /** + * The data that we are using during the stack walk, allocated on the stack. + */ + private static final FastThreadLocalWord writerData = FastThreadLocalFactory.createWord("SamplerThreadLocal.writerData"); + + @Override + @Uninterruptible(reason = "Only uninterruptible code may be executed before Thread.run.") + public void beforeThreadRun(IsolateThread isolateThread, Thread javaThread) { + if (SubstrateSigprofHandler.singleton().isProfilingEnabled()) { + initialize(isolateThread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static void initialize(IsolateThread isolateThread) { + if (SamplerIsolateLocal.isKeySet()) { + /* Adjust the number of buffers. */ + SamplerBufferPool.adjustBufferCount(); + + /* + * Save isolate thread in thread-local area. + * + * Once this value is set, the signal handler may interrupt this thread at any time. So, + * it is essential that this value is set at the very end of this method. + */ + UnsignedWord key = SamplerIsolateLocal.getKey(); + SubstrateSigprofHandler.singleton().setThreadLocalKeyValue(key, isolateThread); + } + } + + @Override + @Uninterruptible(reason = "Only uninterruptible code may be executed after Thread.exit.") + public void afterThreadExit(IsolateThread isolateThread, Thread javaThread) { + if (SubstrateSigprofHandler.singleton().isProfilingEnabled() && SamplerIsolateLocal.isKeySet()) { + teardown(isolateThread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static void teardown(IsolateThread isolateThread) { + /* + * Invalidate thread-local area. + * + * Once this value is set to null, the signal handler can't interrupt this thread anymore. + * So, it is essential that this value is set at the very beginning of this method i.e. + * before doing cleanup. + */ + UnsignedWord key = SamplerIsolateLocal.getKey(); + SubstrateSigprofHandler.singleton().setThreadLocalKeyValue(key, WordFactory.nullPointer()); + + /* Adjust the number of buffers (including the thread-local buffer). */ + SamplerBuffer threadLocalBuffer = localBuffer.get(isolateThread); + SamplerBufferPool.releaseBufferAndAdjustCount(threadLocalBuffer); + localBuffer.set(isolateThread, WordFactory.nullPointer()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static SamplerBuffer getThreadLocalBuffer() { + return localBuffer.get(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void setThreadLocalBuffer(SamplerBuffer buffer) { + buffer.setOwner(SubstrateJVM.get().getThreadId(CurrentIsolate.getCurrentThread())); + localBuffer.set(buffer); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void increaseMissedSamples() { + missedSamples.set(getMissedSamples() + 1); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long getMissedSamples() { + return missedSamples.get(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void increaseUnparseableStacks() { + unparseableStacks.set(getUnparseableStacks() + 1); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long getUnparseableStacks() { + return unparseableStacks.get(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void setWriterData(SamplerSampleWriterData data) { + writerData.set(data); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static SamplerSampleWriterData getWriterData() { + return writerData.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java new file mode 100644 index 000000000000..b3751bce98dc --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import java.lang.management.ManagementFactory; +import java.util.Arrays; +import java.util.List; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.options.Option; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.IsolateListenerSupport; +import com.oracle.svm.core.RegisterDumper; +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.code.CodeInfoTable; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.nodes.WriteHeapBaseNode; +import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.jdk.RuntimeSupport; +import com.oracle.svm.core.jdk.management.ManagementFeature; +import com.oracle.svm.core.jdk.management.SubstrateThreadMXBean; +import com.oracle.svm.core.jfr.JfrFeature; +import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.stack.JavaFrameAnchor; +import com.oracle.svm.core.stack.JavaFrameAnchors; +import com.oracle.svm.core.stack.JavaStackWalker; +import com.oracle.svm.core.thread.JavaVMOperation; +import com.oracle.svm.core.thread.ThreadListenerFeature; +import com.oracle.svm.core.thread.ThreadListenerSupport; +import com.oracle.svm.core.thread.VMThreads; + +@AutomaticFeature +@SuppressWarnings("unused") +class SubstrateSigprofHandlerFeature implements Feature { + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return VMInspectionOptions.AllowVMInspection.getValue() && SubstrateSigprofHandler.isProfilingSupported(); + } + + @Override + public List> getRequiredFeatures() { + return Arrays.asList(ThreadListenerFeature.class, JfrFeature.class, ManagementFeature.class); + } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + if (!ImageSingletons.contains(SubstrateSigprofHandler.class)) { + /* No sigprof handler. */ + return; + } + + /* Create stack visitor. */ + ImageSingletons.add(SamplerStackWalkVisitor.class, new SamplerStackWalkVisitor()); + + /* Add listeners. */ + ThreadListenerSupport.get().register(new SamplerThreadLocal()); + IsolateListenerSupport.singleton().register(new SamplerIsolateLocal()); + + /* Add startup hook. */ + RuntimeSupport.getRuntimeSupport().addStartupHook(new SubstrateSigprofHandlerStartupHook()); + } +} + +final class SubstrateSigprofHandlerStartupHook implements RuntimeSupport.Hook { + @Override + public void execute(boolean isFirstIsolate) { + if (isFirstIsolate) { + SubstrateSigprofHandler.singleton().install(); + } + } +} + +/** + *

+ * The core class of low overhead asynchronous sampling based profiler. + * {@link SubstrateSigprofHandler} handles the periodic signal generated by the OS with a given + * time-frequency. The asynchronous nature of the signal means that the OS could invoke the signal + * handler at any time (could be during GC, VM operation, uninterruptible code) and that the signal + * handler can only execute specific code i.e. the calls that are asynchronous signal safe. + *

+ * + *

+ * The signal handler is divided into three part: restore isolate, isolate-thread, stack and + * instruction pointers, prepare everything necessary for stack walk, do a stack walk and write IPs + * into buffer. + *

+ * + *

+ * The signal handler is as a producer. On the other side of relation is + * {@link com.oracle.svm.core.jfr.JfrRecorderThread} that is consumer. The + * {@link SamplerBuffer} that we are using in this consumer-producer communication is allocated + * eagerly, in a part of the heap that is not accessible via GC, and there will always be more + * available buffers that threads. + *

+ * + *

+ * The communication between consumer and producer goes as follows: + *

    + *
  • Signal handler (producer): pops the buffer from the pool of available buffers (the buffer now + * becomes thread-local), writes the IPs into buffer, if the buffer is full and moves it to a pool + * with buffers that awaits processing.
    • + *
  • Recorder thread (consumer): pops the buffer from the pool of full buffers, reconstructs the + * stack walk based on IPs and pushes the buffer into pool of available buffers.
    • + *
+ * NOTE: The producer and the consumer are always accessing different buffers. + *

+ * + *

+ * In some rare cases, the profiling is impossible e.g. no available buffers in the pool, unknown IP + * during stack walk, the thread holds the pool's lock when the signal arrives, etc. + *

+ * + * @see SamplerSpinLock + * @see SamplerBufferStack + */ +public abstract class SubstrateSigprofHandler { + + public static class Options { + @Option(help = "Allow sampling-based profiling. Default: disabled in execution.")// + static final RuntimeOptionKey SamplingBasedProfiling = new RuntimeOptionKey<>(Boolean.FALSE); + } + + private boolean enabled; + private final SamplerBufferStack availableBuffers; + private final SamplerBufferStack fullBuffers; + private SubstrateThreadMXBean threadMXBean; + + @Platforms(Platform.HOSTED_ONLY.class) + protected SubstrateSigprofHandler() { + this.availableBuffers = new SamplerBufferStack(); + this.fullBuffers = new SamplerBufferStack(); + } + + @Fold + static SubstrateSigprofHandler singleton() { + return ImageSingletons.lookup(SubstrateSigprofHandler.class); + } + + @Fold + static SamplerStackWalkVisitor visitor() { + return ImageSingletons.lookup(SamplerStackWalkVisitor.class); + } + + @Fold + static boolean isProfilingSupported() { + return Platform.includedIn(Platform.LINUX.class); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + boolean isProfilingEnabled() { + return enabled; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + SamplerBufferStack availableBuffers() { + return availableBuffers; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + SamplerBufferStack fullBuffers() { + return fullBuffers; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + SubstrateThreadMXBean substrateThreadMXBean() { + return threadMXBean; + } + + /** + * Installs the platform dependent sigprof handler. + */ + void install() { + if (Options.SamplingBasedProfiling.getValue()) { + threadMXBean = (SubstrateThreadMXBean) ManagementFactory.getThreadMXBean(); + /* Call VM operation to initialize the profiler and the threads. */ + InitializeProfilerOperation initializeProfilerOperation = new InitializeProfilerOperation(); + initializeProfilerOperation.enqueue(); + + /* After the VM operations finishes. Install handler and start profiling. */ + install0(); + } + } + + protected abstract void install0(); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract UnsignedWord createThreadLocalKey(); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract void deleteThreadLocalKey(UnsignedWord key); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract void setThreadLocalKeyValue(UnsignedWord key, IsolateThread value); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract IsolateThread getThreadLocalKeyValue(UnsignedWord key); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean isIPInJavaCode(RegisterDumper.Context uContext) { + Pointer ip = (Pointer) RegisterDumper.singleton().getIP(uContext); + CodeInfo codeInfo = CodeInfoTable.getImageCodeInfo(); + Pointer codeStart = (Pointer) CodeInfoAccess.getCodeStart(codeInfo); + UnsignedWord codeSize = CodeInfoAccess.getCodeSize(codeInfo); + return ip.aboveOrEqual(codeStart) && ip.belowOrEqual(codeStart.add(codeSize)); + } + + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + private static void doUninterruptibleStackWalk(RegisterDumper.Context uContext, boolean isIPInJavaCode) { + CodePointer ip; + Pointer sp; + if (isIPInJavaCode) { + ip = (CodePointer) RegisterDumper.singleton().getIP(uContext); + sp = (Pointer) RegisterDumper.singleton().getSP(uContext); + } else { + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(); + if (anchor.isNull()) { + /* + * The anchor is still null if the function is interrupted during prologue. See: + * com.oracle.svm.core.graal.snippets.CFunctionSnippets.prologueSnippet + */ + return; + } + + ip = anchor.getLastJavaIP(); + sp = anchor.getLastJavaSP(); + + if (ip.isNull() || sp.isNull()) { + /* + * It can happen that anchor is in list of all anchors, but its IP and SP are not + * filled yet. + */ + return; + } + } + + /* Initialize stack walk. */ + SamplerSampleWriterData data = StackValue.get(SamplerSampleWriterData.class); + if (prepareStackWalk(data)) { + /* Walk the stack. */ + if (JavaStackWalker.walkCurrentThread(sp, ip, visitor())) { + SamplerSampleWriter.commit(data); + } + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean prepareStackWalk(SamplerSampleWriterData data) { + if (singleton().availableBuffers().isLockedByCurrentThread() || singleton().fullBuffers().isLockedByCurrentThread()) { + /* + * The current thread already holds the stack lock, so we can't access it. It's way + * better to lose one sample, then potentially the whole buffer. + */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + + SamplerBuffer buffer = SamplerThreadLocal.getThreadLocalBuffer(); + if (buffer.isNull()) { + /* Pop first free buffer from the pool. */ + buffer = singleton().availableBuffers().popBuffer(); + if (buffer.isNull()) { + /* No available buffers on the pool. Fallback! */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + SamplerThreadLocal.setThreadLocalBuffer(buffer); + } + + /* Initialize the buffer. */ + data.setSamplerBuffer(buffer); + data.setStartPos(buffer.getPos()); + data.setCurrentPos(buffer.getPos()); + data.setEndPos(SamplerBufferAccess.getDataEnd(buffer)); + SamplerThreadLocal.setWriterData(data); + return true; + } + + /** Called from the platform dependent sigprof handler to enter isolate. */ + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + protected static void tryEnterIsolateAndDoWalk(RegisterDumper.Context uContext) { + Isolate isolate = SamplerIsolateLocal.getIsolate(); + if (isolate.isNull()) { + /* It may happen that the initial isolate exited. */ + return; + } + + /* Write isolate pointer (heap base) into register. */ + WriteHeapBaseNode.writeCurrentVMHeapBase(isolate); + + /* We are keeping reference to isolate thread inside OS thread local area. */ + if (!SamplerIsolateLocal.isKeySet()) { + /* The key becomes invalid during initial isolate teardown. */ + return; + } + UnsignedWord key = SamplerIsolateLocal.getKey(); + IsolateThread thread = singleton().getThreadLocalKeyValue(key); + if (thread.isNull()) { + /* Thread is not yet initialized or already detached from isolate. */ + return; + } + + /* Write isolate thread pointer into register. */ + WriteCurrentVMThreadNode.writeCurrentVMThread(thread); + + /* Check if the instruction pointer (IP) is inside native or java code. */ + boolean isIPInJavaCode = isIPInJavaCode(uContext); + + /* Walk the stack. */ + doUninterruptibleStackWalk(uContext, isIPInJavaCode); + } + + private class InitializeProfilerOperation extends JavaVMOperation { + + protected InitializeProfilerOperation() { + super(VMOperationInfos.get(InitializeProfilerOperation.class, "Initialize Profiler", SystemEffect.SAFEPOINT)); + } + + @Override + protected void operate() { + initialize(); + } + + /** + * We need to ensure that all threads are properly initialized at a moment when we start a + * profiling. + */ + @Uninterruptible(reason = "Prevent pollution of the current thread's thread local JFR buffer.") + private void initialize() { + /* + * Iterate all over all thread and initialize the thread-local storage of each thread. + */ + for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) { + SamplerThreadLocal.initialize(thread); + } + enabled = true; + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java index 2436aaf19898..a0283ea6b14b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java @@ -65,7 +65,7 @@ public void beforeThreadStart(IsolateThread isolateThread, Thread javaThread) { @Uninterruptible(reason = "Force that all listeners are uninterruptible.") public void afterThreadExit(IsolateThread isolateThread, Thread javaThread) { - for (int i = 0; i < listeners.length; i++) { + for (int i = listeners.length - 1; i >= 0; i--) { listeners[i].afterThreadExit(isolateThread, javaThread); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index fd596ed88cca..6a28ba4f18ef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.thread; +import com.oracle.svm.core.IsolateListenerSupport; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.replacements.ReplacementsUtil; @@ -450,6 +451,7 @@ public void tearDown() { VMOperationControl.shutdownAndDetachVMOperationThread(); } // At this point, it is guaranteed that all other threads were detached. + IsolateListenerSupport.singleton().onIsolateTeardown(); waitUntilLastOsThreadExited(); }