From fd23532d1ab18a8b20f60c470ce75f81434563d9 Mon Sep 17 00:00:00 2001 From: jovanstevanovic Date: Thu, 31 Mar 2022 17:37:05 +0200 Subject: [PATCH 1/3] SIGPROF signal handler. --- .../posix/PosixSubstrateSigprofHandler.java | 143 +++++++ .../svm/core/posix/headers/Pthread.java | 3 + .../svm/core/IsolateListenerSupport.java | 11 + .../jdk/management/SubstrateThreadMXBean.java | 1 + .../core/jfr/JfrNativeEventWriterData.java | 5 +- .../com/oracle/svm/core/jfr/SubstrateJVM.java | 4 +- .../svm/core/sampler/SamplerBuffer.java | 100 +++++ .../svm/core/sampler/SamplerBufferAccess.java | 91 +++++ .../svm/core/sampler/SamplerBufferPool.java | 130 ++++++ .../svm/core/sampler/SamplerBufferStack.java | 91 +++++ .../svm/core/sampler/SamplerIsolateLocal.java | 88 ++++ .../svm/core/sampler/SamplerSampleWriter.java | 133 ++++++ .../core/sampler/SamplerSampleWriterData.java | 87 ++++ .../svm/core/sampler/SamplerSpinLock.java | 73 ++++ .../core/sampler/SamplerStackWalkVisitor.java | 49 +++ .../svm/core/sampler/SamplerThreadLocal.java | 141 +++++++ .../core/sampler/SubstrateSigprofHandler.java | 381 ++++++++++++++++++ .../core/thread/ThreadListenerSupport.java | 2 +- .../com/oracle/svm/core/thread/VMThreads.java | 2 + 19 files changed, 1530 insertions(+), 5 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java create mode 100644 substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java new file mode 100644 index 000000000000..50541c51b511 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.posix; + +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.VoidPointer; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.RestrictHeapAccess; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.c.function.CEntryPointOptions; +import com.oracle.svm.core.headers.LibC; +import com.oracle.svm.core.posix.headers.Pthread; +import com.oracle.svm.core.posix.headers.Signal; +import com.oracle.svm.core.posix.headers.Time; +import com.oracle.svm.core.sampler.SubstrateSigprofHandler; + +@AutomaticFeature +@SuppressWarnings("unused") +class PosixSubstrateSigprofHandlerFeature implements Feature { + @Override + public void afterRegistration(AfterRegistrationAccess access) { + ImageSingletons.add(SubstrateSigprofHandler.class, new PosixSubstrateSigprofHandler()); + } +} + +public class PosixSubstrateSigprofHandler extends SubstrateSigprofHandler { + + public static final long INTERVAL_S = 0; + public static final long INTERVAL_uS = 20_000; + + @Platforms(Platform.HOSTED_ONLY.class) + public PosixSubstrateSigprofHandler() { + } + + /** The address of the signal handler for signals handled by Java code, below. */ + private static final CEntryPointLiteral advancedSignalDispatcher = CEntryPointLiteral.create(PosixSubstrateSigprofHandler.class, + "dispatch", int.class, Signal.siginfo_t.class, Signal.ucontext_t.class); + + @SuppressWarnings("unused") + @CEntryPoint(include = CEntryPoint.NotIncludedAutomatically.class, publishAs = CEntryPoint.Publish.NotPublished) + @CEntryPointOptions(prologue = CEntryPointOptions.NoPrologue.class, epilogue = CEntryPointOptions.NoEpilogue.class) + @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate in sigprof signal handler.") + @Uninterruptible(reason = "Signal handler may only execute uninterruptible code.") + private static void dispatch(@SuppressWarnings("unused") int signalNumber, @SuppressWarnings("unused") Signal.siginfo_t sigInfo, Signal.ucontext_t uContext) { + tryEnterIsolateAndDoWalk(uContext); + } + + private static void registerSigprofSignal() { + int structSigActionSize = SizeOf.get(Signal.sigaction.class); + Signal.sigaction structSigAction = StackValue.get(structSigActionSize); + LibC.memset(structSigAction, WordFactory.signed(0), WordFactory.unsigned(structSigActionSize)); + + /* Register sa_sigaction signal handler */ + structSigAction.sa_flags(Signal.SA_SIGINFO() | Signal.SA_NODEFER()); + structSigAction.sa_sigaction(advancedSignalDispatcher.getFunctionPointer()); + Signal.sigaction(Signal.SignalEnum.SIGPROF.getCValue(), structSigAction, WordFactory.nullPointer()); + } + + private static int callSetitimer() { + /* Call setitimer to start profiling. */ + Time.itimerval newValue = StackValue.get(Time.itimerval.class); + Time.itimerval oldValue = StackValue.get(Time.itimerval.class); + + newValue.it_value().set_tv_sec(INTERVAL_S); + newValue.it_value().set_tv_usec(INTERVAL_uS); + newValue.it_interval().set_tv_sec(INTERVAL_S); + newValue.it_interval().set_tv_usec(INTERVAL_uS); + + return Time.NoTransitions.setitimer(Time.TimerTypeEnum.ITIMER_PROF, newValue, oldValue); + } + + @Override + protected void install0() { + registerSigprofSignal(); + PosixUtils.checkStatusIs0(callSetitimer(), "setitimer(which, newValue, oldValue): wrong arguments."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected UnsignedWord createThreadLocalKey() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected void deleteThreadLocalKey(UnsignedWord key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected void setThreadLocalKeyValue(UnsignedWord key, IsolateThread value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected IsolateThread getThreadLocalKeyValue(UnsignedWord key) { + /* + * Although this method is not async-signal-safe in general we rely on + * implementation-specific behavior here. + */ + return (IsolateThread) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } +} diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java index fb447ac4ad57..c3163b6c2540 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pthread.java @@ -177,6 +177,9 @@ public interface pthread_key_tPointer extends PointerBase { @CFunction(transition = Transition.NO_TRANSITION) public static native int pthread_key_create(pthread_key_tPointer key, PointerBase keyDestructor); + @CFunction(transition = Transition.NO_TRANSITION) + public static native int pthread_key_delete(pthread_key_t key); + @CFunction(transition = Transition.NO_TRANSITION) public static native int pthread_setspecific(pthread_key_t key, VoidPointer value); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java index ed4c2d76447b..c2bf212c4c8a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateListenerSupport.java @@ -62,8 +62,19 @@ public void afterCreateIsolate(Isolate isolate) { } } + @Uninterruptible(reason = "The isolate teardown is in progress.") + public void onIsolateTeardown() { + for (int i = 0; i < listeners.length; i++) { + listeners[i].onIsolateTeardown(); + } + } + public interface IsolateListener { @Uninterruptible(reason = "Thread state not yet set up.") void afterCreateIsolate(Isolate isolate); + + @Uninterruptible(reason = "The isolate teardown is in progress.") + default void onIsolateTeardown() { + } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java index ecc54763cd03..322a5487b911 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/SubstrateThreadMXBean.java @@ -115,6 +115,7 @@ public boolean isCurrentThreadCpuTimeSupported() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getThreadCount() { return threadCount.get(); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java index 5d13e056578c..a67410ee653f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrNativeEventWriterData.java @@ -73,14 +73,13 @@ public interface JfrNativeEventWriterData extends PointerBase { void setCurrentPos(Pointer value); /** - * Returns the end position for the current event write. Writing of data cannot exceed this - * position. + * Returns the position where the buffer ends. Writing of data cannot exceed this position. */ @RawField Pointer getEndPos(); /** - * Sets the end position for the current event write. + * Sets the position where the buffer ends. */ @RawField void setEndPos(Pointer value); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java index 7fe7efb266bb..2c6d918d12c6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java @@ -261,7 +261,9 @@ public long getThreadId(Thread thread) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long getThreadId(IsolateThread isolateThread) { - return threadLocal.getTraceId(isolateThread); + long threadId = threadLocal.getTraceId(isolateThread); + VMError.guarantee(threadId > 0); + return threadId; } /** See {@link JVM#storeMetadataDescriptor}. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java new file mode 100644 index 000000000000..5e5f95e17776 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +/** + * A {@link SamplerBuffer} is a block of native memory into which the results of stack walks are + * written. + */ +@RawStructure +interface SamplerBuffer extends PointerBase { + + /** + * Returns a buffer that is next in the {@link SamplerBufferStack}, otherwise null. + */ + @RawField + SamplerBuffer getNext(); + + /** + * Sets a buffer as a new head in the {@link SamplerBufferStack}. + */ + @RawField + void setNext(SamplerBuffer buffer); + + /** + * Returns the JFR id of the thread that owns this buffer. + */ + @RawField + long getOwner(); + + /** + * Sets the JFR id of the thread that owns this buffer. + */ + @RawField + void setOwner(long threadId); + + /** + * Returns the current position. Any data before this position is valid sample data. + */ + @RawField + Pointer getPos(); + + /** + * Sets the current position. + */ + @RawField + void setPos(Pointer pos); + + /** + * Returns the size of the buffer. This excludes the header of the buffer. + */ + @RawField + UnsignedWord getSize(); + + /** + * Sets the size of the buffer. + */ + @RawField + void setSize(UnsignedWord value); + + /** + * Should this buffer be freed after processing the data in it. + */ + @RawField + boolean getFreeable(); + + /** + * Sets the freeable status of the buffer. + */ + @RawField + void setFreeable(boolean freeable); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java new file mode 100644 index 000000000000..f8021d4ee885 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferAccess.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.config.ConfigurationValues; +import com.oracle.svm.core.util.UnsignedUtils; + +/** + * Used to access the raw memory of a {@link SamplerBufferAccess}. + */ +final class SamplerBufferAccess { + + private SamplerBufferAccess() { + } + + @Fold + public static UnsignedWord getHeaderSize() { + return UnsignedUtils.roundUp(SizeOf.unsigned(SamplerBuffer.class), WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static SamplerBuffer allocate(UnsignedWord dataSize) { + UnsignedWord headerSize = SamplerBufferAccess.getHeaderSize(); + SamplerBuffer result = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(headerSize.add(dataSize)); + if (result.isNonNull()) { + result.setSize(dataSize); + result.setFreeable(false); + reinitialize(result); + } + return result; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void free(SamplerBuffer buffer) { + ImageSingletons.lookup(UnmanagedMemorySupport.class).free(buffer); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void reinitialize(SamplerBuffer buffer) { + Pointer dataStart = getDataStart(buffer); + buffer.setPos(dataStart); + buffer.setOwner(0L); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer getDataStart(SamplerBuffer buffer) { + return ((Pointer) buffer).add(getHeaderSize()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean isEmpty(SamplerBuffer buffer) { + return getDataStart(buffer).equal(buffer.getPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Pointer getDataEnd(SamplerBuffer buffer) { + return getDataStart(buffer).add(buffer.getSize()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java new file mode 100644 index 000000000000..58243c0d8dba --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.util.VMError; + +import jdk.jfr.internal.Options; + +/** + * The pool that maintains the desirable number of buffers in the system by allocating/releasing + * extra buffers. + */ +class SamplerBufferPool { + + private static final long THREAD_BUFFER_SIZE = Options.getThreadBufferSize(); + + private static final VMMutex mutex = new VMMutex("profilerBufferUtils"); + + private static long bufferCount; + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + public static void adjustBufferCount(SamplerBuffer threadLocalBuffer) { + mutex.lockNoTransition(); + try { + releaseThreadLocalBuffer(threadLocalBuffer); + adjustBufferCount0(); + } finally { + mutex.unlock(); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + public static void adjustBufferCount() { + mutex.lockNoTransition(); + try { + adjustBufferCount0(); + } finally { + mutex.unlock(); + } + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + public static void adjustBufferCount0() { + long diff = diff(); + if (diff > 0) { + for (int i = 0; i < diff; i++) { + allocateAndPush(); + } + } else { + for (long i = diff; i < 0; i++) { + if (!popAndFree()) { + break; + } + } + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void allocateAndPush() { + VMError.guarantee(bufferCount >= 0); + SamplerBuffer buffer = SamplerBufferAccess.allocate(WordFactory.unsigned(THREAD_BUFFER_SIZE)); + if (buffer.isNull()) { + return; + } + SubstrateSigprofHandler.availableBuffers().pushBuffer(buffer); + bufferCount++; + } + + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) + private static void releaseThreadLocalBuffer(SamplerBuffer buffer) { + /* buffer will be null if no stack walk were performed. */ + if (buffer.isNonNull()) { + if (SamplerBufferAccess.isEmpty(buffer)) { + /* We can free it right away. */ + SamplerBufferAccess.free(buffer); + } else { + /* Put it in the stack with other unprocessed buffers. */ + buffer.setFreeable(true); + SubstrateSigprofHandler.fullBuffers().pushBuffer(buffer); + } + VMError.guarantee(bufferCount > 0); + bufferCount--; + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean popAndFree() { + VMError.guarantee(bufferCount > 0); + SamplerBuffer buffer = SubstrateSigprofHandler.availableBuffers().popBuffer(); + if (buffer.isNonNull()) { + SamplerBufferAccess.free(buffer); + bufferCount--; + return false; + } + return true; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static long diff() { + double diffD = SubstrateSigprofHandler.getSubstrateThreadMXBean().getThreadCount() * 1.5 - bufferCount; + return (long) (diffD + 0.5); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java new file mode 100644 index 000000000000..db7a82e44577 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; + +/** + * The linked-list implementation of the stack that holds a sequence of native memory buffers. + * + * The stack uses spin-lock to protect itself from races with competing pop operations (ABA + * problem). + * + * @see SamplerSpinLock + */ +class SamplerBufferStack { + + private SamplerBuffer head; + private final SamplerSpinLock spinLock; + + @Platforms(Platform.HOSTED_ONLY.class) + SamplerBufferStack() { + this.spinLock = new SamplerSpinLock(); + } + + /** + * Push a profiler buffer into the global linked list. + */ + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + public void pushBuffer(SamplerBuffer buffer) { + spinLock.lock(); + try { + buffer.setNext(head); + head = buffer; + } finally { + spinLock.unlock(); + } + } + + /** + * Pop a profiler buffer from the global linked list. Returns {@code null} if the list is empty. + */ + @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") + public SamplerBuffer popBuffer() { + spinLock.lock(); + try { + SamplerBuffer result = head; + if (result.isNonNull()) { + head = head.getNext(); + result.setNext(WordFactory.nullPointer()); + } + return result; + } finally { + spinLock.unlock(); + } + } + + /** + * Returns the lock that this stack is using. + */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean isLockedByCurrentThread() { + return spinLock.isOwner(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java new file mode 100644 index 000000000000..28b00171363f --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.word.LocationIdentity; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.IsolateListenerSupport; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.c.CGlobalData; +import com.oracle.svm.core.c.CGlobalDataFactory; + +class SamplerIsolateLocal implements IsolateListenerSupport.IsolateListener { + + /** Stores the address of the first isolate created. */ + private static final CGlobalData firstIsolate = CGlobalDataFactory.createWord(); + + /** Stores the isolate-specific key. */ + private static UnsignedWord key = WordFactory.unsigned(0); + + @Override + @Uninterruptible(reason = "Thread state not yet set up.") + public void afterCreateIsolate(Isolate isolate) { + if (SubstrateSigprofHandler.isProfilingSupported()) { + if (firstIsolate.get().logicCompareAndSwapWord(0, WordFactory.zero(), isolate, LocationIdentity.ANY_LOCATION)) { + key = SubstrateSigprofHandler.singleton().createThreadLocalKey(); + } + } + } + + @Override + @Uninterruptible(reason = "The isolate teardown is in progress.") + public void onIsolateTeardown() { + if (SubstrateSigprofHandler.isProfilingEnabled() && isKeySet()) { + /* Invalidate the isolate-specific key. */ + UnsignedWord oldKey = key; + key = WordFactory.unsigned(0); + + /* We need to manually call teardown because isKeySet will now return false. */ + SamplerThreadLocal.teardown(CurrentIsolate.getCurrentThread()); + + /* Now, it's safe to delete the isolate-specific key. */ + SubstrateSigprofHandler.singleton().deleteThreadLocalKey(oldKey); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static Isolate getIsolate() { + return firstIsolate.get().readWord(0); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static UnsignedWord getKey() { + return key; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean isKeySet() { + return key.aboveThan(0); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java new file mode 100644 index 000000000000..89802833d164 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.util.VMError; + +class SamplerSampleWriter { + + private static final int END_MARKER_SIZE = Long.BYTES; + private static final long END_MARKER = -1; + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static boolean putLong(SamplerSampleWriterData data, long value) { + if (ensureSize(data, Long.BYTES)) { + data.getCurrentPos().writeLong(0, value); + increaseCurrentPos(data, WordFactory.unsigned(Long.BYTES)); + return true; + } else { + return false; + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void commit(SamplerSampleWriterData data) { + SamplerBuffer buffer = data.getSamplerBuffer(); + /* + * put END_MARKER should not fail as ensureSize takes end marker size in consideration. + */ + VMError.guarantee(getAvailableSize(data).aboveOrEqual(END_MARKER_SIZE)); + data.getCurrentPos().writeLong(0, END_MARKER); + increaseCurrentPos(data, WordFactory.unsigned(Long.BYTES)); + + buffer.setPos(data.getCurrentPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean ensureSize(SamplerSampleWriterData data, int requested) { + assert requested > 0; + int totalRequested = requested + END_MARKER_SIZE; + if (getAvailableSize(data).belowThan(totalRequested)) { + if (!accommodate(data, getUncommittedSize(data))) { + return false; + } + } + assert getAvailableSize(data).aboveOrEqual(totalRequested); + return true; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean accommodate(SamplerSampleWriterData data, UnsignedWord uncommitted) { + if (SamplerBufferAccess.isEmpty(data.getSamplerBuffer())) { + /* + * Sample is too big to fit into the size of one buffer i.e. we want to do + * accommodations while nothing is committed into buffer. + */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + + /* Pop first free buffer from the pool. */ + SamplerBuffer newBuffer = SubstrateSigprofHandler.availableBuffers().popBuffer(); + if (newBuffer.isNull()) { + /* No available buffers on the pool. Fallback! */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + SamplerThreadLocal.setThreadLocalBuffer(newBuffer); + + /* Copy the uncommitted content of old buffer into new one. */ + UnmanagedMemoryUtil.copy(data.getStartPos(), SamplerBufferAccess.getDataStart(newBuffer), uncommitted); + + /* Put in the stack with other unprocessed buffers. */ + SamplerBuffer oldBuffer = data.getSamplerBuffer(); + SubstrateSigprofHandler.fullBuffers().pushBuffer(oldBuffer); + + /* Reinitialize data structure. */ + data.setSamplerBuffer(newBuffer); + reset(data); + increaseCurrentPos(data, uncommitted); + return true; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static UnsignedWord getAvailableSize(SamplerSampleWriterData data) { + return data.getEndPos().subtract(data.getCurrentPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static UnsignedWord getUncommittedSize(SamplerSampleWriterData data) { + return data.getCurrentPos().subtract(data.getStartPos()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void increaseCurrentPos(SamplerSampleWriterData data, UnsignedWord delta) { + data.setCurrentPos(data.getCurrentPos().add(delta)); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void reset(SamplerSampleWriterData data) { + SamplerBuffer buffer = data.getSamplerBuffer(); + data.setStartPos(buffer.getPos()); + data.setCurrentPos(buffer.getPos()); + data.setEndPos(SamplerBufferAccess.getDataEnd(buffer)); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java new file mode 100644 index 000000000000..49cdaaf18635 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriterData.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; + +/** + * A data structure that holds the mutable state of a {@link SamplerSampleWriter}. Typically, it is + * allocated on the stack. + */ +@RawStructure +interface SamplerSampleWriterData extends PointerBase { + /** + * Gets the buffer that data will be written to. + */ + @RawField + SamplerBuffer getSamplerBuffer(); + + /** + * Sets the buffer that data will be written to. + */ + @RawField + void setSamplerBuffer(SamplerBuffer buffer); + + /** + * Gets the start position for the current sample write. + */ + @RawField + Pointer getStartPos(); + + /** + * Sets the start position for the current sample write. + */ + @RawField + void setStartPos(Pointer value); + + /** + * Gets the current position of the sample write. This position is moved forward as data is + * written for a sample. + */ + @RawField + Pointer getCurrentPos(); + + /** + * Sets the current position of the sample write. + */ + @RawField + void setCurrentPos(Pointer value); + + /** + * Returns the position where the buffer ends. Writing of data cannot exceed this position. + */ + @RawField + Pointer getEndPos(); + + /** + * Sets the position where the buffer ends. + */ + @RawField + void setEndPos(Pointer value); +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java new file mode 100644 index 000000000000..97ed3f3165e5 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import com.oracle.svm.core.util.VMError; +import org.graalvm.compiler.nodes.PauseNode; +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.jdk.UninterruptibleUtils; + +/** + * The custom implementation of spin lock that is async signal safe. + * + * In some specific situations, the signal handler can interrupt execution while the same thread + * already has the lock. Other spin lock implementations can deadlock in such a case. So it is + * essential to check if the current thread is the owner of the lock, before acquiring it. + */ +class SamplerSpinLock { + private final UninterruptibleUtils.AtomicPointer owner; + + @Platforms(Platform.HOSTED_ONLY.class) + SamplerSpinLock() { + this.owner = new UninterruptibleUtils.AtomicPointer<>(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public boolean isOwner() { + return owner.get().equal(CurrentIsolate.getCurrentThread()); + } + + @Uninterruptible(reason = "This method does not do a transition, so the whole critical section must be uninterruptible.", callerMustBe = true) + public void lock() { + VMError.guarantee(!isOwner(), "The current thread already has the lock!"); + IsolateThread currentThread = CurrentIsolate.getCurrentThread(); + while (!owner.compareAndSet(WordFactory.nullPointer(), currentThread)) { + PauseNode.pause(); + } + } + + @Uninterruptible(reason = "The whole critical section must be uninterruptible.", callerMustBe = true) + public void unlock() { + VMError.guarantee(isOwner(), "The current thread doesn't have the lock!"); + owner.compareAndSet(CurrentIsolate.getCurrentThread(), WordFactory.nullPointer()); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java new file mode 100644 index 000000000000..3998a5b550d4 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerStackWalkVisitor.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.word.Pointer; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.deopt.DeoptimizedFrame; +import com.oracle.svm.core.stack.ParameterizedStackFrameVisitor; + +final class SamplerStackWalkVisitor extends ParameterizedStackFrameVisitor { + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected boolean visitFrame(Pointer sp, CodePointer ip, CodeInfo codeInfo, DeoptimizedFrame deoptimizedFrame, Void voidData) { + return SamplerSampleWriter.putLong(SamplerThreadLocal.getWriterData(), ip.rawValue()); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected boolean unknownFrame(Pointer sp, CodePointer ip, DeoptimizedFrame deoptimizedFrame, Void data) { + SamplerThreadLocal.increaseUnparseableStacks(); + return false; + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java new file mode 100644 index 000000000000..66c83f47ab54 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; + +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.jfr.SubstrateJVM; +import com.oracle.svm.core.thread.ThreadListener; +import com.oracle.svm.core.threadlocal.FastThreadLocalFactory; +import com.oracle.svm.core.threadlocal.FastThreadLocalLong; +import com.oracle.svm.core.threadlocal.FastThreadLocalWord; + +class SamplerThreadLocal implements ThreadListener { + + private static final FastThreadLocalWord localBuffer = FastThreadLocalFactory.createWord("SamplerThreadLocal.localBuffer"); + private static final FastThreadLocalLong missedSamples = FastThreadLocalFactory.createLong("SamplerThreadLocal.missedSamples"); + private static final FastThreadLocalLong unparseableStacks = FastThreadLocalFactory.createLong("SamplerThreadLocal.unparseableStacks"); + /** + * The data that we are using during the stack walk, allocated on the stack. + */ + private static final FastThreadLocalWord writerData = FastThreadLocalFactory.createWord("SamplerThreadLocal.writerData"); + + @Override + @Uninterruptible(reason = "Only uninterruptible code may be executed before Thread.run.") + public void beforeThreadRun(IsolateThread isolateThread, Thread javaThread) { + if (SubstrateSigprofHandler.isProfilingEnabled()) { + initialize(isolateThread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static void initialize(IsolateThread isolateThread) { + if (SamplerIsolateLocal.isKeySet()) { + /* Adjust the number of buffers. */ + SamplerBufferPool.adjustBufferCount(); + + /* + * Save isolate thread in thread-local area. + * + * Once this value is set, the signal handler may interrupt this thread at any time. So, + * it is essential that this value is set at the very end of this method. + */ + UnsignedWord key = SamplerIsolateLocal.getKey(); + SubstrateSigprofHandler.singleton().setThreadLocalKeyValue(key, isolateThread); + } + } + + @Override + @Uninterruptible(reason = "Only uninterruptible code may be executed after Thread.exit.") + public void afterThreadExit(IsolateThread isolateThread, Thread javaThread) { + if (SubstrateSigprofHandler.isProfilingEnabled() && SamplerIsolateLocal.isKeySet()) { + teardown(isolateThread); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static void teardown(IsolateThread isolateThread) { + /* + * Invalidate thread-local area. + * + * Once this value is set to null, the signal handler can't interrupt this thread anymore. + * So, it is essential that this value is set at the very beginning of this method i.e. + * before doing cleanup. + */ + UnsignedWord key = SamplerIsolateLocal.getKey(); + SubstrateSigprofHandler.singleton().setThreadLocalKeyValue(key, WordFactory.nullPointer()); + + /* Adjust the number of buffers (including the thread-local buffer). */ + SamplerBuffer threadLocalBuffer = localBuffer.get(isolateThread); + SamplerBufferPool.adjustBufferCount(threadLocalBuffer); + localBuffer.set(isolateThread, WordFactory.nullPointer()); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static SamplerBuffer getThreadLocalBuffer() { + return localBuffer.get(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void setThreadLocalBuffer(SamplerBuffer buffer) { + buffer.setOwner(SubstrateJVM.get().getThreadId(CurrentIsolate.getCurrentThread())); + localBuffer.set(buffer); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void increaseMissedSamples() { + missedSamples.set(getMissedSamples() + 1); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long getMissedSamples() { + return missedSamples.get(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void increaseUnparseableStacks() { + unparseableStacks.set(getUnparseableStacks() + 1); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static long getUnparseableStacks() { + return unparseableStacks.get(); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static void setWriterData(SamplerSampleWriterData data) { + writerData.set(data); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static SamplerSampleWriterData getWriterData() { + return writerData.get(); + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java new file mode 100644 index 000000000000..09daa64dfd85 --- /dev/null +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package com.oracle.svm.core.sampler; + +import java.lang.management.ManagementFactory; +import java.util.Arrays; +import java.util.List; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.options.Option; +import org.graalvm.nativeimage.ImageSingletons; +import org.graalvm.nativeimage.Isolate; +import org.graalvm.nativeimage.IsolateThread; +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CodePointer; +import org.graalvm.nativeimage.hosted.Feature; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.IsolateListenerSupport; +import com.oracle.svm.core.RegisterDumper; +import com.oracle.svm.core.VMInspectionOptions; +import com.oracle.svm.core.annotate.AutomaticFeature; +import com.oracle.svm.core.annotate.Uninterruptible; +import com.oracle.svm.core.code.CodeInfo; +import com.oracle.svm.core.code.CodeInfoAccess; +import com.oracle.svm.core.code.CodeInfoTable; +import com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode; +import com.oracle.svm.core.graal.nodes.WriteHeapBaseNode; +import com.oracle.svm.core.heap.VMOperationInfos; +import com.oracle.svm.core.jdk.RuntimeSupport; +import com.oracle.svm.core.jdk.management.ManagementFeature; +import com.oracle.svm.core.jdk.management.SubstrateThreadMXBean; +import com.oracle.svm.core.jfr.JfrFeature; +import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.stack.JavaFrameAnchor; +import com.oracle.svm.core.stack.JavaFrameAnchors; +import com.oracle.svm.core.stack.JavaStackWalker; +import com.oracle.svm.core.thread.JavaVMOperation; +import com.oracle.svm.core.thread.ThreadListenerFeature; +import com.oracle.svm.core.thread.ThreadListenerSupport; +import com.oracle.svm.core.thread.VMThreads; + +@AutomaticFeature +@SuppressWarnings("unused") +class SubstrateSigprofHandlerFeature implements Feature { + + @Override + public boolean isInConfiguration(IsInConfigurationAccess access) { + return VMInspectionOptions.AllowVMInspection.getValue() && SubstrateSigprofHandler.isProfilingSupported(); + } + + @Override + public List> getRequiredFeatures() { + return Arrays.asList(ThreadListenerFeature.class, JfrFeature.class, ManagementFeature.class); + } + + @Override + public void beforeAnalysis(BeforeAnalysisAccess access) { + if (!ImageSingletons.contains(SubstrateSigprofHandler.class)) { + /* No sigprof handler. */ + return; + } + + /* Create stack visitor. */ + ImageSingletons.add(SamplerStackWalkVisitor.class, new SamplerStackWalkVisitor()); + + /* Add listeners. */ + ThreadListenerSupport.get().register(new SamplerThreadLocal()); + IsolateListenerSupport.singleton().register(new SamplerIsolateLocal()); + + /* Add startup hook. */ + RuntimeSupport.getRuntimeSupport().addStartupHook(new SubstrateSigprofHandlerStartupHook()); + } +} + +final class SubstrateSigprofHandlerStartupHook implements RuntimeSupport.Hook { + @Override + public void execute(boolean isFirstIsolate) { + if (isFirstIsolate) { + SubstrateSigprofHandler.singleton().install(); + } + } +} + +/** + *

+ * The core class of low overhead asynchronous sampling based profiler. + * {@link SubstrateSigprofHandler} handles the periodic signal generated by the OS with a given + * time-frequency. The asynchronous nature of the signal means that the OS could invoke the signal + * handler at any time (could be during GC, VM operation, uninterruptible code) and that the signal + * handler can only execute specific code i.e. the calls that are asynchronous signal safe. + *

+ * + *

+ * The signal handler is divided into three part: restore isolate, isolate-thread, stack and + * instruction pointers, prepare everything necessary for stack walk, do a stack walk and write IPs + * into buffer. + *

+ * + *

+ * The signal handler is as a producer. On the other side of relation is + * {@link com.oracle.svm.core.jfr.JfrRecorderThread} that is consumer. The + * {@link SamplerBuffer} that we are using in this consumer-producer communication is allocated + * eagerly, in a part of the heap that is not accessible via GC, and there will always be more + * available buffers that threads. + *

+ * + *

+ * The communication between consumer and producer goes as follows: + *

    + *
  • Signal handler (producer): pops the buffer from the pool of available buffers (the buffer now + * becomes thread-local), writes the IPs into buffer, if the buffer is full and moves it to a pool + * with buffers that awaits processing.
    • + *
  • Recorder thread (consumer): pops the buffer from the pool of full buffers, reconstructs the + * stack walk based on IPs and pushes the buffer into pool of available buffers.
    • + *
+ * NOTE: The producer and the consumer are always accessing different buffers. + *

+ * + *

+ * In some rare cases, the profiling is impossible e.g. no available buffers in the pool, unknown IP + * during stack walk, the thread holds the pool's lock when the signal arrives, etc. + *

+ * + * @see SamplerSpinLock + * @see SamplerBufferStack + */ +public abstract class SubstrateSigprofHandler { + + public static class Options { + @Option(help = "Allow sampling-based profiling. Default: disabled in execution.")// + static final RuntimeOptionKey SamplingBasedProfiling = new RuntimeOptionKey<>(Boolean.FALSE); + } + + private boolean enabled; + private final SamplerBufferStack availableBuffers; + private final SamplerBufferStack fullBuffers; + private SubstrateThreadMXBean threadMXBean; + + @Platforms(Platform.HOSTED_ONLY.class) + protected SubstrateSigprofHandler() { + this.availableBuffers = new SamplerBufferStack(); + this.fullBuffers = new SamplerBufferStack(); + } + + @Fold + static SubstrateSigprofHandler singleton() { + return ImageSingletons.lookup(SubstrateSigprofHandler.class); + } + + @Fold + static SamplerStackWalkVisitor visitor() { + return ImageSingletons.lookup(SamplerStackWalkVisitor.class); + } + + @Fold + static boolean isProfilingSupported() { + return Platform.includedIn(Platform.LINUX.class); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static boolean isProfilingEnabled() { + return singleton().enabled; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static SamplerBufferStack availableBuffers() { + return singleton().availableBuffers; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static SamplerBufferStack fullBuffers() { + return singleton().fullBuffers; + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + static SubstrateThreadMXBean getSubstrateThreadMXBean() { + return singleton().threadMXBean; + } + + /** + * Installs the platform dependent sigprof handler. + */ + void install() { + if (Options.SamplingBasedProfiling.getValue()) { + threadMXBean = (SubstrateThreadMXBean) ManagementFactory.getThreadMXBean(); + /* Call VM operation to initialize the profiler and the threads. */ + InitializeProfilerOperation initializeProfilerOperation = new InitializeProfilerOperation(); + initializeProfilerOperation.enqueue(); + + /* After the VM operations finishes. Install handler and start profiling. */ + install0(); + } + } + + protected abstract void install0(); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract UnsignedWord createThreadLocalKey(); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract void deleteThreadLocalKey(UnsignedWord key); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract void setThreadLocalKeyValue(UnsignedWord key, IsolateThread value); + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + protected abstract IsolateThread getThreadLocalKeyValue(UnsignedWord key); + + /** Is sigprof handler called from native code? */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean isIPInJavaCode(RegisterDumper.Context uContext) { + Pointer ip = (Pointer) RegisterDumper.singleton().getIP(uContext); + CodeInfo codeInfo = CodeInfoTable.getImageCodeInfo(); + Pointer codeStart = (Pointer) CodeInfoAccess.getCodeStart(codeInfo); + UnsignedWord codeSize = CodeInfoAccess.getCodeSize(codeInfo); + return ip.aboveOrEqual(codeStart) && ip.belowOrEqual(codeStart.add(codeSize)); + } + + /** + * Walk the current stacktrace using isolate thread local area and isolate heap base. + */ + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + private static void doUninterruptibleStackWalk(RegisterDumper.Context uContext, boolean isIPInJavaCode) { + CodePointer ip; + Pointer sp; + if (isIPInJavaCode) { + ip = (CodePointer) RegisterDumper.singleton().getIP(uContext); + sp = (Pointer) RegisterDumper.singleton().getSP(uContext); + } else { + JavaFrameAnchor anchor = JavaFrameAnchors.getFrameAnchor(); + if (anchor.isNull()) { + /* + * The anchor is still null if the function is interrupted during prologue. See: + * com.oracle.svm.core.graal.snippets.CFunctionSnippets.prologueSnippet + */ + return; + } + + ip = anchor.getLastJavaIP(); + sp = anchor.getLastJavaSP(); + + if (ip.isNull() || sp.isNull()) { + /* + * It can happen that anchor is in list of all anchors, but its IP and SP are not + * filled yet. + */ + return; + } + } + + /* Initialize stack walk. */ + SamplerSampleWriterData data = StackValue.get(SamplerSampleWriterData.class); + if (prepareStackWalk(data)) { + /* Walk the stack. */ + if (JavaStackWalker.walkCurrentThread(sp, ip, visitor())) { + SamplerSampleWriter.commit(data); + } + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static boolean prepareStackWalk(SamplerSampleWriterData data) { + if (availableBuffers().isLockedByCurrentThread() || fullBuffers().isLockedByCurrentThread()) { + /* + * The current thread already holds the stack lock, so we can't access it. It's way + * better to lose one sample, then potentially the whole buffer. + */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + + SamplerBuffer buffer = SamplerThreadLocal.getThreadLocalBuffer(); + if (buffer.isNull()) { + /* Pop first free buffer from the pool. */ + buffer = availableBuffers().popBuffer(); + if (buffer.isNull()) { + /* No available buffers on the pool. Fallback! */ + SamplerThreadLocal.increaseMissedSamples(); + return false; + } + SamplerThreadLocal.setThreadLocalBuffer(buffer); + } + + /* Initialize the buffer. */ + data.setSamplerBuffer(buffer); + data.setStartPos(buffer.getPos()); + data.setCurrentPos(buffer.getPos()); + data.setEndPos(SamplerBufferAccess.getDataEnd(buffer)); + SamplerThreadLocal.setWriterData(data); + return true; + } + + /** Called from the platform dependent sigprof handler to enter isolate. */ + @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) + protected static void tryEnterIsolateAndDoWalk(RegisterDumper.Context uContext) { + if (!SamplerIsolateLocal.isKeySet()) { + /* The key is set for initial isolate only. */ + return; + } + + Isolate isolate = SamplerIsolateLocal.getIsolate(); + if (isolate.isNull()) { + /* It may happen that the initial isolate exited. */ + return; + } + + /* Write isolate pointer (heap base) into register. */ + WriteHeapBaseNode.writeCurrentVMHeapBase(isolate); + + /* We are keeping reference to isolate thread inside OS thread local area. */ + UnsignedWord key = SamplerIsolateLocal.getKey(); + IsolateThread thread = singleton().getThreadLocalKeyValue(key); + if (thread.isNull()) { + /* Thread is not yet initialized or already detached from isolate. */ + return; + } + + /* Write isolate thread pointer into register. */ + WriteCurrentVMThreadNode.writeCurrentVMThread(thread); + + /* Check if the instruction pointer (IP) is inside native or java code. */ + boolean isIPInJavaCode = isIPInJavaCode(uContext); + + /* Walk the stack. */ + doUninterruptibleStackWalk(uContext, isIPInJavaCode); + } + + private class InitializeProfilerOperation extends JavaVMOperation { + + protected InitializeProfilerOperation() { + super(VMOperationInfos.get(InitializeProfilerOperation.class, "Initialize Profiler", SystemEffect.SAFEPOINT)); + } + + @Override + protected void operate() { + initialize(); + } + + /** + * We need to ensure that all threads are properly initialized at a moment when we start a + * profiling. + */ + @Uninterruptible(reason = "Prevent pollution of the current thread's thread local JFR buffer.") + private void initialize() { + enabled = true; + /* + * Iterate all over all thread and initialize the thread-local storage of each thread. + */ + for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) { + SamplerThreadLocal.initialize(thread); + } + } + } +} diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java index 2436aaf19898..a0283ea6b14b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/ThreadListenerSupport.java @@ -65,7 +65,7 @@ public void beforeThreadStart(IsolateThread isolateThread, Thread javaThread) { @Uninterruptible(reason = "Force that all listeners are uninterruptible.") public void afterThreadExit(IsolateThread isolateThread, Thread javaThread) { - for (int i = 0; i < listeners.length; i++) { + for (int i = listeners.length - 1; i >= 0; i--) { listeners[i].afterThreadExit(isolateThread, javaThread); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java index fd596ed88cca..6a28ba4f18ef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/VMThreads.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.thread; +import com.oracle.svm.core.IsolateListenerSupport; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.replacements.ReplacementsUtil; @@ -450,6 +451,7 @@ public void tearDown() { VMOperationControl.shutdownAndDetachVMOperationThread(); } // At this point, it is guaranteed that all other threads were detached. + IsolateListenerSupport.singleton().onIsolateTeardown(); waitUntilLastOsThreadExited(); } From b7af1f99691c9f5cafde270d7567b5b72ff1bd6c Mon Sep 17 00:00:00 2001 From: jovanstevanovic Date: Thu, 12 May 2022 09:18:23 +0200 Subject: [PATCH 2/3] Changes according to comments. --- .../svm/core/sampler/SamplerBuffer.java | 4 +- .../svm/core/sampler/SamplerBufferPool.java | 81 +++++++++---------- .../svm/core/sampler/SamplerBufferStack.java | 7 +- .../svm/core/sampler/SamplerIsolateLocal.java | 4 +- .../svm/core/sampler/SamplerSampleWriter.java | 9 ++- .../svm/core/sampler/SamplerSpinLock.java | 7 +- .../svm/core/sampler/SamplerThreadLocal.java | 6 +- .../core/sampler/SubstrateSigprofHandler.java | 22 ++--- 8 files changed, 70 insertions(+), 70 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java index 5e5f95e17776..d7c1dfa20c2e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java @@ -39,13 +39,13 @@ interface SamplerBuffer extends PointerBase { /** - * Returns a buffer that is next in the {@link SamplerBufferStack}, otherwise null. + * Returns the buffer that is next in the {@link SamplerBufferStack}, otherwise null. */ @RawField SamplerBuffer getNext(); /** - * Sets a buffer as a new head in the {@link SamplerBufferStack}. + * Sets the successor to this node in the {@link SamplerBufferStack}. */ @RawField void setNext(SamplerBuffer buffer); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java index 58243c0d8dba..e50ebc6533b4 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -41,61 +41,48 @@ class SamplerBufferPool { private static final long THREAD_BUFFER_SIZE = Options.getThreadBufferSize(); - private static final VMMutex mutex = new VMMutex("profilerBufferUtils"); + private static final VMMutex mutex = new VMMutex("SamplerBufferPool"); private static long bufferCount; @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) - public static void adjustBufferCount(SamplerBuffer threadLocalBuffer) { - mutex.lockNoTransition(); - try { - releaseThreadLocalBuffer(threadLocalBuffer); - adjustBufferCount0(); - } finally { - mutex.unlock(); - } + public static void releaseBufferAndAdjustCount(SamplerBuffer threadLocalBuffer) { + adjustBufferCount0(threadLocalBuffer); } @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) public static void adjustBufferCount() { - mutex.lockNoTransition(); - try { - adjustBufferCount0(); - } finally { - mutex.unlock(); - } + adjustBufferCount0(WordFactory.nullPointer()); } @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) - public static void adjustBufferCount0() { - long diff = diff(); - if (diff > 0) { - for (int i = 0; i < diff; i++) { - allocateAndPush(); - } - } else { - for (long i = diff; i < 0; i++) { - if (!popAndFree()) { - break; + private static void adjustBufferCount0(SamplerBuffer threadLocalBuffer) { + mutex.lockNoTransition(); + try { + releaseThreadLocalBuffer(threadLocalBuffer); + long diff = diff(); + if (diff > 0) { + for (int i = 0; i < diff; i++) { + allocateAndPush(); + } + } else { + for (long i = diff; i < 0; i++) { + if (!popAndFree()) { + break; + } } } + } finally { + mutex.unlock(); } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static void allocateAndPush() { - VMError.guarantee(bufferCount >= 0); - SamplerBuffer buffer = SamplerBufferAccess.allocate(WordFactory.unsigned(THREAD_BUFFER_SIZE)); - if (buffer.isNull()) { - return; - } - SubstrateSigprofHandler.availableBuffers().pushBuffer(buffer); - bufferCount++; - } - @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.", mayBeInlined = true) private static void releaseThreadLocalBuffer(SamplerBuffer buffer) { - /* buffer will be null if no stack walk were performed. */ + /* + * buffer is null if the thread is not running yet, or we did not perform the stack walk for + * this thread during the run. + */ if (buffer.isNonNull()) { if (SamplerBufferAccess.isEmpty(buffer)) { /* We can free it right away. */ @@ -103,28 +90,40 @@ private static void releaseThreadLocalBuffer(SamplerBuffer buffer) { } else { /* Put it in the stack with other unprocessed buffers. */ buffer.setFreeable(true); - SubstrateSigprofHandler.fullBuffers().pushBuffer(buffer); + SubstrateSigprofHandler.singleton().fullBuffers().pushBuffer(buffer); } VMError.guarantee(bufferCount > 0); bufferCount--; } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private static void allocateAndPush() { + VMError.guarantee(bufferCount >= 0); + SamplerBuffer buffer = SamplerBufferAccess.allocate(WordFactory.unsigned(THREAD_BUFFER_SIZE)); + if (buffer.isNull()) { + return; + } + SubstrateSigprofHandler.singleton().availableBuffers().pushBuffer(buffer); + bufferCount++; + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean popAndFree() { VMError.guarantee(bufferCount > 0); - SamplerBuffer buffer = SubstrateSigprofHandler.availableBuffers().popBuffer(); + SamplerBuffer buffer = SubstrateSigprofHandler.singleton().availableBuffers().popBuffer(); if (buffer.isNonNull()) { SamplerBufferAccess.free(buffer); bufferCount--; + return true; + } else { return false; } - return true; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static long diff() { - double diffD = SubstrateSigprofHandler.getSubstrateThreadMXBean().getThreadCount() * 1.5 - bufferCount; + double diffD = SubstrateSigprofHandler.singleton().substrateThreadMXBean().getThreadCount() * 1.5 - bufferCount; return (long) (diffD + 0.5); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java index db7a82e44577..229e942b6892 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferStack.java @@ -50,7 +50,7 @@ class SamplerBufferStack { } /** - * Push a profiler buffer into the global linked list. + * Push the buffer into the linked-list. */ @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") public void pushBuffer(SamplerBuffer buffer) { @@ -64,7 +64,7 @@ public void pushBuffer(SamplerBuffer buffer) { } /** - * Pop a profiler buffer from the global linked list. Returns {@code null} if the list is empty. + * Pop the buffer from the linked-list. Returns {@code null} if the list is empty. */ @Uninterruptible(reason = "Locking without transition requires that the whole critical section is uninterruptible.") public SamplerBuffer popBuffer() { @@ -81,9 +81,6 @@ public SamplerBuffer popBuffer() { } } - /** - * Returns the lock that this stack is using. - */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isLockedByCurrentThread() { return spinLock.isOwner(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java index 28b00171363f..bd25d3101b71 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerIsolateLocal.java @@ -58,12 +58,12 @@ public void afterCreateIsolate(Isolate isolate) { @Override @Uninterruptible(reason = "The isolate teardown is in progress.") public void onIsolateTeardown() { - if (SubstrateSigprofHandler.isProfilingEnabled() && isKeySet()) { + if (SubstrateSigprofHandler.singleton().isProfilingEnabled() && isKeySet()) { /* Invalidate the isolate-specific key. */ UnsignedWord oldKey = key; key = WordFactory.unsigned(0); - /* We need to manually call teardown because isKeySet will now return false. */ + /* Manually disable sampling for the current thread (no other threads are remaining). */ SamplerThreadLocal.teardown(CurrentIsolate.getCurrentThread()); /* Now, it's safe to delete the isolate-specific key. */ diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java index 89802833d164..7290fea6c6c6 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSampleWriter.java @@ -32,11 +32,14 @@ import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.util.VMError; -class SamplerSampleWriter { +final class SamplerSampleWriter { private static final int END_MARKER_SIZE = Long.BYTES; private static final long END_MARKER = -1; + private SamplerSampleWriter() { + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean putLong(SamplerSampleWriterData data, long value) { if (ensureSize(data, Long.BYTES)) { @@ -86,7 +89,7 @@ private static boolean accommodate(SamplerSampleWriterData data, UnsignedWord un } /* Pop first free buffer from the pool. */ - SamplerBuffer newBuffer = SubstrateSigprofHandler.availableBuffers().popBuffer(); + SamplerBuffer newBuffer = SubstrateSigprofHandler.singleton().availableBuffers().popBuffer(); if (newBuffer.isNull()) { /* No available buffers on the pool. Fallback! */ SamplerThreadLocal.increaseMissedSamples(); @@ -99,7 +102,7 @@ private static boolean accommodate(SamplerSampleWriterData data, UnsignedWord un /* Put in the stack with other unprocessed buffers. */ SamplerBuffer oldBuffer = data.getSamplerBuffer(); - SubstrateSigprofHandler.fullBuffers().pushBuffer(oldBuffer); + SubstrateSigprofHandler.singleton().fullBuffers().pushBuffer(oldBuffer); /* Reinitialize data structure. */ data.setSamplerBuffer(newBuffer); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java index 97ed3f3165e5..85dc89e26577 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerSpinLock.java @@ -25,7 +25,6 @@ package com.oracle.svm.core.sampler; -import com.oracle.svm.core.util.VMError; import org.graalvm.compiler.nodes.PauseNode; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.IsolateThread; @@ -35,13 +34,15 @@ import com.oracle.svm.core.annotate.Uninterruptible; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.util.VMError; /** * The custom implementation of spin lock that is async signal safe. * * In some specific situations, the signal handler can interrupt execution while the same thread - * already has the lock. Other spin lock implementations can deadlock in such a case. So it is - * essential to check if the current thread is the owner of the lock, before acquiring it. + * already has the lock. This implementation will check and fatally fail while other spin locks + * implementations can deadlock in this case. So it is essential to check if the current thread is + * the owner of the lock, before acquiring it. */ class SamplerSpinLock { private final UninterruptibleUtils.AtomicPointer owner; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java index 66c83f47ab54..6cd132cda1fe 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerThreadLocal.java @@ -50,7 +50,7 @@ class SamplerThreadLocal implements ThreadListener { @Override @Uninterruptible(reason = "Only uninterruptible code may be executed before Thread.run.") public void beforeThreadRun(IsolateThread isolateThread, Thread javaThread) { - if (SubstrateSigprofHandler.isProfilingEnabled()) { + if (SubstrateSigprofHandler.singleton().isProfilingEnabled()) { initialize(isolateThread); } } @@ -75,7 +75,7 @@ static void initialize(IsolateThread isolateThread) { @Override @Uninterruptible(reason = "Only uninterruptible code may be executed after Thread.exit.") public void afterThreadExit(IsolateThread isolateThread, Thread javaThread) { - if (SubstrateSigprofHandler.isProfilingEnabled() && SamplerIsolateLocal.isKeySet()) { + if (SubstrateSigprofHandler.singleton().isProfilingEnabled() && SamplerIsolateLocal.isKeySet()) { teardown(isolateThread); } } @@ -94,7 +94,7 @@ static void teardown(IsolateThread isolateThread) { /* Adjust the number of buffers (including the thread-local buffer). */ SamplerBuffer threadLocalBuffer = localBuffer.get(isolateThread); - SamplerBufferPool.adjustBufferCount(threadLocalBuffer); + SamplerBufferPool.releaseBufferAndAdjustCount(threadLocalBuffer); localBuffer.set(isolateThread, WordFactory.nullPointer()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java index 09daa64dfd85..08a0d4b72b36 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java @@ -185,23 +185,23 @@ static boolean isProfilingSupported() { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static boolean isProfilingEnabled() { - return singleton().enabled; + boolean isProfilingEnabled() { + return enabled; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static SamplerBufferStack availableBuffers() { - return singleton().availableBuffers; + SamplerBufferStack availableBuffers() { + return availableBuffers; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static SamplerBufferStack fullBuffers() { - return singleton().fullBuffers; + SamplerBufferStack fullBuffers() { + return fullBuffers; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - static SubstrateThreadMXBean getSubstrateThreadMXBean() { - return singleton().threadMXBean; + SubstrateThreadMXBean substrateThreadMXBean() { + return threadMXBean; } /** @@ -287,7 +287,7 @@ private static void doUninterruptibleStackWalk(RegisterDumper.Context uContext, @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean prepareStackWalk(SamplerSampleWriterData data) { - if (availableBuffers().isLockedByCurrentThread() || fullBuffers().isLockedByCurrentThread()) { + if (singleton().availableBuffers().isLockedByCurrentThread() || singleton().fullBuffers().isLockedByCurrentThread()) { /* * The current thread already holds the stack lock, so we can't access it. It's way * better to lose one sample, then potentially the whole buffer. @@ -299,7 +299,7 @@ private static boolean prepareStackWalk(SamplerSampleWriterData data) { SamplerBuffer buffer = SamplerThreadLocal.getThreadLocalBuffer(); if (buffer.isNull()) { /* Pop first free buffer from the pool. */ - buffer = availableBuffers().popBuffer(); + buffer = singleton().availableBuffers().popBuffer(); if (buffer.isNull()) { /* No available buffers on the pool. Fallback! */ SamplerThreadLocal.increaseMissedSamples(); @@ -369,13 +369,13 @@ protected void operate() { */ @Uninterruptible(reason = "Prevent pollution of the current thread's thread local JFR buffer.") private void initialize() { - enabled = true; /* * Iterate all over all thread and initialize the thread-local storage of each thread. */ for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) { SamplerThreadLocal.initialize(thread); } + enabled = true; } } } From c6a7989ade3650b4265c48a6669f7a50c7999d87 Mon Sep 17 00:00:00 2001 From: jovanstevanovic Date: Thu, 12 May 2022 13:23:48 +0200 Subject: [PATCH 3/3] Small fixes. --- .../oracle/svm/core/sampler/SamplerBuffer.java | 2 +- .../svm/core/sampler/SamplerBufferPool.java | 16 ++++++++++------ .../core/sampler/SubstrateSigprofHandler.java | 13 ++++--------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java index d7c1dfa20c2e..5f1f57f15cbc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBuffer.java @@ -45,7 +45,7 @@ interface SamplerBuffer extends PointerBase { SamplerBuffer getNext(); /** - * Sets the successor to this node in the {@link SamplerBufferStack}. + * Sets the successor to this buffer in the {@link SamplerBufferStack}. */ @RawField void setNext(SamplerBuffer buffer); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java index e50ebc6533b4..96af063d0e6b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SamplerBufferPool.java @@ -63,7 +63,9 @@ private static void adjustBufferCount0(SamplerBuffer threadLocalBuffer) { long diff = diff(); if (diff > 0) { for (int i = 0; i < diff; i++) { - allocateAndPush(); + if (!allocateAndPush()) { + break; + } } } else { for (long i = diff; i < 0; i++) { @@ -98,14 +100,16 @@ private static void releaseThreadLocalBuffer(SamplerBuffer buffer) { } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - private static void allocateAndPush() { + private static boolean allocateAndPush() { VMError.guarantee(bufferCount >= 0); SamplerBuffer buffer = SamplerBufferAccess.allocate(WordFactory.unsigned(THREAD_BUFFER_SIZE)); - if (buffer.isNull()) { - return; + if (buffer.isNonNull()) { + SubstrateSigprofHandler.singleton().availableBuffers().pushBuffer(buffer); + bufferCount++; + return true; + } else { + return false; } - SubstrateSigprofHandler.singleton().availableBuffers().pushBuffer(buffer); - bufferCount++; } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java index 08a0d4b72b36..b3751bce98dc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java @@ -233,7 +233,6 @@ void install() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract IsolateThread getThreadLocalKeyValue(UnsignedWord key); - /** Is sigprof handler called from native code? */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isIPInJavaCode(RegisterDumper.Context uContext) { Pointer ip = (Pointer) RegisterDumper.singleton().getIP(uContext); @@ -243,9 +242,6 @@ private static boolean isIPInJavaCode(RegisterDumper.Context uContext) { return ip.aboveOrEqual(codeStart) && ip.belowOrEqual(codeStart.add(codeSize)); } - /** - * Walk the current stacktrace using isolate thread local area and isolate heap base. - */ @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) private static void doUninterruptibleStackWalk(RegisterDumper.Context uContext, boolean isIPInJavaCode) { CodePointer ip; @@ -320,11 +316,6 @@ private static boolean prepareStackWalk(SamplerSampleWriterData data) { /** Called from the platform dependent sigprof handler to enter isolate. */ @Uninterruptible(reason = "The method executes during signal handling.", callerMustBe = true) protected static void tryEnterIsolateAndDoWalk(RegisterDumper.Context uContext) { - if (!SamplerIsolateLocal.isKeySet()) { - /* The key is set for initial isolate only. */ - return; - } - Isolate isolate = SamplerIsolateLocal.getIsolate(); if (isolate.isNull()) { /* It may happen that the initial isolate exited. */ @@ -335,6 +326,10 @@ protected static void tryEnterIsolateAndDoWalk(RegisterDumper.Context uContext) WriteHeapBaseNode.writeCurrentVMHeapBase(isolate); /* We are keeping reference to isolate thread inside OS thread local area. */ + if (!SamplerIsolateLocal.isKeySet()) { + /* The key becomes invalid during initial isolate teardown. */ + return; + } UnsignedWord key = SamplerIsolateLocal.getKey(); IsolateThread thread = singleton().getThreadLocalKeyValue(key); if (thread.isNull()) {