From 541d7d110f77ff8e0761c44de45d621790c32e85 Mon Sep 17 00:00:00 2001 From: Yudi Zheng Date: Mon, 22 Jul 2024 17:46:08 +0200 Subject: [PATCH] Adapt fibers changes: use JavaThread::_lock_id as monitor owner. --- .../hotspot/GraalHotSpotVMConfig.java | 4 + .../meta/HotSpotGraphBuilderPlugins.java | 17 +++++ .../replacements/HotSpotReplacementsUtil.java | 25 ++++++ .../hotspot/replacements/MonitorSnippets.java | 76 +++++++++++++------ .../Target_java_lang_VirtualThread.java | 34 ++++----- 5 files changed, 116 insertions(+), 40 deletions(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index 479041187184..7fd6e69ffca3 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -276,6 +276,8 @@ public final int arrayOopDescLengthOffset() { public final int threadCarrierThreadObjectOffset = getFieldOffset("JavaThread::_threadObj", Integer.class, "OopHandle"); public final int threadScopedValueCacheOffset = getFieldOffset("JavaThread::_scopedValueCache", Integer.class, "OopHandle"); + public final int javaThreadLockIDOffset = getFieldOffset("JavaThread::_lock_id", Integer.class, "int64_t", -1, JDK >= 24); + public final int threadIsInVTMSTransitionOffset = getFieldOffset("JavaThread::_is_in_VTMS_transition", Integer.class, "bool"); public final int threadIsInTmpVTMSTransitionOffset = getFieldOffset("JavaThread::_is_in_tmp_VTMS_transition", Integer.class, "bool"); public final int threadIsDisableSuspendOffset = getFieldOffset("JavaThread::_is_disable_suspend", Integer.class, "bool", -1, JDK >= 22); @@ -351,6 +353,7 @@ public int threadLastJavaFpOffset() { public final int ageMaskInPlace = getConstant("markWord::age_mask_in_place", Integer.class); public final int unlockedMask = getConstant("markWord::unlocked_value", Integer.class); public final int monitorMask = getConstant("markWord::monitor_value", Integer.class); + public final int unusedMark = getConstant("markWord::marked_value", Integer.class); // This field has no type in vmStructs.cpp public final int objectMonitorOwner = getFieldOffset("ObjectMonitor::_owner", Integer.class, null); @@ -358,6 +361,7 @@ public int threadLastJavaFpOffset() { public final int objectMonitorCxq = getFieldOffset("ObjectMonitor::_cxq", Integer.class, "ObjectWaiter*"); public final int objectMonitorEntryList = getFieldOffset("ObjectMonitor::_EntryList", Integer.class, "ObjectWaiter*"); public final int objectMonitorSucc = getFieldOffset("ObjectMonitor::_succ", Integer.class, "JavaThread*"); + public final int objectMonitorStackLocker = getFieldOffset("ObjectMonitor::_stack_locker", Integer.class, "BasicLock*"); public final long objectMonitorAnonymousOwner = getConstant("ObjectMonitor::ANONYMOUS_OWNER", Long.class, 1L, JDK >= 22); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java index 1e6e5172d398..9bf11beedc7a 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java @@ -40,6 +40,7 @@ import static jdk.graal.compiler.hotspot.HotSpotBackend.SHAREDRUNTIME_NOTIFY_JVMTI_VTHREAD_UNMOUNT; import static jdk.graal.compiler.hotspot.HotSpotBackend.UPDATE_BYTES_CRC32; import static jdk.graal.compiler.hotspot.HotSpotBackend.UPDATE_BYTES_CRC32C; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_ID_LOCATION; import static jdk.graal.compiler.java.BytecodeParserOptions.InlineDuringParsing; import static jdk.graal.compiler.nodes.ConstantNode.forBoolean; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; @@ -848,6 +849,22 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec } }); } + + if (JavaVersionUtil.JAVA_SPEC >= 24) { + r.register(new InvocationPlugin("setLockId", Receiver.class, long.class) { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode tid) { + try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { + ValueNode nonNullReceiver = receiver.get(true); + GraalError.guarantee(config.javaThreadLockIDOffset != -1, "JavaThread::_lock_id is not exported"); + CurrentJavaThreadNode javaThread = b.add(new CurrentJavaThreadNode(helper.getWordKind())); + OffsetAddressNode address = b.add(new OffsetAddressNode(javaThread, helper.asWord(config.javaThreadLockIDOffset))); + b.add(new JavaWriteNode(JavaKind.Long, address, JAVA_THREAD_LOCK_ID_LOCATION, tid, BarrierType.NONE, false)); + } + return true; + } + }); + } } private static ResolvedJavaType resolveTypeAESCrypt(ResolvedJavaType context) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index 821617be5cc9..e2277ab447e6 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -60,6 +60,7 @@ import jdk.graal.compiler.nodes.type.StampTool; import jdk.graal.compiler.replacements.ReplacementsUtil; import jdk.graal.compiler.replacements.nodes.ReadRegisterNode; +import jdk.graal.compiler.serviceprovider.JavaVersionUtil; import jdk.graal.compiler.word.Word; import jdk.vm.ci.code.Register; import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime; @@ -296,6 +297,8 @@ public static Object getPendingException(Word thread) { public static final LocationIdentity JAVA_THREAD_CARRIER_THREAD_OBJECT_LOCATION = NamedLocationIdentity.mutable("JavaThread::_threadObj"); + public static final LocationIdentity JAVA_THREAD_LOCK_ID_LOCATION = NamedLocationIdentity.mutable("JavaThread::_lock_id"); + public static final LocationIdentity JAVA_THREAD_OSTHREAD_LOCATION = NamedLocationIdentity.mutable("JavaThread::_osthread"); public static final LocationIdentity JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION = NamedLocationIdentity.mutable("JavaThread::_held_monitor_count"); @@ -353,6 +356,11 @@ public ValueNode canonicalizeRead(ValueNode read, ValueNode object, ValueNode lo } }; + @Fold + public static boolean isJDK21() { + return JavaVersionUtil.JAVA_SPEC == 21; + } + @Fold public static int allocatePrefetchStyle(@InjectedParameter GraalHotSpotVMConfig config) { return config.allocatePrefetchStyle; @@ -490,6 +498,11 @@ public static int monitorMask(@InjectedParameter GraalHotSpotVMConfig config) { return config.monitorMask; } + @Fold + public static int unusedMark(@InjectedParameter GraalHotSpotVMConfig config) { + return config.unusedMark; + } + @Fold public static int objectMonitorOwnerOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.objectMonitorOwner; @@ -520,6 +533,11 @@ public static int objectMonitorSuccOffset(@InjectedParameter GraalHotSpotVMConfi return config.objectMonitorSucc; } + @Fold + public static int objectMonitorStackLockerOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.objectMonitorStackLocker; + } + /** * Mask for a biasable, locked or unlocked mark word. It is the least significant 3 bits prior * to Java 18 (1 bit for biased locking and 2 bits for stack locking or heavy locking), and 2 @@ -665,6 +683,8 @@ public static int secondarySupersOffset(@InjectedParameter GraalHotSpotVMConfig public static final LocationIdentity OBJECT_MONITOR_SUCC_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_succ"); + public static final LocationIdentity OBJECT_MONITOR_STACK_LOCKER_LOCATION = NamedLocationIdentity.mutable("ObjectMonitor::_stack_locker"); + @Fold public static int lockDisplacedMarkOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.basicLockDisplacedHeaderOffset; @@ -675,6 +695,11 @@ static int heldMonitorCountOffset(@InjectedParameter GraalHotSpotVMConfig config return config.threadHeldMonitorCountOffset; } + @Fold + static int javaThreadLockIDOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.javaThreadLockIDOffset; + } + @Fold static int javaThreadLockStackTopOffset(@InjectedParameter GraalHotSpotVMConfig config) { return config.threadLockStackOffset + config.lockStackTopOffset; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java index c1cbaa2749b7..7602bce593ac 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java @@ -34,30 +34,41 @@ import static jdk.graal.compiler.hotspot.nodes.VMErrorNode.vmError; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_ID_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_STACK_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_LOCK_STACK_TOP_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_ACCESS_FLAGS_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_CXQ_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_ENTRY_LIST_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_STACK_LOCKER_LOCATION; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJECT_MONITOR_SUCC_LOCATION; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.heldMonitorCountOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.isCAssertEnabled; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.isJDK21; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadLockIDOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadLockStackEndOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.javaThreadLockStackTopOffset; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.jvmAccIsValueBasedClass; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.klassAccessFlagsOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadWordFromObject; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockDisplacedMarkOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.lockMaskInPlace; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.markOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.monitorMask; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorAnonymousOwner; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorCxqOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorEntryListOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorOwnerOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorRecursionsOffset; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorStackLockerOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.objectMonitorSuccOffset; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.pageSize; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.unlockedMask; +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.unusedMark; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.useLightweightLocking; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.useStackLocking; import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop; @@ -70,11 +81,13 @@ import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_LIKELY_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.VERY_FAST_PATH_PROBABILITY; +import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.VERY_SLOW_PATH_PROBABILITY; import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability; import static jdk.graal.compiler.nodes.extended.MembarNode.memoryBarrier; import static jdk.graal.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; import static jdk.graal.compiler.replacements.nodes.CStringConstant.cstring; import static org.graalvm.word.LocationIdentity.any; +import static org.graalvm.word.WordFactory.nullPointer; import static org.graalvm.word.WordFactory.unsigned; import static org.graalvm.word.WordFactory.zero; @@ -205,8 +218,7 @@ public static void monitorenter(Object object, KlassPointer hub, @ConstantParame incCounter(); if (HotSpotReplacementsUtil.diagnoseSyncOnValueBasedClasses(INJECTED_VMCONFIG)) { - if (hub.readWord(HotSpotReplacementsUtil.klassAccessFlagsOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.KLASS_ACCESS_FLAGS_LOCATION).and( - HotSpotReplacementsUtil.jvmAccIsValueBasedClass(INJECTED_VMCONFIG)).notEqual(0)) { + if (hub.readWord(klassAccessFlagsOffset(INJECTED_VMCONFIG), KLASS_ACCESS_FLAGS_LOCATION).and(jvmAccIsValueBasedClass(INJECTED_VMCONFIG)).notEqual(0)) { monitorenterStubC(MONITORENTER, object, lock); return; } @@ -234,18 +246,21 @@ private static boolean tryFastPathLocking(Object object, Register stackPointerRe } } - private static boolean tryEnterInflated(Object object, Word mark, Word thread, boolean trace, Counters counters) { + private static boolean tryEnterInflated(Object object, Word lock, Word mark, Word thread, boolean trace, Counters counters) { // mark is a pointer to the ObjectMonitor + monitorMask - Word monitor = mark.subtract(HotSpotReplacementsUtil.monitorMask(INJECTED_VMCONFIG)); - int ownerOffset = HotSpotReplacementsUtil.objectMonitorOwnerOffset(INJECTED_VMCONFIG); - Word owner = monitor.readWord(ownerOffset, HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION); + Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG)); + Word owner = monitor.readWord(objectMonitorOwnerOffset(INJECTED_VMCONFIG), OBJECT_MONITOR_OWNER_LOCATION); + + // TODO: JDK21 compatibility + Word newOwner = isJDK21() ? thread : thread.readWord(javaThreadLockIDOffset(INJECTED_VMCONFIG), JAVA_THREAD_LOCK_ID_LOCATION); + // The following owner null check is essential. In the case where the null check fails, it // avoids the subsequent bound-to-fail CAS operation, which would have caused the // invalidation of the L1 cache of the core that runs the lock owner thread, and thus causes // the lock to be held slightly longer. - if (probability(FREQUENT_PROBABILITY, owner.equal(0))) { - // it appears unlocked (owner == 0) - if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(ownerOffset, owner, thread, HotSpotReplacementsUtil.OBJECT_MONITOR_OWNER_LOCATION))) { + if (probability(FREQUENT_PROBABILITY, owner.equal(nullPointer()))) { + // it appears unlocked (owner == null) + if (probability(FREQUENT_PROBABILITY, monitor.logicCompareAndSwapWord(objectMonitorOwnerOffset(INJECTED_VMCONFIG), owner, newOwner, OBJECT_MONITOR_OWNER_LOCATION))) { // success traceObject(trace, "+lock{heavyweight:cas}", object, true); counters.lockHeavyCas.inc(); @@ -254,10 +269,10 @@ private static boolean tryEnterInflated(Object object, Word mark, Word thread, b traceObject(trace, "+lock{heavyweight:failed-cas}", object, true); counters.lockHeavyFailedCas.inc(); } - } else if (probability(NOT_LIKELY_PROBABILITY, owner.equal(thread))) { - int recursionsOffset = HotSpotReplacementsUtil.objectMonitorRecursionsOffset(INJECTED_VMCONFIG); - Word recursions = monitor.readWord(recursionsOffset, HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION); - monitor.writeWord(recursionsOffset, recursions.add(1), HotSpotReplacementsUtil.OBJECT_MONITOR_RECURSION_LOCATION); + } else if (probability(NOT_LIKELY_PROBABILITY, owner.equal(newOwner))) { + int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG); + Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION); + monitor.writeWord(recursionsOffset, recursions.add(1), OBJECT_MONITOR_RECURSION_LOCATION); traceObject(trace, "+lock{heavyweight:recursive}", object, true); counters.lockHeavyRecursive.inc(); return true; @@ -272,8 +287,8 @@ private static boolean tryStackLocking(Object object, Word lock, Word mark, Word if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) { // Inflated case // Set the lock slot's displaced mark to unused. Any non-0 value suffices. - lock.writeWord(HotSpotReplacementsUtil.lockDisplacedMarkOffset(INJECTED_VMCONFIG), WordFactory.unsigned(3), HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION); - return tryEnterInflated(object, mark, thread, trace, counters); + lock.writeWord(lockDisplacedMarkOffset(INJECTED_VMCONFIG), WordFactory.unsigned(unusedMark(INJECTED_VMCONFIG)), DISPLACED_MARK_WORD_LOCATION); + return tryEnterInflated(object, lock, mark, thread, trace, counters); } Pointer objectPointer = Word.objectToTrackedPointer(object); @@ -340,7 +355,7 @@ private static boolean tryLightweightLocking(Object object, Word lock, Word mark if (probability(SLOW_PATH_PROBABILITY, mark.and(monitorMask(INJECTED_VMCONFIG)).notEqual(0))) { // Inflated case - return tryEnterInflated(object, mark, thread, trace, counters); + return tryEnterInflated(object, lock, mark, thread, trace, counters); } // Check if lock-stack is full. @@ -517,6 +532,15 @@ private static boolean tryExitInflated(Object object, Word thread, boolean trace Word mark = loadWordFromObject(object, markOffset(INJECTED_VMCONFIG)); Word monitor = mark.subtract(monitorMask(INJECTED_VMCONFIG)); int ownerOffset = objectMonitorOwnerOffset(INJECTED_VMCONFIG); + + Word owner = monitor.readWord(ownerOffset, OBJECT_MONITOR_OWNER_LOCATION); + // TODO C2HandleAnonOMOwnerStub was deleted long ago. Double check why we are re-enabling it + if (!isJDK21() && useStackLocking(INJECTED_VMCONFIG) && probability(VERY_SLOW_PATH_PROBABILITY, owner.equal(WordFactory.unsigned(objectMonitorAnonymousOwner(INJECTED_VMCONFIG))))) { + Word realOwner = thread.readWord(javaThreadLockIDOffset(INJECTED_VMCONFIG), JAVA_THREAD_LOCK_ID_LOCATION); + monitor.writeWord(ownerOffset, realOwner, OBJECT_MONITOR_OWNER_LOCATION); + monitor.writeWord(objectMonitorStackLockerOffset(INJECTED_VMCONFIG), nullPointer(), OBJECT_MONITOR_STACK_LOCKER_LOCATION); + } + int recursionsOffset = objectMonitorRecursionsOffset(INJECTED_VMCONFIG); Word recursions = monitor.readWord(recursionsOffset, OBJECT_MONITOR_RECURSION_LOCATION); if (probability(FAST_PATH_PROBABILITY, recursions.equal(0))) { @@ -530,7 +554,7 @@ private static boolean tryExitInflated(Object object, Word thread, boolean trace // Nobody is waiting, success // release_store memoryBarrier(MembarNode.FenceKind.STORE_RELEASE); - monitor.writeWord(ownerOffset, zero()); + monitor.writeWord(ownerOffset, nullPointer(), OBJECT_MONITOR_OWNER_LOCATION); traceObject(trace, "-lock{heavyweight:simple}", object, false); counters.unlockHeavySimple.inc(); return true; @@ -540,7 +564,7 @@ private static boolean tryExitInflated(Object object, Word thread, boolean trace if (probability(FREQUENT_PROBABILITY, succ.isNonNull())) { // There may be a thread spinning on this monitor. Temporarily setting // the monitor owner to null, and hope that the other thread will grab it. - monitor.writeWordVolatile(ownerOffset, zero()); + monitor.writeWordVolatile(ownerOffset, nullPointer()); succ = monitor.readWordVolatile(succOffset, OBJECT_MONITOR_SUCC_LOCATION); if (probability(NOT_FREQUENT_PROBABILITY, succ.isNonNull())) { // We manage to release the monitor before the other running thread even @@ -551,7 +575,8 @@ private static boolean tryExitInflated(Object object, Word thread, boolean trace } else { // Either the monitor is grabbed by a spinning thread, or the spinning // thread parks. Now we attempt to reset the owner of the monitor. - if (probability(FREQUENT_PROBABILITY, !monitor.logicCompareAndSwapWord(ownerOffset, zero(), thread, OBJECT_MONITOR_OWNER_LOCATION))) { + Word oldOwner = isJDK21() ? thread : thread.readWord(javaThreadLockIDOffset(INJECTED_VMCONFIG), JAVA_THREAD_LOCK_ID_LOCATION); + if (probability(FREQUENT_PROBABILITY, !monitor.logicCompareAndSwapWord(ownerOffset, nullPointer(), oldOwner, OBJECT_MONITOR_OWNER_LOCATION))) { // The monitor is stolen. traceObject(trace, "-lock{heavyweight:transfer}", object, false); counters.unlockHeavyTransfer.inc(); @@ -581,8 +606,8 @@ private static void decrementHeldMonitorCount(Word thread) { } private static void updateHeldMonitorCount(Word thread, int increment) { - Word heldMonitorCount = thread.readWord(HotSpotReplacementsUtil.heldMonitorCountOffset(INJECTED_VMCONFIG), HotSpotReplacementsUtil.JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); - thread.writeWord(HotSpotReplacementsUtil.heldMonitorCountOffset(INJECTED_VMCONFIG), heldMonitorCount.add(increment), HotSpotReplacementsUtil.JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); + Word heldMonitorCount = thread.readWord(heldMonitorCountOffset(INJECTED_VMCONFIG), JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); + thread.writeWord(heldMonitorCountOffset(INJECTED_VMCONFIG), heldMonitorCount.add(increment), JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION); } @Fold @@ -707,7 +732,8 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot enterLocations = new LocationIdentity[]{ JAVA_THREAD_LOCK_STACK_LOCATION, JAVA_THREAD_LOCK_STACK_TOP_LOCATION, - JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; + JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION, + JAVA_THREAD_LOCK_ID_LOCATION}; exitLocations = new LocationIdentity[]{ JAVA_THREAD_LOCK_STACK_LOCATION, JAVA_THREAD_LOCK_STACK_TOP_LOCATION, @@ -717,10 +743,13 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot OBJECT_MONITOR_ENTRY_LIST_LOCATION, OBJECT_MONITOR_RECURSION_LOCATION, OBJECT_MONITOR_SUCC_LOCATION, + OBJECT_MONITOR_STACK_LOCKER_LOCATION, MARK_WORD_LOCATION, JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; } else { - enterLocations = new LocationIdentity[]{JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; + enterLocations = new LocationIdentity[]{ + JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION, + JAVA_THREAD_LOCK_ID_LOCATION}; exitLocations = new LocationIdentity[]{ DISPLACED_MARK_WORD_LOCATION, OBJECT_MONITOR_OWNER_LOCATION, @@ -728,6 +757,7 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot OBJECT_MONITOR_ENTRY_LIST_LOCATION, OBJECT_MONITOR_RECURSION_LOCATION, OBJECT_MONITOR_SUCC_LOCATION, + OBJECT_MONITOR_STACK_LOCKER_LOCATION, MARK_WORD_LOCATION, JAVA_THREAD_HOLD_MONITOR_COUNT_LOCATION}; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_VirtualThread.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_VirtualThread.java index bed689092924..ee7afebf1762 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_VirtualThread.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/Target_java_lang_VirtualThread.java @@ -83,15 +83,15 @@ public final class Target_java_lang_VirtualThread { */ @Alias // @InjectAccessors(DefaultSchedulerAccessor.class) // - public static ForkJoinPool DEFAULT_SCHEDULER; + public static Executor DEFAULT_SCHEDULER; /** * (Re)initialize the unparker at runtime so that it does not reference any platform threads of * the image builder. */ @Alias // - @InjectAccessors(UnparkerAccessor.class) // - private static ScheduledExecutorService UNPARKER; + @InjectAccessors(DelayedTaskSchedulersAccessor.class) // + private static ScheduledExecutorService[] DELAYED_TASK_SCHEDULERS; /** Go through {@link #nondefaultScheduler}. */ @Alias // @@ -108,24 +108,24 @@ public final class Target_java_lang_VirtualThread { // Checkstyle: resume @Alias - private static native ForkJoinPool createDefaultScheduler(); + private static native Executor createDefaultScheduler(); @Alias - private static native ScheduledExecutorService createDelayedTaskScheduler(); + private static native ScheduledExecutorService[] createDelayedTaskSchedulers(); private static final class DefaultSchedulerAccessor { - private static volatile ForkJoinPool defaultScheduler; + private static volatile Executor defaultScheduler; - public static ForkJoinPool get() { - ForkJoinPool result = defaultScheduler; + public static Executor get() { + Executor result = defaultScheduler; if (result == null) { result = initializeDefaultScheduler(); } return result; } - private static synchronized ForkJoinPool initializeDefaultScheduler() { - ForkJoinPool result = defaultScheduler; + private static synchronized Executor initializeDefaultScheduler() { + Executor result = defaultScheduler; if (result == null) { result = createDefaultScheduler(); defaultScheduler = result; @@ -134,22 +134,22 @@ private static synchronized ForkJoinPool initializeDefaultScheduler() { } } - private static final class UnparkerAccessor { - private static volatile ScheduledExecutorService delayedTaskScheduler; + private static final class DelayedTaskSchedulersAccessor { + private static volatile ScheduledExecutorService[] delayedTaskScheduler; @SuppressWarnings("unused") - public static ScheduledExecutorService get() { - ScheduledExecutorService result = delayedTaskScheduler; + public static ScheduledExecutorService[] get() { + ScheduledExecutorService[] result = delayedTaskScheduler; if (result == null) { result = initializeDelayedTaskScheduler(); } return result; } - private static synchronized ScheduledExecutorService initializeDelayedTaskScheduler() { - ScheduledExecutorService result = delayedTaskScheduler; + private static synchronized ScheduledExecutorService[] initializeDelayedTaskScheduler() { + ScheduledExecutorService[] result = delayedTaskScheduler; if (result == null) { - result = createDelayedTaskScheduler(); + result = createDelayedTaskSchedulers(); delayedTaskScheduler = result; } return result;