Improvements and minor fixes.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java

@@ -52,17 +52,14 @@
  * Most allocation within a AlignedHeapChunk is via fast-path allocation snippets, but a slow-path
  * allocation method is available.
  * <p>
- * Objects in a AlignedHeapChunk have to be promoted by copying from their current HeapChunk to a
- * destination HeapChunk.
- * <p>
- * An AlignedHeapChunk is laid out:
+ * An AlignedHeapChunk is laid out as follows:
  *
  * <pre>
- * +===============+-------+--------+----------------------+
- * | AlignedHeader | Card  | First  | Object ...           |
- * | Fields        | Table | Object |                      |
- * |               |       | Table  |                      |
- * +===============+-------+--------+----------------------+
+ * +===============+-------+--------+-----------------+-----------------+
+ * | AlignedHeader | Card  | First  | Initial Object  | Object ...      |
+ * | Fields        | Table | Object | Move Info (only |                 |
+ * |               |       | Table  | Compacting GC)  |                 |
+ * +===============+-------+--------+-----------------+-----------------+
  * </pre>
  *
  * The size of both the CardTable and the FirstObjectTable depends on the used {@link RememberedSet}
@@ -107,6 +104,10 @@ public static Pointer getObjectsEnd(AlignedHeader that) {
         return HeapChunk.getEndPointer(that);
     }
 
+    public static boolean isEmpty(AlignedHeader that) {
+        return HeapChunk.getTopOffset(that).equal(getObjectsStartOffset());
+    }
+
     /** Allocate uninitialized memory within this AlignedHeapChunk. */
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CopyingOldGeneration.java

@@ -24,9 +24,6 @@
  */
 package com.oracle.svm.core.genscavenge;
 
-import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY;
-import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability;
-
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.Pointer;
@@ -38,8 +35,6 @@
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
-import com.oracle.svm.core.thread.VMOperation;
-import com.oracle.svm.core.util.VMError;
 
 /**
  * An OldGeneration has two Spaces, {@link #fromSpace} for existing objects, and {@link #toSpace}
@@ -109,7 +104,7 @@ void releaseSpaces(ChunkReleaser chunkReleaser) {
 
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    void beginPromotion(YoungGeneration youngGen, boolean incrementalGc) {
+    void beginPromotion(boolean incrementalGc) {
         if (incrementalGc) {
             emptyFromSpaceIntoToSpace();
         }
@@ -161,9 +156,8 @@ void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor) {
     }
 
     @Override
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     boolean isInSpace(Pointer ptr) {
-        return HeapImpl.findPointerInSpace(fromSpace, ptr) || HeapImpl.findPointerInSpace(toSpace, ptr);
+        return fromSpace.contains(ptr) || toSpace.contains(ptr);
     }
 
     @Override
@@ -216,18 +210,6 @@ UnsignedWord computeObjectBytes() {
         return fromSpace.computeObjectBytes().add(toSpace.computeObjectBytes());
     }
 
-    @Override
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    AlignedHeapChunk.AlignedHeader requestAlignedChunk() {
-        assert VMOperation.isGCInProgress() : "Should only be called from the collector.";
-        AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk();
-        if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) {
-            throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk");
-        }
-        RememberedSet.get().enableRememberedSetForChunk(chunk);
-        return chunk;
-    }
-
     @Override
     void checkSanityBeforeCollection() {
         assert toSpace.isEmpty() : "toSpace should be empty before a collection.";

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -999,7 +999,7 @@ private void blackenDirtyCardRoots() {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     private static void beginPromotion(boolean isIncremental) {
         HeapImpl heap = HeapImpl.getHeapImpl();
-        heap.getOldGeneration().beginPromotion(heap.getYoungGeneration(), isIncremental);
+        heap.getOldGeneration().beginPromotion(isIncremental);
         if (isIncremental) {
             heap.getYoungGeneration().beginPromotion();
         }
@@ -1084,7 +1084,7 @@ private void promotePinnedObject(PinnedObjectImpl pinned) {
             boolean isAligned = ObjectHeaderImpl.isAlignedObject(referent);
             Header<?> originalChunk = getChunk(referent, isAligned);
             Space originalSpace = HeapChunk.getSpace(originalChunk);
-            if (originalSpace.isFromSpace() || originalSpace.isCompactingOldSpace()) {
+            if (originalSpace.isFromSpace() || (originalSpace.isCompactingOldSpace() && completeCollection)) {
                 boolean promoted = false;
                 if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) {
                     promoted = heap.getYoungGeneration().promotePinnedObject(referent, originalChunk, isAligned, originalSpace);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java

@@ -751,10 +751,10 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc
 
         if (allowJavaHeapAccess) {
             // Accessing spaces and chunks is safe if we prevent a GC.
-            if (isInYoungGen(ptr)) {
+            if (youngGeneration.isInSpace(ptr)) {
                 log.string("points into the young generation");
                 return true;
-            } else if (isInOldGen(ptr)) {
+            } else if (oldGeneration.isInSpace(ptr)) {
                 log.string("points into the old generation");
                 return true;
             }
@@ -769,51 +769,7 @@ private boolean printLocationInfo(Log log, Pointer ptr, boolean allowJavaHeapAcc
     }
 
     boolean isInHeap(Pointer ptr) {
-        return isInImageHeap(ptr) || isInYoungGen(ptr) || isInOldGen(ptr);
-    }
-
-    @Uninterruptible(reason = "Prevent that chunks are freed.")
-    private boolean isInYoungGen(Pointer ptr) {
-        if (findPointerInSpace(youngGeneration.getEden(), ptr)) {
-            return true;
-        }
-
-        for (int i = 0; i < youngGeneration.getMaxSurvivorSpaces(); i++) {
-            if (findPointerInSpace(youngGeneration.getSurvivorFromSpaceAt(i), ptr)) {
-                return true;
-            }
-            if (findPointerInSpace(youngGeneration.getSurvivorToSpaceAt(i), ptr)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Uninterruptible(reason = "Prevent that chunks are freed.")
-    private boolean isInOldGen(Pointer ptr) {
-        return oldGeneration.isInSpace(ptr);
-    }
-
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    static boolean findPointerInSpace(Space space, Pointer p) {
-        AlignedHeapChunk.AlignedHeader aChunk = space.getFirstAlignedHeapChunk();
-        while (aChunk.isNonNull()) {
-            Pointer start = AlignedHeapChunk.getObjectsStart(aChunk);
-            if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(aChunk))) {
-                return true;
-            }
-            aChunk = HeapChunk.getNext(aChunk);
-        }
-
-        UnalignedHeapChunk.UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk();
-        while (uChunk.isNonNull()) {
-            Pointer start = UnalignedHeapChunk.getObjectStart(uChunk);
-            if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(uChunk))) {
-                return true;
-            }
-            uChunk = HeapChunk.getNext(uChunk);
-        }
-        return false;
+        return isInImageHeap(ptr) || youngGeneration.isInSpace(ptr) || oldGeneration.isInSpace(ptr);
     }
 
     private static boolean printTlabInfo(Log log, Pointer ptr) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java

@@ -196,6 +196,11 @@ private static boolean verifyAlignedChunks(Space space, AlignedHeader firstAlign
                 success = false;
             }
 
+            if (aChunk.getShouldSweepInsteadOfCompact()) {
+                Log.log().string("Aligned chunk ").zhex(aChunk).string(" is marked for sweeping while this should only be used during collections.").newline();
+                success = false;
+            }
+
             OBJECT_VERIFIER.initialize(aChunk, WordFactory.nullPointer());
             AlignedHeapChunk.walkObjects(aChunk, OBJECT_VERIFIER);
             aChunk = HeapChunk.getNext(aChunk);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java

@@ -358,14 +358,14 @@ public static boolean hasRememberedSet(UnsignedWord header) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static void setMarked(Object o) {
-        if (!SerialGCOptions.useCompactingOldGen()) {
+        if (!SerialGCOptions.useCompactingOldGen()) { // not guarantee(): always folds, prevent call
             throw VMError.shouldNotReachHere("Only for compacting GC.");
         }
         UnsignedWord header = readHeaderFromObject(o);
         assert header.and(FORWARDED_OR_MARKED2_BIT).equal(0) : "forwarded or already marked";
         /*
          * The remembered bit is already set if the object was in the old generation before, or
-         * unset if it was only just absorbed from the young generation.
+         * unset if it was only just absorbed from the young generation, in which case we set it.
          */
         writeHeaderToObject(o, header.or(MARKED_BITS));
     }
@@ -398,7 +398,7 @@ static boolean isPointerToForwardedObject(Pointer p) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean isForwardedHeader(UnsignedWord header) {
-        return header.and(REMSET_OR_MARKED1_BIT.or(FORWARDED_OR_MARKED2_BIT)).equal(FORWARDED_OR_MARKED2_BIT);
+        return header.and(MARKED_BITS).equal(FORWARDED_OR_MARKED2_BIT);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java

@@ -24,29 +24,32 @@
  */
 package com.oracle.svm.core.genscavenge;
 
+import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.EXTREMELY_SLOW_PATH_PROBABILITY;
+import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.probability;
+
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser;
+import com.oracle.svm.core.genscavenge.remset.RememberedSet;
+import com.oracle.svm.core.thread.VMOperation;
+import com.oracle.svm.core.util.VMError;
 
 public abstract class OldGeneration extends Generation {
     OldGeneration(String name) {
         super(name);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    abstract void beginPromotion(YoungGeneration youngGen, boolean incrementalGc);
+    abstract void beginPromotion(boolean incrementalGc);
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     abstract void blackenDirtyCardRoots(GreyToBlackObjectVisitor visitor);
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     abstract boolean scanGreyObjects(boolean incrementalGc);
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    abstract AlignedHeapChunk.AlignedHeader requestAlignedChunk();
-
     abstract void sweepAndCompact(Timers timers, ChunkReleaser chunkReleaser);
 
     abstract void releaseSpaces(ChunkReleaser chunkReleaser);
@@ -58,7 +61,6 @@ public abstract class OldGeneration extends Generation {
 
     abstract UnsignedWord computeObjectBytes();
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     abstract boolean isInSpace(Pointer ptr);
 
     abstract boolean verifyRememberedSets();
@@ -67,4 +69,15 @@ public abstract class OldGeneration extends Generation {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     abstract void tearDown();
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    AlignedHeapChunk.AlignedHeader requestAlignedChunk() {
+        assert VMOperation.isGCInProgress() : "Should only be called from the collector.";
+        AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk();
+        if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) {
+            throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk");
+        }
+        RememberedSet.get().enableRememberedSetForChunk(chunk);
+        return chunk;
+    }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java

@@ -250,8 +250,9 @@ private static boolean willSurviveThisCollection(Object obj) {
     }
 
     static void updateForwardedRefs() {
-        Reference<?> current = rememberedRefsList;
+        assert SerialGCOptions.useCompactingOldGen();
 
+        Reference<?> current = rememberedRefsList;
         while (current != null) {
             // Get the next node (the last node has a cyclic reference to self).
             Reference<?> next = ReferenceInternals.getNextDiscovered(current);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java

@@ -112,7 +112,7 @@ public Integer getValue(OptionValues values) {
     /** Query these options only through an appropriate method. */
     public static class ConcealedOptions {
         @Option(help = "Collect old generation by compacting in-place instead of copying.", type = OptionType.Expert) //
-        public static final HostedOptionKey<Boolean> CompactingOldGen = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly);
+        public static final HostedOptionKey<Boolean> CompactingOldGen = new HostedOptionKey<>(true, SerialGCOptions::validateCompactingOldGen);
 
         @Option(help = "Determines if a remembered set is used, which is necessary for collecting the young and old generation independently.", type = OptionType.Expert) //
         public static final HostedOptionKey<Boolean> UseRememberedSet = new HostedOptionKey<>(true, SerialGCOptions::serialGCOnly);
@@ -127,16 +127,11 @@ private static void serialGCOnly(OptionKey<?> optionKey) {
         }
     }
 
-    @Fold
-    public static boolean useRememberedSet() {
-        return !SubstrateOptions.UseEpsilonGC.getValue() && ConcealedOptions.UseRememberedSet.getValue();
-    }
-
-    @Fold
-    public static boolean useCompactingOldGen() {
-        if (SubstrateOptions.UseEpsilonGC.getValue() || !ConcealedOptions.CompactingOldGen.getValue()) {
-            return false;
+    private static void validateCompactingOldGen(HostedOptionKey<Boolean> compactingOldGen) {
+        if (!compactingOldGen.getValue()) {
+            return;
         }
+        serialGCOnly(compactingOldGen);
         if (!useRememberedSet()) {
             throw UserError.abort("%s requires %s.", SubstrateOptionsParser.commandArgument(ConcealedOptions.CompactingOldGen, "+"),
                             SubstrateOptionsParser.commandArgument(ConcealedOptions.UseRememberedSet, "+"));
@@ -145,6 +140,15 @@ public static boolean useCompactingOldGen() {
             throw UserError.abort("%s requires %s.", SubstrateOptionsParser.commandArgument(ConcealedOptions.CompactingOldGen, "+"),
                             SubstrateOptionsParser.commandArgument(SerialAndEpsilonGCOptions.AlignedHeapChunkSize, "<value below or equal to " + ObjectMoveInfo.MAX_CHUNK_SIZE + ">"));
         }
-        return true;
+    }
+
+    @Fold
+    public static boolean useRememberedSet() {
+        return !SubstrateOptions.UseEpsilonGC.getValue() && ConcealedOptions.UseRememberedSet.getValue();
+    }
+
+    @Fold
+    public static boolean useCompactingOldGen() {
+        return !SubstrateOptions.UseEpsilonGC.getValue() && ConcealedOptions.CompactingOldGen.getValue();
     }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java

@@ -177,6 +177,17 @@ public boolean walkObjects(ObjectVisitor visitor) {
         return true;
     }
 
+    boolean walkAlignedHeapChunks(AlignedHeapChunk.Visitor visitor) {
+        AlignedHeapChunk.AlignedHeader chunk = getFirstAlignedHeapChunk();
+        while (chunk.isNonNull()) {
+            if (!visitor.visitChunk(chunk)) {
+                return false;
+            }
+            chunk = HeapChunk.getNext(chunk);
+        }
+        return true;
+    }
+
     public void logUsage(Log log, boolean logIfEmpty) {
         UnsignedWord chunkBytes;
         if (isEdenSpace() && !VMOperation.isGCInProgress()) {
@@ -526,17 +537,6 @@ void absorb(Space src) {
         assert src.isEmpty();
     }
 
-    boolean walkAlignedHeapChunks(AlignedHeapChunk.Visitor visitor) {
-        AlignedHeapChunk.AlignedHeader chunk = getFirstAlignedHeapChunk();
-        while (chunk.isNonNull()) {
-            if (!visitor.visitChunk(chunk)) {
-                return false;
-            }
-            chunk = HeapChunk.getNext(chunk);
-        }
-        return true;
-    }
-
     /**
      * This value is only updated during a GC. Be careful when calling this method during a GC as it
      * might wrongly include chunks that will be freed at the end of the GC.
@@ -596,4 +596,25 @@ private UnsignedWord computeUnalignedObjectBytes() {
         }
         return result;
     }
+
+    boolean contains(Pointer p) {
+        AlignedHeapChunk.AlignedHeader aChunk = getFirstAlignedHeapChunk();
+        while (aChunk.isNonNull()) {
+            Pointer start = AlignedHeapChunk.getObjectsStart(aChunk);
+            if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(aChunk))) {
+                return true;
+            }
+            aChunk = HeapChunk.getNext(aChunk);
+        }
+
+        UnalignedHeapChunk.UnalignedHeader uChunk = getFirstUnalignedHeapChunk();
+        while (uChunk.isNonNull()) {
+            Pointer start = UnalignedHeapChunk.getObjectStart(uChunk);
+            if (start.belowOrEqual(p) && p.belowThan(HeapChunk.getTopPointer(uChunk))) {
+                return true;
+            }
+            uChunk = HeapChunk.getNext(uChunk);
+        }
+        return false;
+    }
 }