[GR-64496] Disallow FFM API objects with associated native resources in image heap.

PullRequest: graal/20767

Author: fangerer

substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/ForeignFunctionsRuntime.java

@@ -27,6 +27,8 @@
 import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.HAS_SIDE_EFFECT;
 
 import java.lang.constant.DirectMethodHandleDesc;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.MemorySegment.Scope;
 import java.lang.invoke.MethodHandle;
 import java.util.HashMap;
 import java.util.Locale;
@@ -42,13 +44,16 @@
 import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.Pointer;
 
+import com.oracle.svm.core.ForeignSupport;
 import com.oracle.svm.core.FunctionPointerHolder;
 import com.oracle.svm.core.OS;
 import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.c.InvokeJavaFunctionPointer;
 import com.oracle.svm.core.headers.LibC;
 import com.oracle.svm.core.headers.WindowsAPIs;
+import com.oracle.svm.core.image.DisallowedImageHeapObjects.DisallowedObjectReporter;
 import com.oracle.svm.core.snippets.SnippetRuntime;
 import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
 import com.oracle.svm.core.util.BasedOnJDKFile;
@@ -57,15 +62,16 @@
 import jdk.graal.compiler.api.replacements.Fold;
 import jdk.graal.compiler.word.Word;
 import jdk.internal.foreign.CABI;
+import jdk.internal.foreign.MemorySessionImpl;
 import jdk.internal.foreign.abi.CapturableState;
 
-public class ForeignFunctionsRuntime {
+public class ForeignFunctionsRuntime implements ForeignSupport {
     @Fold
     public static ForeignFunctionsRuntime singleton() {
         return ImageSingletons.lookup(ForeignFunctionsRuntime.class);
     }
 
-    private final AbiUtils.TrampolineTemplate trampolineTemplate = AbiUtils.singleton().generateTrampolineTemplate();
+    private final AbiUtils.TrampolineTemplate trampolineTemplate;
     private final EconomicMap<NativeEntryPointInfo, FunctionPointerHolder> downcallStubs = EconomicMap.create();
     private final EconomicMap<DirectMethodHandleDesc, FunctionPointerHolder> directUpcallStubs = EconomicMap.create();
     private final EconomicMap<JavaEntryPointInfo, FunctionPointerHolder> upcallStubs = EconomicMap.create();
@@ -77,7 +83,8 @@ public static ForeignFunctionsRuntime singleton() {
     private BiConsumer<Long, DirectMethodHandleDesc> usingSpecializedUpcallListener;
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public ForeignFunctionsRuntime() {
+    public ForeignFunctionsRuntime(AbiUtils abiUtils) {
+        this.trampolineTemplate = abiUtils.generateTrampolineTemplate();
     }
 
     public static boolean areFunctionCallsSupported() {
@@ -224,6 +231,60 @@ private static String generateMessage(JavaEntryPointInfo jep) {
         }
     }
 
+    /**
+     * Arguments follow the same structure as described in {@link NativeEntryPointInfo}, with an
+     * additional {@link Target_jdk_internal_foreign_abi_NativeEntryPoint} (NEP) as the last
+     * argument, i.e.
+     *
+     * <pre>
+     * {@code
+     *      [return buffer address] <call address> [capture state address] <actual arg 1> <actual arg 2> ... <NEP>
+     * }
+     * </pre>
+     *
+     * where <actual arg i>s are the arguments which end up being passed to the C native function
+     */
+    @Override
+    public Object linkToNative(Object... args) throws Throwable {
+        Target_jdk_internal_foreign_abi_NativeEntryPoint nep = (Target_jdk_internal_foreign_abi_NativeEntryPoint) args[args.length - 1];
+        StubPointer pointer = Word.pointer(nep.downcallStubAddress);
+        /* The nep argument will be dropped in the invoked function */
+        return pointer.invoke(args);
+    }
+
+    @Override
+    public void onMemorySegmentReachable(Object memorySegmentObj, DisallowedObjectReporter reporter) {
+        VMError.guarantee(memorySegmentObj instanceof MemorySegment);
+
+        MemorySegment memorySegment = (MemorySegment) memorySegmentObj;
+        if (memorySegment.isNative() && !MemorySegment.NULL.equals(memorySegment)) {
+            throw reporter.raise("Detected a native MemorySegment in the image heap. " +
+                            "A native MemorySegment has a pointer to unmanaged C memory, and C memory from the image generator is not available at image runtime.", memorySegment,
+                            "Try avoiding to initialize the class that called 'MemorySegment.ofAddress'.");
+        }
+    }
+
+    @Override
+    @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/java.base/share/classes/java/lang/foreign/MemorySegment.java#L2708")
+    public void onScopeReachable(Object scopeObj, DisallowedObjectReporter reporter) {
+        VMError.guarantee(scopeObj instanceof Scope);
+
+        /*
+         * We never allow memory sessions with state 'OPEN' to be included in the image heap because
+         * native memory may be associated with them which will be attempted to be free'd if the
+         * session is closed. Non-closable or closed sessions are allowed.
+         * 
+         * Note: This assumes that there is only one implementor of interface Scope which is
+         * MemorySessionImpl. If JDK's class hierarchy changes, we need to adapt this as well.
+         */
+        if (scopeObj instanceof MemorySessionImpl memorySessionImpl && memorySessionImpl.isAlive() && memorySessionImpl.isCloseable()) {
+            throw reporter.raise("Detected an open but closable MemorySegment.Scope in the image heap. " +
+                            "A MemorySegment.Scope may have associated unmanaged C memory that will be attempted to be free'd if the scope is closed. " +
+                            "However, C memory from the image generator is no longer available at image runtime.", memorySessionImpl,
+                            "Try avoiding to initialize the class that called 'Arena.ofConfined/ofShared'.");
+        }
+    }
+
     /**
      * Workaround for CapturableState.mask() being interruptible.
      */
@@ -276,3 +337,8 @@ public static void captureCallState(int statesToCapture, CIntPointer captureBuff
     public static final SnippetRuntime.SubstrateForeignCallDescriptor CAPTURE_CALL_STATE = SnippetRuntime.findForeignCall(ForeignFunctionsRuntime.class,
                     "captureCallState", HAS_SIDE_EFFECT, LocationIdentity.any());
 }
+
+interface StubPointer extends CFunctionPointer {
+    @InvokeJavaFunctionPointer
+    Object invoke(Object... args);
+}

substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/LinkToNativeSupportImpl.java

@@ -28,13 +28,22 @@
 import com.oracle.svm.core.annotate.RecomputeFieldValue;
 import com.oracle.svm.core.annotate.RecomputeFieldValue.Kind;
 import com.oracle.svm.core.annotate.TargetClass;
-import com.oracle.svm.core.jdk.JDKLatest;
 
-@TargetClass(className = "jdk.internal.foreign.MemorySessionImpl", onlyWith = {JDKLatest.class, ForeignAPIPredicates.Enabled.class})
+import jdk.internal.foreign.MemorySessionImpl.ResourceList;
+
+@TargetClass(className = "jdk.internal.foreign.MemorySessionImpl", onlyWith = ForeignAPIPredicates.Enabled.class)
 final class Target_jdk_internal_foreign_MemorySessionImpl {
     @Alias //
     int state;
 
+    /**
+     * Non-closable or closed memory sessions may land in the image heap but the resource list is no
+     * longer required.
+     */
+    @Alias //
+    @RecomputeFieldValue(isFinal = true, kind = Kind.Reset) //
+    ResourceList resourceList;
+
     @Alias //
     @RecomputeFieldValue(isFinal = true, kind = Kind.None) //
     static int CLOSED;

substratevm/src/com.oracle.svm.core.foreign/src/com/oracle/svm/core/foreign/Target_jdk_internal_foreign_MemorySessionImpl.java

@@ -24,19 +24,28 @@
  */
 package com.oracle.svm.core;
 
-import jdk.graal.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
 
-public interface LinkToNativeSupport {
+import com.oracle.svm.core.image.DisallowedImageHeapObjects.DisallowedObjectReporter;
+
+import jdk.graal.compiler.api.replacements.Fold;
+
+public interface ForeignSupport {
     @Fold
     static boolean isAvailable() {
-        return ImageSingletons.contains(LinkToNativeSupport.class);
+        boolean result = ImageSingletons.contains(ForeignSupport.class);
+        assert result || !SubstrateOptions.ForeignAPISupport.getValue();
+        return result;
     }
 
     @Fold
-    static LinkToNativeSupport singleton() {
-        return ImageSingletons.lookup(LinkToNativeSupport.class);
+    static ForeignSupport singleton() {
+        return ImageSingletons.lookup(ForeignSupport.class);
     }
 
     Object linkToNative(Object... args) throws Throwable;
+
+    void onMemorySegmentReachable(Object obj, DisallowedObjectReporter reporter);
+
+    void onScopeReachable(Object obj, DisallowedObjectReporter reporter);
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/LinkToNativeSupport.java renamed to substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/ForeignSupport.java

@@ -38,6 +38,7 @@
 import java.util.random.RandomGenerator;
 import java.util.zip.ZipFile;
 
+import com.oracle.svm.core.ForeignSupport;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.util.ReflectionUtil;
@@ -53,12 +54,14 @@ public interface DisallowedObjectReporter {
         RuntimeException raise(String msg, Object obj, String initializerAction);
     }
 
-    public static final Class<?> CANCELLABLE_CLASS = ReflectionUtil.lookupClass(false, "sun.nio.fs.Cancellable");
-    private static final Class<?> VIRTUAL_THREAD_CLASS = ReflectionUtil.lookupClass(false, "java.lang.VirtualThread");
-    public static final Class<?> CONTINUATION_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.vm.Continuation");
+    public static final Class<?> CANCELLABLE_CLASS = ReflectionUtil.lookupClass("sun.nio.fs.Cancellable");
+    private static final Class<?> VIRTUAL_THREAD_CLASS = ReflectionUtil.lookupClass("java.lang.VirtualThread");
+    public static final Class<?> CONTINUATION_CLASS = ReflectionUtil.lookupClass("jdk.internal.vm.Continuation");
     private static final Method CONTINUATION_IS_STARTED_METHOD = ReflectionUtil.lookupMethod(CONTINUATION_CLASS, "isStarted");
-    private static final Class<?> CLEANER_CLEANABLE_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.ref.CleanerImpl$CleanerCleanable");
-    public static final Class<?> LEGACY_CLEANER_CLASS = ReflectionUtil.lookupClass(false, "jdk.internal.ref.Cleaner");
+    private static final Class<?> CLEANER_CLEANABLE_CLASS = ReflectionUtil.lookupClass("jdk.internal.ref.CleanerImpl$CleanerCleanable");
+    public static final Class<?> LEGACY_CLEANER_CLASS = ReflectionUtil.lookupClass("jdk.internal.ref.Cleaner");
+    public static final Class<?> MEMORY_SEGMENT_CLASS = ReflectionUtil.lookupClass("java.lang.foreign.MemorySegment");
+    public static final Class<?> SCOPE_CLASS = ReflectionUtil.lookupClass("java.lang.foreign.MemorySegment$Scope");
 
     public static void check(Object obj, DisallowedObjectReporter reporter) {
         if (obj instanceof SplittableRandom random) {
@@ -98,6 +101,14 @@ public static void check(Object obj, DisallowedObjectReporter reporter) {
         if (CANCELLABLE_CLASS.isInstance(obj)) {
             onCancellableReachable(obj, reporter);
         }
+
+        if (MEMORY_SEGMENT_CLASS.isInstance(obj) && ForeignSupport.isAvailable()) {
+            ForeignSupport.singleton().onMemorySegmentReachable(obj, reporter);
+        }
+
+        if (SCOPE_CLASS.isInstance(obj) && ForeignSupport.isAvailable()) {
+            ForeignSupport.singleton().onScopeReachable(obj, reporter);
+        }
     }
 
     public static void onRandomReachable(Random random, DisallowedObjectReporter reporter) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/image/DisallowedImageHeapObjects.java

@@ -37,7 +37,7 @@
 import java.lang.reflect.Modifier;
 import java.util.Arrays;
 
-import com.oracle.svm.core.LinkToNativeSupport;
+import com.oracle.svm.core.ForeignSupport;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.annotate.Alias;
 import com.oracle.svm.core.annotate.Delete;
@@ -144,8 +144,8 @@ static Object linkToSpecial(Object... args) throws Throwable {
 
     @Substitute(polymorphicSignature = true)
     static Object linkToNative(Object... args) throws Throwable {
-        if (LinkToNativeSupport.isAvailable()) {
-            return LinkToNativeSupport.singleton().linkToNative(args);
+        if (ForeignSupport.isAvailable()) {
+            return ForeignSupport.singleton().linkToNative(args);
         } else {
             throw unsupportedFeature("The foreign downcalls feature is not available. Please make sure that preview features are enabled with '--enable-preview'.");
         }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/methodhandles/Target_java_lang_invoke_MethodHandle.java

@@ -35,7 +35,6 @@
 import com.oracle.svm.core.foreign.AbiUtils;
 import com.oracle.svm.core.foreign.DowncallStubsHolder;
 import com.oracle.svm.core.foreign.ForeignFunctionsRuntime;
-import com.oracle.svm.core.foreign.LinkToNativeSupportImpl;
 import com.oracle.svm.core.foreign.NativeEntryPointInfo;
 import com.oracle.svm.core.foreign.Target_jdk_internal_foreign_abi_NativeEntryPoint;
 import com.oracle.svm.core.graal.code.AssignedLocation;
@@ -65,7 +64,7 @@
  * float, double, pointer) which fit in a register --- done by HotSpot's implementation using method
  * handles (or specialized classes);</li>
  * <li>Unbox the arguments (the arguments are in an array of Objects, due to funneling through
- * {@link LinkToNativeSupportImpl#linkToNative}) --- done by
+ * {@link ForeignFunctionsRuntime#linkToNative}) --- done by
  * {@link ForeignGraphKit#unboxArguments};</li>
  * <li>Further adapt arguments as to satisfy SubstrateVM's backends --- done by
  * {@link AbiUtils.adapt}</li>
@@ -128,7 +127,7 @@ public AnnotationValue[] getInjectedAnnotations() {
 
     /**
      * The arguments follow the structure described in
-     * {@link LinkToNativeSupportImpl#linkToNative(Object...)}.
+     * {@link ForeignFunctionsRuntime#linkToNative(Object...)}.
      */
     @Override
     public StructuredGraph buildGraph(DebugContext debug, AnalysisMethod method, HostedProviders providers, Purpose purpose) {

substratevm/src/com.oracle.svm.hosted.foreign/src/com/oracle/svm/hosted/foreign/DowncallStub.java

@@ -67,7 +67,7 @@
 import com.oracle.graal.pointsto.meta.AnalysisUniverse;
 import com.oracle.svm.configure.ConfigurationFile;
 import com.oracle.svm.configure.ConfigurationParser;
-import com.oracle.svm.core.LinkToNativeSupport;
+import com.oracle.svm.core.ForeignSupport;
 import com.oracle.svm.core.OS;
 import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.SubstrateUtil;
@@ -78,7 +78,6 @@
 import com.oracle.svm.core.foreign.AbiUtils;
 import com.oracle.svm.core.foreign.ForeignFunctionsRuntime;
 import com.oracle.svm.core.foreign.JavaEntryPointInfo;
-import com.oracle.svm.core.foreign.LinkToNativeSupportImpl;
 import com.oracle.svm.core.foreign.NativeEntryPointInfo;
 import com.oracle.svm.core.foreign.RuntimeSystemLookup;
 import com.oracle.svm.core.foreign.SubstrateMappedMemoryUtils;
@@ -280,13 +279,19 @@ public boolean isInConfiguration(IsInConfigurationAccess access) {
     }
 
     @Override
-    public void duringSetup(DuringSetupAccess a) {
-        var access = (FeatureImpl.DuringSetupAccessImpl) a;
+    public void afterRegistration(AfterRegistrationAccess access) {
         AbiUtils abiUtils = AbiUtils.create();
+        ForeignFunctionsRuntime foreignFunctionsRuntime = new ForeignFunctionsRuntime(abiUtils);
+
         ImageSingletons.add(AbiUtils.class, abiUtils);
-        ImageSingletons.add(ForeignFunctionsRuntime.class, new ForeignFunctionsRuntime());
+        ImageSingletons.add(ForeignSupport.class, foreignFunctionsRuntime);
+        ImageSingletons.add(ForeignFunctionsRuntime.class, foreignFunctionsRuntime);
+    }
+
+    @Override
+    public void duringSetup(DuringSetupAccess a) {
+        var access = (FeatureImpl.DuringSetupAccessImpl) a;
         ImageSingletons.add(RuntimeForeignAccessSupport.class, accessSupport);
-        ImageSingletons.add(LinkToNativeSupport.class, new LinkToNativeSupportImpl());
         ImageSingletons.add(SharedArenaSupport.class, new SharedArenaSupportImpl());
 
         ImageClassLoader imageClassLoader = access.getImageClassLoader();