From b5ac5f349fb9e17e6324e476e48117da89cef090 Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Mon, 27 Oct 2025 14:13:01 -0400 Subject: [PATCH] Fix deadlock in RestrictedSecurity caused by JAR verification and hash check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit updates isJarVerifierInStackTrace() to inspect all thread stacks, resolving the deadlock that occurred in ProfileParser.checkHashValues(). In the failing scenario, one thread (Thread-1) calls SecureRandom.getInstance(), which triggers checkHashValues() in RestrictedSecurity mode. This method attempts to obtain a MessageDigest from provider OpenJCEPlusFIPS, requiring the corresponding module to be loaded. However, another thread (Thread-2) is already holding the loader lock while performing signature verification for a signed class. Thread-1 is then blocked waiting for that module to load, while Thread-2 waits for the MessageDigest initialization to complete, resulting in a circular deadlock. Previously, isJarVerifierInStackTrace() only checked the current thread’s stack to decide whether to skip check hash. It has now been updated to scan the stack traces of all threads, but except for the VM service threads, allowing it to detect active JAR verification in other non-VM threads and prevent potential cross-thread deadlocks. Signed-off-by: Tao Liu --- .../openj9/internal/security/RestrictedSecurity.java | 12 +++++++++++- .../share/classes/java/util/jar/JarFile.java | 9 +++++++++ test/jdk/ProblemList-FIPS140_3_OpenJcePlus.txt | 1 - 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java b/closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java index 71f3640aa31..230878a766a 100644 --- a/closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java +++ b/closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java @@ -43,6 +43,7 @@ import java.util.Objects; import java.util.Properties; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -82,6 +83,7 @@ public final class RestrictedSecurity { private static RestrictedSecurityProperties restricts; + private static final AtomicInteger hashPauseCount = new AtomicInteger(0); private static final Set unmodifiableProperties = new HashSet<>(); static { @@ -164,13 +166,21 @@ private static void checkHashValues(boolean fromProviders) { if (fromProviders) { enableCheckHashes = true; } - if (enableCheckHashes && !isJarVerifierInStackTrace()) { + if (enableCheckHashes && !isJarVerifierInStackTrace() && (hashPauseCount.get() == 0)) { profileParser = null; parser.checkHashValues(); } } } + public static void pauseHashCheck() { + hashPauseCount.incrementAndGet(); + } + + public static void resumeHashCheck() { + hashPauseCount.decrementAndGet(); + } + /** * Check if restricted security mode is enabled. * diff --git a/src/java.base/share/classes/java/util/jar/JarFile.java b/src/java.base/share/classes/java/util/jar/JarFile.java index dc2d65bcc99..5fe1c6a05bb 100644 --- a/src/java.base/share/classes/java/util/jar/JarFile.java +++ b/src/java.base/share/classes/java/util/jar/JarFile.java @@ -23,11 +23,18 @@ * questions. */ +/* + * =========================================================================== + * (c) Copyright IBM Corp. 2025, 2025 All Rights Reserved + * =========================================================================== + */ + package java.util.jar; import jdk.internal.access.SharedSecrets; import jdk.internal.access.JavaUtilZipFileAccess; import jdk.internal.misc.ThreadTracker; +import openj9.internal.security.RestrictedSecurity; import sun.security.util.ManifestEntryVerifier; import sun.security.util.SignatureFileVerifier; @@ -1045,10 +1052,12 @@ synchronized void ensureInitialization() { } if (jv != null && !jvInitialized) { Object key = beginInit(); + RestrictedSecurity.pauseHashCheck(); try { initializeVerifier(); jvInitialized = true; } finally { + RestrictedSecurity.resumeHashCheck(); endInit(key); } } diff --git a/test/jdk/ProblemList-FIPS140_3_OpenJcePlus.txt b/test/jdk/ProblemList-FIPS140_3_OpenJcePlus.txt index e463a076e99..2d31bb6befc 100644 --- a/test/jdk/ProblemList-FIPS140_3_OpenJcePlus.txt +++ b/test/jdk/ProblemList-FIPS140_3_OpenJcePlus.txt @@ -309,7 +309,6 @@ java/security/SecureRandom/Serialize.java https://github.com/eclipse-openj9/open java/security/SecureRandom/SerializedSeedTest.java https://github.com/eclipse-openj9/openj9/issues/20978 generic-all java/security/SecureRandom/ThreadSafe.java https://github.com/eclipse-openj9/openj9/issues/20978 generic-all java/security/Security/CaseInsensitiveAlgNames.java https://github.com/eclipse-openj9/openj9/issues/20978 generic-all -java/security/Security/ClassLoaderDeadlock/Deadlock.java https://github.com/eclipse-openj9/openj9/issues/21919 generic-all java/security/Security/ConfigFileTest.java https://github.com/eclipse-openj9/openj9/issues/20978 generic-all java/security/Security/ProviderFiltering.java https://github.com/eclipse-openj9/openj9/issues/20978 generic-all java/security/Security/removing/RemoveProviderByIdentity.java https://github.com/eclipse-openj9/openj9/issues/20978 generic-all