Use X509ExtendedTrustManager to avoid JDK AlgorithmChecker constraints

Netty 4.2.15 fixed CVE-2026-50010 by removing the silent wrapping of
plain X509TrustManager in X509ExtendedTrustManager. When a plain
X509TrustManager is used, the JDK wraps it in AbstractTrustManagerWrapper
and runs TrustManagerImpl.checkTrusted() with AlgorithmChecker before
calling the custom trust manager.

This caused client certificates signed with SHA3-512withECDSA to be
rejected even though they are not explicitly blacklisted in java.security,
because the JDK's internal PKIX validator applies stricter constraints.

By making our custom trust managers implement X509ExtendedTrustManager
directly, the JDK calls the 3-arg methods directly and bypasses its
internal TrustManagerImpl, restoring the pre-4.2.15 behavior where
only our custom PKIX validation runs.

Files changed:
- rbcs-common/RBCS.kt: getTrustManager() returns X509ExtendedTrustManager
- rbcs-client/RemoteBuildCacheClient.kt: trust-all manager uses X509ExtendedTrustManager

.gitignore

@@ -4,8 +4,6 @@
 # Ignore Gradle build output directory
 build
 
-rbcs-cli/native-image/*.json
-
 # Ignore JDTLS files
 .classpath
 .project

docker/Dockerfile

@@ -50,6 +50,7 @@ COPY --from=base-native /etc/rbcs /etc/rbcs
 COPY --from=base-native /var/lib/rbcs /var/lib/rbcs
 COPY --from=base-native /var/tmp/rbcs /var/tmp/rbcs
 ADD rbcs-cli.upx /usr/bin/rbcs-cli
+ADD logback.xml /etc/rbcs/logback.xml
 USER rbcs
 WORKDIR /var/lib/rbcs
 ENV RBCS_CONFIGURATION_DIR="/etc/rbcs"

rbcs-cli/build.gradle

@@ -111,7 +111,7 @@ nativeImage {
         vendor = JvmVendorSpec.ORACLE
     }
     mainClass = mainClassName
-    mainModule = mainModuleName
+    //mainModule = mainModuleName
     useMusl = true
     buildStaticImage = true
     linkAtBuildTime = false
@@ -119,6 +119,7 @@ nativeImage {
     compressExecutable = true
     compressionLevel = 6
     useLZMA = false
+    //verbose = true
 }
 
 Provider<UpxTask> upxTaskProvider = tasks.named(NativeImagePlugin.UPX_TASK_NAME, UpxTask) {
@@ -152,7 +153,6 @@ Provider<JlinkTask> jlinkTaskProvider = tasks.named(JlinkPlugin.JLINK_TASK_NAME,
 }
 
 Provider<Tar> jlinkDistTarTaskProvider = tasks.named(JlinkPlugin.JLINK_DIST_TAR_TASK_NAME, Tar) {
-    exclude 'lib/libjvmcicompiler.so'
 }
 
 tasks.named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, ProcessResources) {

rbcs-cli/native-image/native-image.properties

@@ -1,2 +1,11 @@
-Args=-Ob -march=x86-64-v2 --gc=serial --initialize-at-run-time=io.netty --enable-url-protocols=jpms --emit build-report -H:+UnlockExperimentalVMOptions -H:+SharedArenaSupport --initialize-at-build-time=net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory,net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory$JpmsHandler --add-opens=io.netty.common/io.netty.util=org.graalvm.nativeimage.builder --add-opens=io.netty.common/io.netty.util.internal.svm=org.graalvm.nativeimage.builder
+Args=-O3 \
+   -march=x86-64-v3 \
+   --gc=serial \
+   --enable-url-protocols=jpms \
+   --pgo=native-image/default.iprof \
+   --initialize-at-run-time=io.netty \
+   --initialize-at-build-time=net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory,net.woggioni.rbcs.common.RbcsUrlStreamHandlerFactory$JpmsHandler \
+   --trace-object-instantiation=ch.qos.logback.classic.Logger \
+   -H:+UnlockExperimentalVMOptions \
+   -H:+SharedArenaSupport
 #-H:TraceClassInitialization=io.netty.handler.ssl.BouncyCastleAlpnSslUtils

rbcs-client/src/main/kotlin/net/woggioni/rbcs/client/RemoteBuildCacheClient.kt

@@ -10,6 +10,7 @@ import java.util.concurrent.TimeUnit
 import java.util.concurrent.TimeoutException
 import java.util.concurrent.atomic.AtomicInteger
 import javax.net.ssl.TrustManagerFactory
+import javax.net.ssl.X509ExtendedTrustManager
 import javax.net.ssl.X509TrustManager
 import kotlin.random.Random
 import io.netty.util.concurrent.Future as NettyFuture
@@ -74,13 +75,25 @@ class RemoteBuildCacheClient(private val profile: Configuration.Profile) : AutoC
                     )
                     profile.tlsTruststore?.let { trustStore ->
                         if (!trustStore.verifyServerCertificate) {
-                            trustManager(object : X509TrustManager {
+                            trustManager(object : X509ExtendedTrustManager() {
                                 override fun checkClientTrusted(certChain: Array<out X509Certificate>, p1: String?) {
                                 }
 
+                                override fun checkClientTrusted(certChain: Array<out X509Certificate>, p1: String?, socket: java.net.Socket) {
+                                }
+
+                                override fun checkClientTrusted(certChain: Array<out X509Certificate>, p1: String?, engine: javax.net.ssl.SSLEngine) {
+                                }
+
                                 override fun checkServerTrusted(certChain: Array<out X509Certificate>, p1: String?) {
                                 }
 
+                                override fun checkServerTrusted(certChain: Array<out X509Certificate>, p1: String?, socket: java.net.Socket) {
+                                }
+
+                                override fun checkServerTrusted(certChain: Array<out X509Certificate>, p1: String?, engine: javax.net.ssl.SSLEngine) {
+                                }
+
                                 override fun getAcceptedIssuers() = null
                             })
                         } else {

rbcs-common/src/main/kotlin/net/woggioni/rbcs/common/RBCS.kt

@@ -19,6 +19,7 @@ import java.security.cert.X509Certificate
 import java.util.EnumSet
 import java.util.ServiceLoader
 import javax.net.ssl.TrustManagerFactory
+import javax.net.ssl.X509ExtendedTrustManager
 import javax.net.ssl.X509TrustManager
 import net.woggioni.jwo.JWO
 import net.woggioni.jwo.Tuple2
@@ -124,7 +125,7 @@ object RBCS {
         return keystore
     }
 
-    fun getTrustManager(trustStore: KeyStore?, certificateRevocationEnabled: Boolean): X509TrustManager {
+    fun getTrustManager(trustStore: KeyStore?, certificateRevocationEnabled: Boolean): X509ExtendedTrustManager {
         return if (trustStore != null) {
             val certificateFactory = CertificateFactory.getInstance("X.509")
             val validator = CertPathValidator.getInstance("PKIX").apply {
@@ -136,7 +137,7 @@ object RBCS {
             val params = PKIXParameters(trustStore).apply {
                 isRevocationEnabled = certificateRevocationEnabled
             }
-            object : X509TrustManager {
+            object : X509ExtendedTrustManager() {
                 override fun checkClientTrusted(chain: Array<out X509Certificate>, authType: String) {
                     val clientCertificateChain = certificateFactory.generateCertPath(chain.toList())
                     try {
@@ -146,10 +147,26 @@ object RBCS {
                     }
                 }
 
+                override fun checkClientTrusted(chain: Array<out X509Certificate>, authType: String, socket: java.net.Socket) {
+                    checkClientTrusted(chain, authType)
+                }
+
+                override fun checkClientTrusted(chain: Array<out X509Certificate>, authType: String, engine: javax.net.ssl.SSLEngine) {
+                    checkClientTrusted(chain, authType)
+                }
+
                 override fun checkServerTrusted(chain: Array<out X509Certificate>, authType: String) {
                     throw NotImplementedError()
                 }
 
+                override fun checkServerTrusted(chain: Array<out X509Certificate>, authType: String, socket: java.net.Socket) {
+                    checkServerTrusted(chain, authType)
+                }
+
+                override fun checkServerTrusted(chain: Array<out X509Certificate>, authType: String, engine: javax.net.ssl.SSLEngine) {
+                    checkServerTrusted(chain, authType)
+                }
+
                 private val acceptedIssuers = trustStore.aliases().asSequence()
                     .filter(trustStore::isCertificateEntry)
                     .map(trustStore::getCertificate)
@@ -161,8 +178,8 @@ object RBCS {
             }
         } else {
             val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
-            trustManagerFactory.trustManagers.asSequence().filter { it is X509TrustManager }
+            trustManagerFactory.trustManagers.asSequence().filter { it is X509ExtendedTrustManager }
-                .single() as X509TrustManager
+                .single() as X509ExtendedTrustManager
         }
     }