ahus1 · Sep 8, 2026 · Sep 6, 2026 · Sep 7, 2026
@@ -78,6 +78,7 @@
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.infinispan.remoting.transport.Transport;
 import org.infinispan.remoting.transport.jgroups.JGroupsTransport;
+import org.infinispan.topology.LocalTopologyManager;
 import org.jboss.logging.Logger;
 
 import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME;
@@ -151,6 +152,7 @@ public void close() {
         }
         runWithWriteLockOnCacheManager(() -> {
             if (cacheManager != null) {
+                leaveCaches(cacheManager);
                 cacheManager.stop();
                 cacheManager = null;
             }
@@ -161,6 +163,63 @@ public void close() {
         }
     }
 
+    /**
+     * Explicitly leave all distributed caches before stopping the cache manager.
+     * <p>
+     * {@code DefaultCacheManager.stop()} calls {@code cache.stop()} which sends a per-cache
+     * {@code CacheLeaveCommand} to the coordinator, but immediately tears down local cache components
+     * without waiting for the new topology (excluding this node) to propagate to all members.
+     * During that gap, other nodes still route RPCs to this node's dead caches, causing
+     * {@code ISPN000427} remote-timeout failures.
+     * <p>
+     * By sending the leave commands upfront and letting the coordinator broadcast the updated topology
+     * before local teardown begins, other nodes stop routing to this node before its caches terminate.
+     * <p>
+     * This method and its call in {@link #close()} can be removed once Keycloak upgrades to
+     * Infinispan 16.2+, which handles this natively via {@code OrderedGracefulLeaveHandler}
+     * (see <a href="https://github.com/infinispan/infinispan/issues/17016">infinispan#17016</a>).
+     */
+    private void leaveCaches(EmbeddedCacheManager cm) {
+        var ltm = GlobalComponentRegistry.componentOf(cm, LocalTopologyManager.class);
+        if (ltm == null) {
+            return;
+        }
+        var transport = GlobalComponentRegistry.componentOf(cm, Transport.class);
+        if (transport == null) {
+            return;
+        }
+        int clusterSize = transport.getMembers().size();
+        if (clusterSize <= 1) {
+            return;
+        }
+        for (String cacheName : CLUSTERED_CACHE_NAMES) {
+            try {
+                var cache = cm.getCache(cacheName, false);
+                if (cache == null) {
+                    continue;
+                }
+                var dm = cache.getAdvancedCache().getDistributionManager();
+                if (dm == null) {
+                    continue;
+                }
+                logger.infof("Leaving cache '%s' before shutdown", cacheName);
+                ltm.leave(cacheName, cm.getCacheManagerConfiguration().transport().distributedSyncTimeout());
+            } catch (Exception e) {
+                logger.warnf(e, "Failed to leave cache '%s' during shutdown", cacheName);
+            }
+        }
+        if (clusterSize >= 3) {
+            try {
+                // Allow time for the coordinator to broadcast the updated topology to all members.
+                // This will guard against new requests reaching this node hitting a cache that is terminated in the next step.
+                // With only 2 members, the single remaining node receives the topology update directly — no wait needed.
+                Thread.sleep(500);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    }
+
     @Override
     public String getId() {
         return "default";

@@ -45,10 +45,12 @@
 import org.infinispan.configuration.cache.BackupConfiguration;
 import org.infinispan.configuration.cache.BackupFailurePolicy;
 import org.infinispan.configuration.cache.CacheMode;
+import org.infinispan.configuration.cache.ClusteringConfiguration;
 import org.infinispan.configuration.cache.ConfigurationBuilder;
 import org.infinispan.configuration.cache.ExpirationConfiguration;
 import org.infinispan.configuration.cache.HashConfiguration;
 import org.infinispan.configuration.cache.HashConfigurationBuilder;
+import org.infinispan.configuration.cache.LockingConfiguration;
 import org.infinispan.configuration.parsing.ConfigurationBuilderHolder;
 import org.infinispan.eviction.EvictionStrategy;
 import org.infinispan.transaction.LockingMode;
@@ -350,6 +352,28 @@ public static void ensureMinimumOwners(ConfigurationBuilderHolder holder) {
         }
     }
 
+    /**
+     * Validates that lock-timeout &lt; remote-timeout for distributed synchronous caches.
+     * <p>
+     * The remote-timeout is the outer timeout for the entire remote RPC, which includes lock acquisition
+     * on the remote node. If lock-timeout &ge; remote-timeout, the remote call times out before lock
+     * acquisition can complete, causing spurious {@code TimeoutException}s.
+     */
+    public static void ensureTimeoutOrdering(ConfigurationBuilderHolder holder) {
+        for (var entry : holder.getNamedConfigurationBuilders().entrySet()) {
+            var builder = entry.getValue();
+            if (!builder.clustering().cacheMode().isDistributed() || !builder.clustering().cacheMode().isSynchronous()) {
+                continue;
+            }
+            long remoteTimeout = builder.clustering().attributes().attribute(ClusteringConfiguration.REMOTE_TIMEOUT).get().longValue();
+            long lockTimeout = builder.locking().attributes().attribute(LockingConfiguration.LOCK_ACQUISITION_TIMEOUT).get().longValue();
+            if (lockTimeout >= remoteTimeout) {
+                logger.errorf("Cache '%s': lock-timeout (%d ms) must be less than remote-timeout (%d ms). "
+                        + "Current values will cause spurious remote timeouts during lock acquisition.", entry.getKey(), lockTimeout, remoteTimeout);
+            }
+        }
+    }
+
     /**
      * Configures (and overwrites) the {@link HashConfigurationBuilder#numOwners(int)} based on the SPI configuration
      * input.
@@ -575,7 +599,15 @@ public static ConfigurationBuilder getCacheConfiguration(String cacheName, boole
             case AUTHENTICATION_SESSIONS_CACHE_NAME:
             case LOGIN_FAILURE_CACHE_NAME:
                 if (clustered) {
-                    builder.clustering().cacheMode(CacheMode.DIST_SYNC);
+                    // lock-timeout < remote-timeout < retry budget in AbstractRefreshTokenProvider (12s).
+                    // Infinispan defaults (15s remote, 10s lock) are too high for a setup like Keycloak.
+                    builder.clustering().cacheMode(CacheMode.DIST_SYNC)
+                            // 5s remote-timeout is 10x the worst-case G1GC pause for in-memory caches.
+                            .remoteTimeout(5, TimeUnit.SECONDS);
+                    builder.locking()
+                            // 3s lock-timeout is the inner timeout; must be less than remote-timeout.
+                            .lockAcquisitionTimeout(3, TimeUnit.SECONDS);
+                    // 12s retry budget in AbstractRefreshTokenProvider allows 2 retries at ~5s each.
                 }
                 builder.encoding().mediaType(MediaType.APPLICATION_OBJECT_TYPE);
                 return builder;

@@ -211,6 +211,7 @@ private static void singleSiteConfiguration(Config.Scope config, ConfigurationBu
         CacheConfigurator.configureNumOwners(config, holder);
         CacheConfigurator.validateWorkCacheConfiguration(holder);
         CacheConfigurator.ensureMinimumOwners(holder);
+        CacheConfigurator.ensureTimeoutOrdering(holder);
         if (JGroupsConfigurator.isClustered(holder)) {
             KeycloakModelUtils.runJobInTransaction(factory, session -> JGroupsConfigurator.configureJGroups(config, holder, session));
         }

@@ -104,6 +104,15 @@ public boolean isLoggable(LogRecord record) {
             }
         }
 
+        // ISPN000208 "No live owners found for segments" fires during state transfer when a node is the sole
+        // remaining owner after another node departed. The data is safe (this node has it), but the message
+        // is logged at ERROR and alarms users. With numOwners>=2 enforced by Keycloak, a single node
+        // departure always leaves at least one live owner, so this is expected during rolling restarts.
+        // https://github.com/keycloak/keycloak/issues/52088
+        if (Objects.equals(record.getLevel(), Level.SEVERE) && record.getLoggerName().equals("org.infinispan.CLUSTER") && record.getMessage().startsWith("ISPN000208")) {
+            return false;
+        }
+
         if (executor != null && ThreadCreator.isVirtual(Thread.currentThread())) {
             executor.submit(new RecordLogger(ExtLogRecord.wrap(record), this));
             return false;

@@ -147,6 +147,8 @@ void failInvalidClusterStack(CLIResult result) {
     void testExplicitCacheConfigFile(CLIResult result) {
         result.assertStarted();
         result.assertClusteredCache();
+        // CacheConfigurator.ensureTimeoutOrdering() validates lock-timeout < remote-timeout
+        result.assertNoMessage("must be less than remote-timeout");
     }
 
     @Test

@@ -251,7 +251,8 @@ protected void rollbackImpl() {
                     KeycloakModelUtils.runJobInTransaction(factory, s -> s.singleUseObjects().remove(lockId));
                 }
             });
-        }, Duration.of(10, ChronoUnit.SECONDS), 10);
+        // 12s allows 2 retries given the 5s remote-timeout on the actionTokens cache (CacheConfigurator).
+        }, Duration.of(12, ChronoUnit.SECONDS), 10);
     }
 
     /**