From ebef70267ab589f457655991c4e3ca36c7279e22 Mon Sep 17 00:00:00 2001 From: Ryan Emerson Date: Fri, 20 Jun 2025 11:43:09 +0100 Subject: [PATCH 1/3] Default to num_owners=2 when the persistent-user-sessions feature is disabled Closes #40472 Signed-off-by: Ryan Emerson --- .../upgrading/topics/changes/changes-26_2_6.adoc | 6 ++++++ .../runtime/storage/infinispan/CacheManagerFactory.java | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc b/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc index 91bd367cf166..a8fb1b524f36 100644 --- a/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc +++ b/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc @@ -16,3 +16,9 @@ If you are upgrading {project_name} by downloading the distribution, the upgrade The new procedure recommends to re-apply any changes to `cache-ispn.xml` or a custom cache configuration based on the file included in the new version. This prevents accidentally downgrading functionality, for example, by using an old `cache-ispn.xml` file from a previous version. + +=== Volatile Session Cache Defaults + +If the `persistent-user-sessions` feature is disabled, i.e. volatile sessions are configured, {project_name} now prevents +`num_owners=1` being configured unless a shared persistent store is also configured. This is to prevent data loss on cache +rebalances. diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java index f26b8b2392ff..185181ce2028 100644 --- a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java @@ -39,6 +39,7 @@ import org.infinispan.commons.dataconversion.MediaType; import org.infinispan.commons.internal.InternalCacheNames; import org.infinispan.commons.util.concurrent.CompletableFutures; +import org.infinispan.configuration.cache.AbstractStoreConfiguration; import org.infinispan.configuration.cache.CacheMode; import org.infinispan.configuration.cache.ConfigurationBuilder; import org.infinispan.configuration.cache.HashConfiguration; @@ -480,9 +481,11 @@ private static void configureSessionsCaches(ConfigurationBuilderHolder builder) logger.warnf("Persistent user sessions disabled and memory limit is set to default value 10000. Ignoring cache limits to avoid losing sessions for cache %s.", cacheName); configurationBuilder.memory().maxCount(-1); } - if (configurationBuilder.clustering().hash().attributes().attribute(HashConfiguration.NUM_OWNERS).get() == 1 - && configurationBuilder.persistence().stores().isEmpty()) { - logger.warnf("Number of owners is one for cache %s, and no persistence is configured. This might be a misconfiguration as you will lose data when a single node is restarted!", cacheName); + if (configurationBuilder.clustering().hash().attributes().attribute(HashConfiguration.NUM_OWNERS).get() == 1 && + configurationBuilder.persistence().stores().stream().noneMatch(p -> p.attributes().attribute(AbstractStoreConfiguration.SHARED).get()) + ) { + logger.warnf("Persistent user sessions disabled with number of owners set to default value 1 for cache %s and no shared persistence store configured. Setting num_owners=2 to avoid data loss.", cacheName); + configurationBuilder.clustering().hash().numOwners(2); } } }); From ba59861381074899b3426b86f7599f2459c5c37b Mon Sep 17 00:00:00 2001 From: Ryan Emerson Date: Fri, 20 Jun 2025 11:48:54 +0100 Subject: [PATCH 2/3] Disabling persistent sessions affects also offline session cache Closes #40483 Signed-off-by: Alexander Schwartz Signed-off-by: Ryan Emerson Co-authored-by: Ryan Emerson --- .../topics/changes/changes-26_2_6.adoc | 5 ++ docs/guides/server/caching.adoc | 70 +++++++++++-------- .../InfinispanUserSessionProviderFactory.java | 4 +- .../infinispan/CacheManagerFactory.java | 5 +- 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc b/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc index a8fb1b524f36..610d31a65a51 100644 --- a/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc +++ b/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc @@ -17,6 +17,11 @@ The new procedure recommends to re-apply any changes to `cache-ispn.xml` or a cu This prevents accidentally downgrading functionality, for example, by using an old `cache-ispn.xml` file from a previous version. +=== Problematic cache configurations ignored + +Previous versions {project_name} warned about problematic configurations, for example, if a wrong number of owners was configured or a cache size was set when it should not have been set when enabling volatile user sessions. +The current version will still warn about those settings, but will also ignore them to ensure a correct behavior and to avoid data loss. + === Volatile Session Cache Defaults If the `persistent-user-sessions` feature is disabled, i.e. volatile sessions are configured, {project_name} now prevents diff --git a/docs/guides/server/caching.adoc b/docs/guides/server/caching.adoc index 9c585215014b..c05c0cdf0916 100644 --- a/docs/guides/server/caching.adoc +++ b/docs/guides/server/caching.adoc @@ -108,21 +108,50 @@ CPU, memory, and network utilization. These in-memory caches for user sessions and client sessions are limited to, by default, 10000 entries per node which reduces the overall memory usage of {project_name} for larger installations. The internal caches will run with only a single owner for each cache entry. -Consider trade-off between memory consumption and the database utilization and set different sizes for the caches, edit {project_name}'s cache config file (`conf/cache-ispn.xml`) to set a `++` for those caches. -.Volatile user sessions +.Offline user sessions +As an OpenID Connect Provider, the server is capable of authenticating users and issuing offline tokens. When issuing an offline token after successful authentication, the server creates an offline user session and offline client session. + +The following caches are used to store offline sessions: + +* offlineSessions +* offlineClientSessions + +Like the user and client sessions caches, the offline user and client session caches are limited to 10000 entries per node by default. Items which are evicted from the memory will be loaded on-demand from the database when needed. + +.Password brute force detection +The `loginFailures` distributed cache is used to track data about failed login attempts. +This cache is needed for the Brute Force Protection feature to work in a multi-node {project_name} setup. + +.Action tokens +Action tokens are used for scenarios when a user needs to confirm an action asynchronously, for example in the emails sent by the forgot password flow. +The `actionTokens` distributed cache is used to track metadata about action tokens. -By default, user sessions are stored in the database and loaded on-demand to the cache. -It is possible to configure {project_name} to store user sessions in the cache only and minimize the database utilization. +=== Volatile user sessions + +By default, regular user sessions are stored in the database and loaded on-demand to the cache. +It is possible to configure {project_name} to store regular user sessions in the cache only and minimize calls to the database. Since all the sessions in this setup are stored in-memory, there are two side effects related to this: -* Losing sessions on when all {project_name} nodes restart. +* Losing sessions when all {project_name} nodes restart. * Increased memory consumption. +When using volatile user sessions, the cache is the source of truth for user and client sessions. +Therefore, it is necessary to not define an upper limit on the total number of entries that can be stored via the setting `max-count`. +Similarly, it's crucial that we replicate each entry to two nodes to prevent data loss on node failures and rolling restarts. + +[WARNING] +==== +When using volatile user sessions, the `max-count` setting for the offline session is ignored. + +It is not recommended to use volatile user sessions when using offline sessions extensively due to potentially high memory usage. +For volatile sessions, the time offline sessions are cached in memory can be limited with the SPI options `spi-user-sessions--infinispan--offline-client-session-cache-entry-lifespan-override` and `spi-user-sessions--infinispan--offline-session-cache-entry-lifespan-override`. +==== + Follow these steps to enable this setup: -1. Since the cache is the only source of truth for user and client sessions, configure caches to not limit the number of entries and to replicate each entry to at least two nodes. To do so, edit {project_name}'s cache config file (`conf/cache-ispn.xml`) for caches `sessions` and `clientSessions` with the following update: +1. Edit {project_name}'s cache config file (`conf/cache-ispn.xml`) for caches `sessions`, `clientSessions`, `offlineSessions` and `offlineClientSessions` with the following update: + -- * Remove the `++` @@ -148,27 +177,8 @@ bin/kc.sh start --features-disabled=persistent-user-sessions ... Disabling `persistent-user-sessions` is not possible when `multi-site` feature is enabled. ==== -.Offline user sessions -As an OpenID Connect Provider, the server is also capable of authenticating users and issuing offline tokens. Similarly to regular user and client sessions, -when an offline token is issued by the server upon successful authentication, the server also creates an offline user session and an offline client session. - -The following caches are used to store offline sessions: - -* offlineSessions -* offlineClientSessions - -Similarly to regular user and client sessions caches, also the offline user and client session caches are limited to 10000 entries per node by default. Items which are evicted from the memory will be loaded on-demand from the database when needed. -Consider trade-off between memory consumption and the database utilization and set different sizes for the caches, edit {project_name}'s cache config file (`conf/cache-ispn.xml`) to set a `++` for those caches. - -.Password brute force detection -The `loginFailures` distributed cache is used to track data about failed login attempts. -This cache is needed for the Brute Force Protection feature to work in a multi-node {project_name} setup. - -.Action tokens -Action tokens are used for scenarios when a user needs to confirm an action asynchronously, for example in the emails sent by the forgot password flow. -The `actionTokens` distributed cache is used to track metadata about action tokens. - === Configuring cache maximum size + In order to reduce memory usage, it's possible to place an upper bound on the number of entries which are stored in a given cache. To specify an upper bound of on a cache, you must provide the following command line argument `--cache-embedded-$\{CACHE_NAME}-max-count=`, with `$\{CACHE_NAME}` replaced with the name of the cache you would like to @@ -176,16 +186,20 @@ apply the upper bound to. For example, to apply an upper-bound of `1000` to the `--cache-embedded-offline-sessions-max-count=1000`. An upper bound can not be defined on the following caches: `actionToken`, `authenticationSessions`, `loginFailures`, `work`. +Setting a maximum cache size for `sessions`, `clientSessions`, `offlineSessions` and `offlineClientSessions` is not supported when volatile sessions are enabled. + === Configuring caches for availability Distributed caches replicate cache entries on a subset of nodes in a cluster and assigns entries to fixed owner nodes. Each distributed cache, that is a primary source of truth of the data (`authenticationSessions`, `loginFailures` and `actionTokens`) has two owners per default, which means that two nodes have a copy of the specific cache entries. Non-owner nodes query the owners of a specific cache to obtain data. +When one of the owners becomes unavailable, the data is restored from the remaining owner and rebalanced across the remaining nodes. When both owner nodes are offline, all data is lost. -The default number of owners is enough to survive 1 node (owner) failure in a cluster setup with at least three nodes. You are free -to change the number of owners accordingly to better fit into your availability requirements. To change the number of owners, open `conf/cache-ispn.xml` and change the value for `owners=` for the distributed caches to your desired value. +The default number of two owners is the minimum number is necessary to survive one node (owner) failure or a rolling restart in a cluster setup with at least two nodes. +A higher number increases the availability of the data, but at the expense of slower writes as more nodes need to be updated. +Therefore, changing the number of owners for the caches `authenticationSessions`, `loginFailures` and `actionTokens` is not recommended. === Specify your own cache configuration file diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java index 7d47d2281e48..ef08e46cedfa 100755 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java @@ -482,13 +482,13 @@ public List getConfigMetadata() { builder.property() .name(CONFIG_OFFLINE_CLIENT_SESSION_CACHE_ENTRY_LIFESPAN_OVERRIDE) .type("int") - .helpText("Override how long offline client sessions should be kept in memory") + .helpText("Override how long offline client sessions should be kept in memory in seconds") .add(); builder.property() .name(CONFIG_OFFLINE_SESSION_CACHE_ENTRY_LIFESPAN_OVERRIDE) .type("int") - .helpText("Override how long offline user sessions should be kept in memory") + .helpText("Override how long offline user sessions should be kept in memory in seconds") .add(); builder.property() diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java index 185181ce2028..7bd293132af0 100644 --- a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java @@ -475,10 +475,7 @@ private static void configureSessionsCaches(ConfigurationBuilderHolder builder) configurationBuilder.clustering().hash().numOwners(1); } else { if (configurationBuilder.memory().maxCount() != -1) { - logger.warnf("Persistent user sessions disabled and memory limit found in configuration for cache %s. This might be a misconfiguration! Update your Infinispan configuration to remove this message.", cacheName); - } - if (configurationBuilder.memory().maxCount() == 10000 && (cacheName.equals(USER_SESSION_CACHE_NAME) || cacheName.equals(CLIENT_SESSION_CACHE_NAME))) { - logger.warnf("Persistent user sessions disabled and memory limit is set to default value 10000. Ignoring cache limits to avoid losing sessions for cache %s.", cacheName); + logger.warnf("Persistent user sessions disabled and memory limit is set. Ignoring cache limits to avoid losing sessions for cache %s.", cacheName); configurationBuilder.memory().maxCount(-1); } if (configurationBuilder.clustering().hash().attributes().attribute(HashConfiguration.NUM_OWNERS).get() == 1 && From ae0644292abd31c9bd68b3a9bda2fffdd97d6e87 Mon Sep 17 00:00:00 2001 From: Pedro Ruivo Date: Fri, 20 Jun 2025 18:45:22 +0200 Subject: [PATCH 3/3] Update documentation about volatile sessions Closes #40639 Signed-off-by: Pedro Ruivo Signed-off-by: Alexander Schwartz Co-authored-by: Alexander Schwartz --- .../topics/changes/changes-26_2_6.adoc | 12 +++++----- docs/guides/server/caching.adoc | 22 ++----------------- .../infinispan/CacheManagerFactory.java | 4 ++-- 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc b/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc index 610d31a65a51..f3595d67aa92 100644 --- a/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc +++ b/docs/documentation/upgrading/topics/changes/changes-26_2_6.adoc @@ -19,11 +19,9 @@ This prevents accidentally downgrading functionality, for example, by using an o === Problematic cache configurations ignored -Previous versions {project_name} warned about problematic configurations, for example, if a wrong number of owners was configured or a cache size was set when it should not have been set when enabling volatile user sessions. -The current version will still warn about those settings, but will also ignore them to ensure a correct behavior and to avoid data loss. +Previous versions of {project_name} warned about problematic configurations, for example, if a wrong number of owners was configured or a cache size was set when it should not have been set when enabling volatile user sessions. +The docs also stated to update the `cache-ispn.xml` configuration file for volatile user sessions. -=== Volatile Session Cache Defaults - -If the `persistent-user-sessions` feature is disabled, i.e. volatile sessions are configured, {project_name} now prevents -`num_owners=1` being configured unless a shared persistent store is also configured. This is to prevent data loss on cache -rebalances. +The current version will always use safe settings for the number of owners and maximum cache size for the affected user and client session caches, and will log only an INFO message. +With this behavior, there is no need any more to update the `cache-ispn.xml` configuration file. +If you previously used a custom `cache-ispn.xml` in order to use volatile user sessions, we recommend reverting those changes and use the standard configuration file. diff --git a/docs/guides/server/caching.adoc b/docs/guides/server/caching.adoc index c05c0cdf0916..c8fa33b6f4f8 100644 --- a/docs/guides/server/caching.adoc +++ b/docs/guides/server/caching.adoc @@ -138,35 +138,17 @@ Since all the sessions in this setup are stored in-memory, there are two side ef * Increased memory consumption. When using volatile user sessions, the cache is the source of truth for user and client sessions. -Therefore, it is necessary to not define an upper limit on the total number of entries that can be stored via the setting `max-count`. -Similarly, it's crucial that we replicate each entry to two nodes to prevent data loss on node failures and rolling restarts. +{project_name} automatically adjusts the number of entries that can be stored in memory, and increases the number of copies to prevent data loss. [WARNING] ==== -When using volatile user sessions, the `max-count` setting for the offline session is ignored. - It is not recommended to use volatile user sessions when using offline sessions extensively due to potentially high memory usage. For volatile sessions, the time offline sessions are cached in memory can be limited with the SPI options `spi-user-sessions--infinispan--offline-client-session-cache-entry-lifespan-override` and `spi-user-sessions--infinispan--offline-session-cache-entry-lifespan-override`. ==== Follow these steps to enable this setup: -1. Edit {project_name}'s cache config file (`conf/cache-ispn.xml`) for caches `sessions`, `clientSessions`, `offlineSessions` and `offlineClientSessions` with the following update: -+ --- -* Remove the `++` -* Change `owners` attribute of the `distributed-cache` tag to 2 or more --- -+ -An example of the resulting configuration for the `sessions` cache would look as follows. -+ -[source,xml] ----- - - - ----- -2. Disable `persistent-user-sessions` feature using the following command: +1. Disable `persistent-user-sessions` feature using the following command: + ---- bin/kc.sh start --features-disabled=persistent-user-sessions ... diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java index 7bd293132af0..614d4862c28b 100644 --- a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/storage/infinispan/CacheManagerFactory.java @@ -475,13 +475,13 @@ private static void configureSessionsCaches(ConfigurationBuilderHolder builder) configurationBuilder.clustering().hash().numOwners(1); } else { if (configurationBuilder.memory().maxCount() != -1) { - logger.warnf("Persistent user sessions disabled and memory limit is set. Ignoring cache limits to avoid losing sessions for cache %s.", cacheName); + logger.infof("Persistent user sessions disabled and memory limit is set. Ignoring cache limits to avoid losing sessions for cache %s.", cacheName); configurationBuilder.memory().maxCount(-1); } if (configurationBuilder.clustering().hash().attributes().attribute(HashConfiguration.NUM_OWNERS).get() == 1 && configurationBuilder.persistence().stores().stream().noneMatch(p -> p.attributes().attribute(AbstractStoreConfiguration.SHARED).get()) ) { - logger.warnf("Persistent user sessions disabled with number of owners set to default value 1 for cache %s and no shared persistence store configured. Setting num_owners=2 to avoid data loss.", cacheName); + logger.infof("Persistent user sessions disabled with number of owners set to default value 1 for cache %s and no shared persistence store configured. Setting num_owners=2 to avoid data loss.", cacheName); configurationBuilder.clustering().hash().numOwners(2); } }