Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,10 @@ public int getBatchSizeForSync() {
return pageSizeConfig!=null ? Integer.parseInt(pageSizeConfig) : LDAPConstants.DEFAULT_BATCH_SIZE_FOR_SYNC;
}

public boolean isRemoveInvalidUsersOnSync() {
return Boolean.parseBoolean(config.getFirst(LDAPConstants.REMOVE_INVALID_USERS_ON_SYNC));
}

public String getUsernameLdapAttribute() {
String username = config.getFirst(LDAPConstants.USERNAME_LDAP_ATTRIBUTE);
if (username == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

import java.net.URI;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand Down Expand Up @@ -47,7 +49,9 @@
import org.keycloak.models.LDAPConstants;
import org.keycloak.models.ModelException;
import org.keycloak.models.RealmModel;
import org.keycloak.models.UserManager;
import org.keycloak.models.UserModel;
import org.keycloak.models.UserProvider;
import org.keycloak.models.cache.UserCache;
import org.keycloak.models.utils.KeycloakModelUtils;
import org.keycloak.provider.ProviderConfigProperty;
Expand Down Expand Up @@ -132,6 +136,10 @@ private static List<ProviderConfigProperty> getConfigProps(ComponentModel parent
.type(ProviderConfigProperty.BOOLEAN_TYPE)
.defaultValue("false")
.add()
.property().name(LDAPConstants.REMOVE_INVALID_USERS_ON_SYNC)
.type(ProviderConfigProperty.BOOLEAN_TYPE)
.defaultValue("false")
.add()
.property().name(LDAPConstants.VENDOR)
.type(ProviderConfigProperty.STRING_TYPE)
.add()
Expand Down Expand Up @@ -636,9 +644,18 @@ public SynchronizationResult sync(KeycloakSessionFactory sessionFactory, String
logger.infof("Sync all users from LDAP to local store: realm: %s, federation provider: %s", realmId, model.getName());

try (LDAPQuery userQuery = createQuery(sessionFactory, realmId, model)) {
SynchronizationResult syncResult = syncImpl(sessionFactory, userQuery, realmId, model);
boolean removeInvalidUsers = model.isImportEnabled() && new LDAPConfig(model.getConfig()).isRemoveInvalidUsersOnSync();

// Snapshot of imported users taken BEFORE the LDAP scan. Only these users are candidates for removal, so users
// imported concurrently (e.g. on login) or by this sync are never removed.
Map<String, String> candidates = removeInvalidUsers ? loadImportedUsers(sessionFactory, realmId, model) : null;
LdapScan scan = candidates != null && !candidates.isEmpty() ? new LdapScan() : null;

SynchronizationResult syncResult = syncImpl(sessionFactory, userQuery, realmId, model, scan);

// TODO: Remove all existing keycloak users, which have federation links, but are not in LDAP. Perhaps don't check users, which were just added or updated during this sync?
if (scan != null) {
removeUsersNotFoundInLdap(sessionFactory, realmId, model, candidates, scan, syncResult);
}

logger.infof("Sync all users finished: %s", syncResult.getStatus());
return syncResult;
Expand Down Expand Up @@ -689,6 +706,13 @@ public void run(KeycloakSession session) {
}

protected SynchronizationResult syncImpl(KeycloakSessionFactory sessionFactory, LDAPQuery userQuery, final String realmId, final ComponentModel fedModel) {
return syncImpl(sessionFactory, userQuery, realmId, fedModel, null);
}

/**
* @param scan if not {@code null}, the UUIDs (LDAP_ID) of all LDAP users returned by the query are collected into it
*/
protected SynchronizationResult syncImpl(KeycloakSessionFactory sessionFactory, LDAPQuery userQuery, final String realmId, final ComponentModel fedModel, LdapScan scan) {

final SynchronizationResult syncResult = new SynchronizationResult();

Expand All @@ -702,19 +726,184 @@ protected SynchronizationResult syncImpl(KeycloakSessionFactory sessionFactory,
userQuery.setLimit(pageSize);
final List<LDAPObject> users = userQuery.getResultList();
nextPage = userQuery.getPaginationContext().hasNextPage();
if (scan != null) {
scan.collect(users);
if (userQuery.getPaginationContext().isIncomplete()) {
scan.markIncomplete();
}
}
SynchronizationResult currentPageSync = importLdapUsers(sessionFactory, realmId, fedModel, users);
syncResult.add(currentPageSync);
}
} else {
// LDAP pagination not available. Do everything in single transaction
final List<LDAPObject> users = userQuery.getResultList();
if (scan != null) {
scan.collect(users);
}
Comment on lines +741 to +743
SynchronizationResult currentSync = importLdapUsers(sessionFactory, realmId, fedModel, users);
syncResult.add(currentSync);
}

return syncResult;
}

/**
* Result of the full LDAP user scan, used to detect users removed from LDAP.
*/
protected static class LdapScan {
private final Set<String> ldapIds = new HashSet<>();
private boolean complete = true;

void collect(List<LDAPObject> users) {
for (LDAPObject user : users) {
if (user.getUuid() == null) {
// we cannot tell which local user this entry belongs to, so we cannot safely remove anybody
complete = false;
} else {
ldapIds.add(user.getUuid());
}
}
}

void markIncomplete() {
complete = false;
}

boolean isComplete() {
return complete;
}

boolean contains(String ldapId) {
return ldapIds.contains(ldapId);
}

boolean isEmpty() {
return ldapIds.isEmpty();
}
}

/**
* Loads ids of local users linked to this provider together with their {@code LDAP_ID}, without validating them against LDAP.
*
* @return map userId -> LDAP_ID, or {@code null} if the local storage does not support this operation
*/
private Map<String, String> loadImportedUsers(KeycloakSessionFactory sessionFactory, String realmId, UserStorageProviderModel model) {
int pageSize = Math.max(new LDAPConfig(model.getConfig()).getBatchSizeForSync(), 1);
Map<String, String> result = new HashMap<>();
try {
int first = 0;
while (true) {
final int offset = first;
List<Map.Entry<String, String>> page = KeycloakModelUtils.runJobInTransactionWithResult(sessionFactory, session -> {
RealmModel realm = session.realms().getRealm(realmId);
session.getContext().setRealm(realm);
return UserStoragePrivateUtil.userLocalStorage(session)
.getImportedUserIdsWithAttributeStream(realm, model.getId(), LDAPConstants.LDAP_ID, offset, pageSize)
.collect(Collectors.toList());
});

for (Map.Entry<String, String> entry : page) {
// users without LDAP_ID are left untouched, we cannot reliably tell whether they exist in LDAP
if (entry.getValue() != null) {
result.put(entry.getKey(), entry.getValue());
}
}

if (page.size() < pageSize) {
return result;
}
first += pageSize;
}
} catch (UnsupportedOperationException e) {
logger.warnf("Local user storage does not support listing imported users. Skipping removal of invalid users for federation provider '%s'.", model.getName());
return null;
}
}

/**
* Removes users from the {@code candidates} snapshot whose {@code LDAP_ID} was not returned by the full LDAP user query.
* Each user is removed in its own transaction, so a failure of one removal does not affect the others.
* <p>
* As a safety measure nothing is removed when the LDAP scan may be incomplete (e.g. server-side size limit hit,
* entries without UUID) or returned no users at all (likely misconfiguration such as wrong Users DN or filter).
*/
protected void removeUsersNotFoundInLdap(KeycloakSessionFactory sessionFactory, String realmId, UserStorageProviderModel model,
Map<String, String> candidates, LdapScan scan, SynchronizationResult syncResult) {
if (!scan.isComplete()) {
logger.warnf("LDAP user query for federation provider '%s' in realm '%s' may have returned incomplete results. Skipping removal of invalid users to prevent accidental data loss.",
model.getName(), realmId);
return;
Comment on lines +833 to +836
}
if (scan.isEmpty()) {
logger.warnf("LDAP query returned no users for federation provider '%s' in realm '%s'. Skipping removal of invalid users to prevent accidental data loss.",
model.getName(), realmId);
return;
}

List<Map.Entry<String, String>> toRemove = candidates.entrySet().stream()
.filter(e -> !scan.contains(e.getValue()))
.collect(Collectors.toList());

if (toRemove.isEmpty()) {
return;
}

logger.infof("Removing %d users of federation provider '%s' in realm '%s', which no longer exist in LDAP", toRemove.size(), model.getName(), realmId);

for (Map.Entry<String, String> candidate : toRemove) {
String userId = candidate.getKey();
String ldapId = candidate.getValue();
try {
KeycloakModelUtils.runJobInTransaction(sessionFactory, session -> {
RealmModel realm = session.realms().getRealm(realmId);
session.getContext().setRealm(realm);
UserProvider localStorage = UserStoragePrivateUtil.userLocalStorage(session);
UserModel user = localStorage.getUserById(realm, userId);
if (user == null) {
return; // already removed concurrently
}
// re-check the user was not unlinked or re-linked to a different LDAP entry since the snapshot
if (!model.getId().equals(user.getFederationLink()) || !ldapId.equals(user.getFirstAttribute(LDAPConstants.LDAP_ID))) {
logger.debugf("Skipping removal of user '%s' (id: %s) as it changed during sync", user.getUsername(), userId);
return;
}
UserCache userCache = UserStorageUtil.userCache(session);
if (userCache != null) {
userCache.evict(realm, user);
}
publishUserPreRemovedEvent(session, realm, user);
if (new UserManager(session).removeUser(realm, user, localStorage)) {
logger.debugf("Removed user '%s' (id: %s), which no longer exists in LDAP", user.getUsername(), userId);
syncResult.increaseRemoved();
}
});
} catch (ModelException e) {
logger.warnf(e, "Failed to remove user with id '%s', which no longer exists in LDAP", userId);
syncResult.increaseFailed();
}
}
}

private static void publishUserPreRemovedEvent(KeycloakSession session, RealmModel realm, UserModel user) {
session.getKeycloakSessionFactory().publish(new UserModel.UserPreRemovedEvent() {
@Override
public RealmModel getRealm() {
return realm;
}

@Override
public UserModel getUser() {
return user;
}

@Override
public KeycloakSession getKeycloakSession() {
return session;
}
});
}

/**
* !! This function must be called from try-with-resources block, otherwise Vault secrets may be leaked !!
* @param sessionFactory
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ public static class PaginationContext {

private final LdapContext ldapContext;
private byte[] cookie;
private boolean incomplete;

private PaginationContext(LdapContext ldapContext) {
if (ldapContext == null) {
Expand All @@ -283,6 +284,18 @@ public void setCookie(byte[] cookie) {
public boolean hasNextPage() {
return this.cookie != null;
}

/**
* Marks the paginated search as possibly incomplete, e.g. when the server did not return the paged results
* response control (typically because a server-side size limit was hit).
*/
public void markIncomplete() {
this.incomplete = true;
}

public boolean isIncomplete() {
return incomplete;
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ public List<SearchResult> execute(LdapContext context) throws NamingException {
* See KEYCLOAK-19036
*/
identityQuery.getPaginationContext().setCookie(null);
identityQuery.getPaginationContext().markIncomplete();
logger.warnf("Did not receive response controls for paginated query using DN [%s], filter [%s]. Did you hit a query result size limit?", baseDN, filter);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3946,6 +3946,8 @@ recentlyUsed=Recently used
viewAll=View all
currentRealmExplain=This realm is selected
removeInvalidUsers=Remove invalid users during searches
removeInvalidUsersOnSync=Remove invalid users during full sync
removeInvalidUsersOnSyncHelp=If enabled, a full synchronization (manual or periodic) deletes users from the local database that are linked to this provider but are no longer returned by the LDAP user query (for example, users deleted in LDAP or excluded by the 'Users DN' or custom user LDAP filter). Changed users sync never removes users. Removal is skipped if LDAP returns no users at all. Be aware that this may be a costly operation with many imported users, and that a misconfigured provider may remove many users. Only applies when 'Import users' is enabled.
removeInvalidUsersHelp=Remove users from the local database if they are not available from the user storage when executing searches. If this is true, users no longer available from their corresponding user storage will be deleted from the local database whenever trying to look up users. If false, then users previously imported from the user storage will be kept in the local database, as read-only and disabled, even if that user is no longer available from the user storage. For example, user was deleted directly from LDAP or the `Users DN` is invalid. Note that this behavior will only happen when the user is not yet cached.
createPermissionPolicy=Create policy
enforceAccessTo=Enforce access to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export const LdapSettingsSynchronization = ({

const watchPeriodicSync = form.watch("config.periodicFullSync", false);
const watchChangedSync = form.watch("config.periodicChangedUsersSync", false);
const watchImportEnabled = form.watch("config.importEnabled", ["true"]);

return (
<FormProvider {...form}>
Expand Down Expand Up @@ -204,6 +205,35 @@ export const LdapSettingsSynchronization = ({
)}
/>
</FormGroup>
<FormGroup
label={t("removeInvalidUsersOnSync")}
labelIcon={
<HelpItem
helpText={t("removeInvalidUsersOnSyncHelp")}
fieldLabelId="removeInvalidUsersOnSync"
/>
}
fieldId="kc-remove-invalid-users-on-sync"
hasNoPaddingTop
>
<Controller
name="config.removeInvalidUsersOnSync"
defaultValue={["false"]}
control={form.control}
render={({ field }) => (
<Switch
id="kc-remove-invalid-users-on-sync"
data-testid="remove-invalid-users-on-sync"
isDisabled={watchImportEnabled?.[0] !== "true"}
onChange={(_event, value) => field.onChange([`${value}`])}
isChecked={field.value[0] === "true"}
label={t("on")}
labelOff={t("off")}
aria-label={t("removeInvalidUsersOnSync")}
/>
)}
/>
</FormGroup>
</FormAccess>
</FormProvider>
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.keycloak.models.jpa;

import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
Expand Down Expand Up @@ -643,6 +644,21 @@ public void unlinkUsers(RealmModel realm, String storageProviderId) {
.executeUpdate();
}

@Override
public Stream<Map.Entry<String, String>> getImportedUserIdsWithAttributeStream(RealmModel realm, String storageProviderId,
String attributeName, Integer firstResult, Integer maxResults) {
TypedQuery<Object[]> query = em.createQuery(
// longValue is selected separately (not via coalesce) because it may be mapped to a LOB column on some databases
"select u.id, attr.value, attr.longValue from UserEntity u left join u.attributes attr on attr.name = :name " +
"where u.realmId = :realmId and u.federationLink = :link order by u.id", Object[].class)
.setParameter("name", attributeName)
.setParameter("realmId", realm.getId())
.setParameter("link", storageProviderId);

return closing(paginateQuery(query, firstResult, maxResults).getResultStream())
.map(row -> new AbstractMap.SimpleImmutableEntry<>((String) row[0], row[1] != null ? (String) row[1] : (String) row[2]));
}

@Override
public void preRemove(RealmModel realm, RoleModel role) {
em.createNamedQuery("deleteUserRoleMappingsByRole").setParameter("roleId", role.getId()).executeUpdate();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1181,6 +1181,12 @@ public void unlinkUsers(RealmModel realm, String storageProviderId) {
localStorage().unlinkUsers(realm, storageProviderId);
}

@Override
public Stream<Map.Entry<String, String>> getImportedUserIdsWithAttributeStream(RealmModel realm, String storageProviderId,
String attributeName, Integer firstResult, Integer maxResults) {
return localStorage().getImportedUserIdsWithAttributeStream(realm, storageProviderId, attributeName, firstResult, maxResults);
}

/** {@link UserProvider} methods implementations end here */

@Override
Expand Down
Loading