Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/documentation/upgrading/topics/migrate_db.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,16 @@ kc.[sh|bat] start --spi-connections-jpa--quarkus--migration-export=<path>/<file.

After the changes have been written to the file, the server exits.

For an empty database the script also creates the Liquibase `DATABASECHANGELOG` table, which it records the applied
changesets in. It does not create the `DATABASECHANGELOGLOCK` table: {project_name} creates that one itself on
startup, so it is missing from a database that was only ever set up by running the script.

Keep this in mind when the database user is not allowed to create tables. Even with the manual strategy,
{project_name} creates both Liquibase bookkeeping tables, `DATABASECHANGELOG` and `DATABASECHANGELOGLOCK`, on the
database it is started against. Have an administrator create them up front, both on the database used to generate
the script and on the database the script is applied to. When they already exist, the generated script contains no
`CREATE TABLE` for them, so it can also be applied to the database it was generated from.

For further details on how to run the statements in the file against the database, see the documentation for your relational database.

Once you have run the statements, you can start the new {project_name} version.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import liquibase.snapshot.SnapshotGeneratorFactory;
import liquibase.statement.SqlStatement;
import liquibase.statement.core.AddColumnStatement;
import liquibase.statement.core.CreateDatabaseChangeLogTableStatement;
import liquibase.statement.core.SetNullableStatement;
import liquibase.statement.core.UpdateStatement;
import liquibase.structure.core.Column;
Expand All @@ -68,10 +69,14 @@ public class LiquibaseJpaUpdaterProvider implements JpaUpdaterProvider {
private static final Logger logger = Logger.getLogger(LiquibaseJpaUpdaterProvider.class);

public static final String CHANGELOG = "META-INF/jpa-changelog-master.xml";
private static final String MASTER_CHANGELOG_TABLE = "DATABASECHANGELOG";

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The map is gone, replaced by a single flag for the master table.


public static final String DEPLOYMENT_ID_COLUMN = "DEPLOYMENT_ID";

private final KeycloakSession session;
// Did the master changelog table already exist when this server started, before validation created it?
// Only the master table is relevant: Liquibase still writes the DDL of custom provider changelog tables itself.
private Boolean masterChangelogTablePreExisted;

public LiquibaseJpaUpdaterProvider(KeycloakSession session) {
this.session = session;
Expand Down Expand Up @@ -136,6 +141,11 @@ protected void updateChangeSet(KeycloakLiquibase liquibase, Writer exportWriter)
String changelog = liquibase.getChangeLogFile();
Database database = liquibase.getDatabase();
Table changelogTable = SnapshotGeneratorFactory.getInstance().getDatabaseChangeLogTable(new SnapshotControl(database, false, Table.class, Column.class), database);
// Validation is not always run before an update, and nothing has touched the changelog table yet at this
// point, so record its state here as well.
if (masterChangelogTablePreExisted == null && isMasterChangelogTable(database)) {
masterChangelogTablePreExisted = changelogTable != null;
}

if (changelogTable != null) {
boolean hasDeploymentIdColumn = changelogTable.getColumn(DEPLOYMENT_ID_COLUMN) != null;
Expand Down Expand Up @@ -210,10 +220,22 @@ private void outputChangeLogTableCreationScript(Liquibase liquibase, final Write
loggingExecutor.comment("* Keycloak database creation script - apply this script to empty DB *");
loggingExecutor.comment("*********************************************************************" + StreamUtil.getLineSeparator());

// DatabaseChangeLogTable is automatically added to the script by Liquibase
// Liquibase adds the changelog table DDL to the script itself, but only while the table does not exist yet.
// Since Liquibase 4.33 parsing a changelog initializes the change log history service (DatabaseChangeLog
// creates the table on the live connection), so by the time the export is written the table is already
// there and Liquibase no longer emits it - leaving a script that inserts into a table it never creates,
// even though its header offers it as a creation script for an empty database. Emit the statement here
// instead, but only when the table was absent before this server started: an administrator may have
// pre-created it so that a user without DDL rights could produce the script at all, and in that case the
// target database already has the table and the DDL would fail the script with "already exists".
// DatabaseChangeLogLockTable is created before this code is executed and recreated if it does not exist automatically
// in org.keycloak.connections.jpa.updater.liquibase.lock.CustomLockService.init() called indirectly from
// KeycloakApplication constructor (search for waitForLock() call). Hence it is not included in the creation script.
if (isMasterChangelogTable(database) && Boolean.FALSE.equals(masterChangelogTablePreExisted)) {
// Same comment Liquibase emits when it writes this statement itself
loggingExecutor.comment("Create Database Change Log Table");
loggingExecutor.execute(new CreateDatabaseChangeLogTableStatement());
}
Comment on lines +234 to +238

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as in the Quarkus provider: the only caller is updateChangeSet, which sets the flag before this runs.


// For MySQL, add primary key to DATABASECHANGELOG table (handled by MySQLCustomChangeLogHistoryService at runtime)
ChangeLogHistoryService changeLogHistoryService = ChangeLogHistoryServiceFactory.getInstance().getChangeLogService(database);
Expand Down Expand Up @@ -270,6 +292,7 @@ protected Status validateSynch(final Connection connection, final String default

protected Status validateChangeSet(KeycloakLiquibase liquibase, String changelog) throws LiquibaseException {
final Status result;
recordWhetherMasterChangelogTablePreExisted(liquibase.getDatabase());
List<ChangeSet> changeSets = getLiquibaseUnrunChangeSets(liquibase);

if (!changeSets.isEmpty()) {
Expand All @@ -296,6 +319,18 @@ private void resetLiquibaseServices(KeycloakLiquibase liquibase) {
ChangeLogHistoryServiceFactory.getInstance().register(new MySQLCustomChangeLogHistoryService());
}

private void recordWhetherMasterChangelogTablePreExisted(Database database) throws LiquibaseException {
if (masterChangelogTablePreExisted == null && isMasterChangelogTable(database)) {
Table table = SnapshotGeneratorFactory.getInstance()
.getDatabaseChangeLogTable(new SnapshotControl(database, false, Table.class, Column.class), database);
masterChangelogTablePreExisted = table != null;
}
}

private static boolean isMasterChangelogTable(Database database) {
return MASTER_CHANGELOG_TABLE.equalsIgnoreCase(database.getDatabaseChangeLogTableName());
}

private List<ChangeSet> getLiquibaseUnrunChangeSets(Liquibase liquibase) throws LiquibaseException {
return liquibase.listUnrunChangeSets(null, new LabelExpression(), false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import liquibase.snapshot.SnapshotGeneratorFactory;
import liquibase.statement.SqlStatement;
import liquibase.statement.core.AddColumnStatement;
import liquibase.statement.core.CreateDatabaseChangeLogTableStatement;
import liquibase.statement.core.SetNullableStatement;
import liquibase.statement.core.UpdateStatement;
import liquibase.structure.core.Column;
Expand All @@ -68,11 +69,15 @@ public class QuarkusJpaUpdaterProvider implements JpaUpdaterProvider {
private static final Logger logger = Logger.getLogger(QuarkusJpaUpdaterProvider.class);

public static final String CHANGELOG = "META-INF/jpa-changelog-master.xml";
private static final String MASTER_CHANGELOG_TABLE = "DATABASECHANGELOG";

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The map is gone, replaced by a single flag for the master table.

private static final String DEPLOYMENT_ID_COLUMN = "DEPLOYMENT_ID";
public static final String VERIFY_AND_RUN_MASTER_CHANGELOG = "VERIFY_AND_RUN_MASTER_CHANGELOG";

private final KeycloakSession session;
private Map<String, List<ChangeSet>> changeSets = new HashMap<>();
// Did the master changelog table already exist when this server started, before validation created it?
// Only the master table is relevant: Liquibase still writes the DDL of custom provider changelog tables itself.
private Boolean masterChangelogTablePreExisted;

public QuarkusJpaUpdaterProvider(KeycloakSession session) {
this.session = session;
Expand Down Expand Up @@ -139,6 +144,11 @@ protected void updateChangeSet(KeycloakLiquibase liquibase, Writer exportWriter)
String changelog = liquibase.getChangeLogFile();
Database database = liquibase.getDatabase();
Table changelogTable = SnapshotGeneratorFactory.getInstance().getDatabaseChangeLogTable(new SnapshotControl(database, false, Table.class, Column.class), database);
// Validation is not always run before an update, and nothing has touched the changelog table yet at this
// point, so record its state here as well.
if (masterChangelogTablePreExisted == null && isMasterChangelogTable(database)) {
masterChangelogTablePreExisted = changelogTable != null;
}

if (changelogTable != null) {
boolean hasDeploymentIdColumn = changelogTable.getColumn(DEPLOYMENT_ID_COLUMN) != null;
Expand Down Expand Up @@ -213,10 +223,22 @@ private void outputChangeLogTableCreationScript(Liquibase liquibase, final Write
loggingExecutor.comment("* Keycloak database creation script - apply this script to empty DB *");
loggingExecutor.comment("*********************************************************************" + StreamUtil.getLineSeparator());

// DatabaseChangeLogTable is automatically added to the script by Liquibase
// Liquibase adds the changelog table DDL to the script itself, but only while the table does not exist yet.
// Since Liquibase 4.33 parsing a changelog initializes the change log history service (DatabaseChangeLog
// creates the table on the live connection), so by the time the export is written the table is already
// there and Liquibase no longer emits it - leaving a script that inserts into a table it never creates,
// even though its header offers it as a creation script for an empty database. Emit the statement here
// instead, but only when the table was absent before this server started: an administrator may have
// pre-created it so that a user without DDL rights could produce the script at all, and in that case the
// target database already has the table and the DDL would fail the script with "already exists".
// DatabaseChangeLogLockTable is created before this code is executed and recreated if it does not exist automatically
// in org.keycloak.connections.jpa.updater.liquibase.lock.CustomLockService.init() called indirectly from
// KeycloakApplication constructor (search for waitForLock() call). Hence it is not included in the creation script.
if (isMasterChangelogTable(database) && Boolean.FALSE.equals(masterChangelogTablePreExisted)) {
// Same comment Liquibase emits when it writes this statement itself
loggingExecutor.comment("Create Database Change Log Table");
loggingExecutor.execute(new CreateDatabaseChangeLogTableStatement());
}
Comment on lines +237 to +241

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It can't be null here: this method's only caller is updateChangeSet, which records the flag for the master table at its start, on the same Database, before reaching this call. ManualMigrationEmptyDatabaseDistTest covers exactly this path.


// For MySQL, add primary key to DATABASECHANGELOG table (handled by MySQLCustomChangeLogHistoryService at runtime)
ChangeLogHistoryService changeLogHistoryService = ChangeLogHistoryServiceFactory.getInstance().getChangeLogService(database);
Expand Down Expand Up @@ -268,6 +290,7 @@ public Status validate(Connection connection, String defaultSchema) {

protected Status validateChangeSet(KeycloakLiquibase liquibase, String changelog) throws LiquibaseException {
final Status result;
recordWhetherMasterChangelogTablePreExisted(liquibase.getDatabase());
List<ChangeSet> changeSets = getLiquibaseUnrunChangeSets(liquibase);

if (!changeSets.isEmpty()) {
Expand Down Expand Up @@ -298,6 +321,18 @@ private ChangeLogHistoryServiceFactory getChangeLogHistoryService() {
return Scope.getCurrentScope().getSingleton(ChangeLogHistoryServiceFactory.class);
}

private void recordWhetherMasterChangelogTablePreExisted(Database database) throws LiquibaseException {
if (masterChangelogTablePreExisted == null && isMasterChangelogTable(database)) {
Table table = SnapshotGeneratorFactory.getInstance()
.getDatabaseChangeLogTable(new SnapshotControl(database, false, Table.class, Column.class), database);
masterChangelogTablePreExisted = table != null;
}
}

private static boolean isMasterChangelogTable(Database database) {
return MASTER_CHANGELOG_TABLE.equalsIgnoreCase(database.getDatabaseChangeLogTableName());
}

private List<ChangeSet> getLiquibaseUnrunChangeSets(Liquibase liquibase) {
// we don't need to fetch change sets if they were previously obtained
return changeSets.computeIfAbsent(liquibase.getChangeLogFile(), s -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
import org.junit.jupiter.api.Test;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.number.OrderingComparison.lessThan;
import static org.junit.jupiter.api.Assertions.assertTrue;

@DistributionTest
Expand Down Expand Up @@ -66,5 +68,14 @@ void testCustomProviderChangesetExportedInManualMode(KeycloakRunner runner, RawD
}

assertThat(output, containsString("MIGRATION_TEST_ENTITY"));

// The custom provider gets its own changelog table, which the script inserts into, so it has to create
// it as well - exactly once, and before the first insert.
String createTable = "CREATE TABLE PUBLIC.DATABASECHANGELOG_MIGRATION_ (";
String insertInto = "INSERT INTO PUBLIC.DATABASECHANGELOG_MIGRATION_ (";
assertThat("custom changelog table must be created exactly once",
output.split(java.util.regex.Pattern.quote(createTable), -1).length - 1, is(1));
assertThat("custom changelog table must be created before it is inserted into",
output.indexOf(createTable), lessThan(output.indexOf(insertInto)));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright 2026 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.keycloak.it.cli.dist;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.regex.Pattern;

import org.keycloak.it.junit5.extension.CLIResult;
import org.keycloak.it.junit5.extension.DistributionTest;
import org.keycloak.it.junit5.extension.KeycloakRunner;
import org.keycloak.it.junit5.extension.RawDistOnly;
import org.keycloak.it.utils.RawDistRootPath;
import org.keycloak.it.utils.RawKeycloakDistribution;

import io.quarkus.deployment.util.FileUtil;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.number.OrderingComparison.lessThan;
import static org.junit.jupiter.api.Assertions.assertTrue;

@DistributionTest
@RawDistOnly(reason = "Containers are immutable")
@Tag(DistributionTest.SLOW)
public class ManualMigrationEmptyDatabaseDistTest {

private static final String CREATE_CHANGELOG_TABLE = "CREATE TABLE PUBLIC.DATABASECHANGELOG (";
private static final String INSERT_INTO_CHANGELOG = "INSERT INTO PUBLIC.DATABASECHANGELOG (";

/**
* The manual migration strategy exists for database users without DDL privileges. On an empty
* database the generated script announces itself as a creation script to be applied to an empty
* database, so it has to contain the DDL of the Liquibase changelog table it inserts into.
*/
@Test
void testEmptyDatabaseExportContainsChangelogTableDdl(KeycloakRunner runner, RawDistRootPath rawDistRootPath) throws IOException {
// force a completely empty database
RawKeycloakDistribution rawDist = runner.getDistribution(RawKeycloakDistribution.class);
FileUtil.deleteDirectory(rawDist.getDistPath().resolve("data").resolve("h2").toAbsolutePath());

CLIResult result = runner.run("start-dev",
"--spi-connections-jpa-quarkus-migration-strategy=manual",
"--spi-connections-jpa-quarkus-initialize-empty=false");

result.assertMessage("Database not initialized, please initialize database with");

File script = rawDistRootPath.getDistRootPath().resolve("bin").resolve("keycloak-database-update.sql").toFile();
assertTrue(script.isFile(), "Export file must exist when migration-strategy=manual");

String output = FileUtils.readFileToString(script, Charset.defaultCharset());

assertThat(output, containsString("Keycloak database creation script - apply this script to empty DB"));

// The script inserts into the changelog table, so it has to create it as well. Exactly once:
// emitting it twice makes the script fail with "table already exists" instead.
assertThat("changelog table must be created exactly once",
countOccurrences(output, CREATE_CHANGELOG_TABLE), is(1));

// ... and it has to be created before the first row is inserted into it.
assertThat("changelog table must be created before it is inserted into",
output.indexOf(CREATE_CHANGELOG_TABLE), lessThan(output.indexOf(INSERT_INTO_CHANGELOG)));

// changeSetExecuted preconditions are resolved against the changelog table, so it must still exist while
// the script is generated. Were it missing, they would all resolve to "not executed" and changesets that
// should be marked as ran would be emitted instead - this one carries MySQL-only index syntax.
assertThat(output, not(containsString("VALUE(255)")));
}

private static int countOccurrences(String haystack, String needle) {
return haystack.split(Pattern.quote(needle), -1).length - 1;
}
}
Loading
Loading