Skip to content

Commit

Permalink
feat: EXPOSED-255 Generate database migration script that can be used…
Browse files Browse the repository at this point in the history 
… with any migration tool

-The script has the .sql extension
-The script's name is decided by the user
-The directory in which the script is created is decided by the user
-The script is editable after it's generated
-The user bears full responsibility for how the migration script is used. It should NOT be executed as regular SQL statements because there is no guarantee that it can be rolled back if a failure happens at any point. It should only be executed using a database migration tool like Flyway, for example, which is used in the tests for this feature.
  • Loading branch information
@joc-a
joc-a committed Jan 18, 2024
1 parent fea957d commit 5a7f198
Show file tree
Hide file tree
Showing 5 changed files with 360 additions and 35 deletions.
8 changes: 7 additions & 1 deletion exposed-core/api/exposed-core.api
View file
Original file line number Diff line number Diff line change
Expand Up @@ -752,6 +752,9 @@ public final class org/jetbrains/exposed/sql/Exists : org/jetbrains/exposed/sql/
public fun toQueryBuilder (Lorg/jetbrains/exposed/sql/QueryBuilder;)V
}

public abstract interface annotation class org/jetbrains/exposed/sql/ExperimentalDatabaseMigrationApi : java/lang/annotation/Annotation {
}

public abstract interface annotation class org/jetbrains/exposed/sql/ExperimentalKeywordApi : java/lang/annotation/Annotation {
}

Expand Down Expand Up @@ -1843,7 +1846,6 @@ public final class org/jetbrains/exposed/sql/SchemaUtils {
public final fun addMissingColumnsStatements ([Lorg/jetbrains/exposed/sql/Table;Z)Ljava/util/List;
public static synthetic fun addMissingColumnsStatements$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;ZILjava/lang/Object;)Ljava/util/List;
public final fun checkCycle ([Lorg/jetbrains/exposed/sql/Table;)Z
public final fun checkExcessiveIndices ([Lorg/jetbrains/exposed/sql/Table;)V
public final fun checkMappingConsistence ([Lorg/jetbrains/exposed/sql/Table;Z)Ljava/util/List;
public static synthetic fun checkMappingConsistence$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;ZILjava/lang/Object;)Ljava/util/List;
public final fun create ([Lorg/jetbrains/exposed/sql/Table;Z)V
Expand All @@ -1867,11 +1869,15 @@ public final class org/jetbrains/exposed/sql/SchemaUtils {
public static synthetic fun dropSchema$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Schema;ZZILjava/lang/Object;)V
public final fun dropSequence ([Lorg/jetbrains/exposed/sql/Sequence;Z)V
public static synthetic fun dropSequence$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Sequence;ZILjava/lang/Object;)V
public final fun generateMigrationScript ([Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Z)Ljava/io/File;
public static synthetic fun generateMigrationScript$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Ljava/io/File;
public final fun listDatabases ()Ljava/util/List;
public final fun listTables ()Ljava/util/List;
public final fun setSchema (Lorg/jetbrains/exposed/sql/Schema;Z)V
public static synthetic fun setSchema$default (Lorg/jetbrains/exposed/sql/SchemaUtils;Lorg/jetbrains/exposed/sql/Schema;ZILjava/lang/Object;)V
public final fun sortTablesByReferences (Ljava/lang/Iterable;)Ljava/util/List;
public final fun statementsRequiredForDatabaseMigration ([Lorg/jetbrains/exposed/sql/Table;Z)Ljava/util/List;
public static synthetic fun statementsRequiredForDatabaseMigration$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;ZILjava/lang/Object;)Ljava/util/List;
public final fun statementsRequiredToActualizeScheme ([Lorg/jetbrains/exposed/sql/Table;Z)Ljava/util/List;
public static synthetic fun statementsRequiredToActualizeScheme$default (Lorg/jetbrains/exposed/sql/SchemaUtils;[Lorg/jetbrains/exposed/sql/Table;ZILjava/lang/Object;)Ljava/util/List;
public final fun withDataBaseLock (Lorg/jetbrains/exposed/sql/Transaction;Lkotlin/jvm/functions/Function0;)V
Expand Down
177 changes: 145 additions & 32 deletions exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/SchemaUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package org.jetbrains.exposed.sql
import org.jetbrains.exposed.exceptions.ExposedSQLException
import org.jetbrains.exposed.sql.transactions.TransactionManager
import org.jetbrains.exposed.sql.vendors.*
import java.io.File
import java.math.BigDecimal

/** Utility functions that assist with creating, altering, and dropping database schema objects. */
Expand Down Expand Up @@ -513,56 +514,97 @@ object SchemaUtils {
*/
fun checkMappingConsistence(vararg tables: Table, withLogs: Boolean = true): List<String> {
if (withLogs) {
checkExcessiveIndices(tables = tables)
checkExcessiveForeignKeyConstraints(tables = tables, withLogs = true)
checkExcessiveIndices(tables = tables, withLogs = true)
}
return checkMissingIndices(tables = tables, withLogs).flatMap { it.createStatement() }
}

/**
* Checks all [tables] for any that have more than one defined foreign key constraint or index and
* logs the findings.
* Log Exposed table mappings <-> real database mapping problems and returns DDL Statements to fix them, including
* DROP/DELETE statements (unlike [checkMappingConsistence])
*/
private fun mappingConsistenceRequiredStatements(vararg tables: Table, withLogs: Boolean = true): List<String> {
return checkMissingIndices(tables = tables, withLogs).flatMap { it.createStatement() } +
checkExcessiveForeignKeyConstraints(tables = tables, withLogs).flatMap { it.dropStatement() } +
checkExcessiveIndices(tables = tables, withLogs).flatMap { it.dropStatement() }
}

/**
* Checks all [tables] for any that have more than one defined index and logs the findings.
*
* If found, this function also logs the SQL statements that can be used to drop these constraints.
* If found, this function also logs and returns the SQL statements that can be used to drop these constraints.
*/
fun checkExcessiveIndices(vararg tables: Table) {
val excessiveConstraints = currentDialect.columnConstraints(*tables).filter { it.value.size > 1 }
@Suppress("NestedBlockDepth")
private fun checkExcessiveIndices(vararg tables: Table, withLogs: Boolean): List<Index> {
val toDrop = HashSet<Index>()

if (excessiveConstraints.isNotEmpty()) {
exposedLogger.warn("List of excessive foreign key constraints:")
excessiveConstraints.forEach { (pair, fk) ->
val constraint = fk.first()
val fkPartToLog = fk.joinToString(", ") { it.fkName }
exposedLogger.warn(
"\t\t\t'${pair.first}'.'${pair.second}' -> '${constraint.fromTableName}':\t$fkPartToLog"
)
val excessiveIndices =
currentDialect.existingIndices(*tables).flatMap { (_, indices) ->
indices
}.groupBy { index -> Triple(index.table, index.unique, index.columns.joinToString { column -> column.name }) }
.filter { (_, indices) -> indices.size > 1 }
if (excessiveIndices.isNotEmpty()) {
if (withLogs) {
exposedLogger.warn("List of excessive indices:")
excessiveIndices.forEach { (triple, indices) ->
val indexNames = indices.joinToString(", ") { index -> index.indexName }
exposedLogger.warn("\t\t\t'${triple.first.tableName}'.'${triple.third}' -> $indexNames")
}

exposedLogger.info("SQL Queries to remove excessive indices:")
}

exposedLogger.info("SQL Queries to remove excessive keys:")
excessiveConstraints.forEach { (_, value) ->
value.take(value.size - 1).forEach {
exposedLogger.info("\t\t\t${it.dropStatement()};")
excessiveIndices.forEach { (_, indices) ->
indices.take(indices.size - 1).forEach { index ->
toDrop.add(index)

if (withLogs) {
exposedLogger.info("\t\t\t${index.dropStatement()};")
}
}
}
}

val excessiveIndices =
currentDialect.existingIndices(*tables).flatMap {
it.value
}.groupBy { Triple(it.table, it.unique, it.columns.joinToString { it.name }) }
.filter { it.value.size > 1 }
if (excessiveIndices.isNotEmpty()) {
exposedLogger.warn("List of excessive indices:")
excessiveIndices.forEach { (triple, indices) ->
val indexNames = indices.joinToString(", ") { it.indexName }
exposedLogger.warn("\t\t\t'${triple.first.tableName}'.'${triple.third}' -> $indexNames")
return toDrop.toList()
}

/**
* Checks all [tables] for any that have more than one defined foreign key constraint and logs the findings.
*
* If found, this function also logs and returns the SQL statements that can be used to drop these constraints.
*/
@Suppress("NestedBlockDepth")
private fun checkExcessiveForeignKeyConstraints(vararg tables: Table, withLogs: Boolean): List<ForeignKeyConstraint> {
val toDrop = HashSet<ForeignKeyConstraint>()

val excessiveConstraints = currentDialect.columnConstraints(*tables).filter { (_, fkConstraints) -> fkConstraints.size > 1 }
if (excessiveConstraints.isNotEmpty()) {
if (withLogs) {
exposedLogger.warn("List of excessive foreign key constraints:")
excessiveConstraints.forEach { (table, columns), fkConstraints ->
val constraint = fkConstraints.first()
val fkPartToLog = fkConstraints.joinToString(", ") { fkConstraint -> fkConstraint.fkName }
exposedLogger.warn(
"\t\t\t'$table'.'$columns' -> '${constraint.fromTableName}':\t$fkPartToLog"
)
}

exposedLogger.info("SQL Queries to remove excessive keys:")
}
exposedLogger.info("SQL Queries to remove excessive indices:")
excessiveIndices.forEach {
it.value.take(it.value.size - 1).forEach {
exposedLogger.info("\t\t\t${it.dropStatement()};")

excessiveConstraints.forEach { (_, fkConstraints) ->
fkConstraints.take(fkConstraints.size - 1).forEach { fkConstraint ->
toDrop.add(fkConstraint)

if (withLogs) {
exposedLogger.info("\t\t\t${fkConstraint.dropStatement()};")
}
}
}
}

return toDrop.toList()
}

/** Returns list of indices missed in database **/
Expand Down Expand Up @@ -624,6 +666,69 @@ object SchemaUtils {
return toCreate.toList()
}

/**
* @param scriptName The name to be used for the generated migration script.
* @param scriptDirectory The directory in which to create the migration script.
* @param withLogs By default, a description for each intermediate step, as well as its execution time, is logged at
* the INFO level. This can be disabled by setting [withLogs] to `false`.
*
* @return The generated migration script.
*
* This function simply generates the migration script without applying the migration. The purpose of it is to show
* the user what the migration script will look like before applying the migration.
*/
@ExperimentalDatabaseMigrationApi
fun generateMigrationScript(vararg tables: Table, scriptDirectory: String, scriptName: String, withLogs: Boolean = true): File {
val allStatements = statementsRequiredForDatabaseMigration(*tables, withLogs = withLogs)

val migrationScript = File("$scriptDirectory/$scriptName.sql")
val migrationScriptExists = if (migrationScript.exists()) true else migrationScript.createNewFile()
if (migrationScriptExists) {
// clear existing content
migrationScript.writeText("")

// append statements
allStatements.forEach { statement ->
migrationScript.appendText("$statement;\n")
}
} else {
throw NoSuchFileException(migrationScript, reason = "Failed to create/find migration script")
}

return migrationScript
}

/**
* Returns the SQL statements that need to be executed to make the existing database schema compatible with
* the table objects defined using Exposed. Unlike [statementsRequiredToActualizeScheme], DROP/DELETE statements are
* included.
*
* **Note:** Some dialects, like SQLite, do not support `ALTER TABLE ADD COLUMN` syntax completely,
* which restricts the behavior when adding some missing columns. Please check the documentation.
*
* By default, a description for each intermediate step, as well as its execution time, is logged at the INFO level.
* This can be disabled by setting [withLogs] to `false`.
*/
fun statementsRequiredForDatabaseMigration(vararg tables: Table, withLogs: Boolean = true): List<String> {
val (tablesToCreate, tablesToAlter) = tables.partition { !it.exists() }
val createStatements = logTimeSpent("Preparing create tables statements", withLogs) {
createStatements(tables = tablesToCreate.toTypedArray())
}
val alterStatements = logTimeSpent("Preparing alter table statements", withLogs) {
addMissingColumnsStatements(tables = tablesToAlter.toTypedArray(), withLogs)
}

val modifyTablesStatements = logTimeSpent("Checking mapping consistence", withLogs) {
mappingConsistenceRequiredStatements(
tables = tables,
withLogs
).filter { it !in (createStatements + alterStatements) }
}

val allStatements = createStatements + alterStatements + modifyTablesStatements
return allStatements
}

/**
* Creates table with name "busy" (if not present) and single column to be used as "synchronization" point. Table wont be dropped after execution.
*
Expand Down Expand Up @@ -745,3 +850,11 @@ object SchemaUtils {
}
}
}

@RequiresOptIn(
message = "This database migration API is experimental. " +
"Its usage must be marked with '@OptIn(org.jetbrains.exposed.sql.ExperimentalDatabaseMigrationApi::class)' " +
"or '@org.jetbrains.exposed.sql.ExperimentalDatabaseMigrationApi'."
)
@Target(AnnotationTarget.FUNCTION)
annotation class ExperimentalDatabaseMigrationApi
4 changes: 4 additions & 0 deletions exposed-tests/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ dependencies {
compileOnly(libs.h2)
testCompileOnly(libs.sqlite.jdbc)
testImplementation(libs.logcaptor)

testApi(libs.flyway.core)
testApi(libs.flyway.mysql)
testApi(libs.flyway.oracle)
}

tasks.withType<Test>().configureEach {
Expand Down
Loading

0 comments on commit 5a7f198

Please sign in to comment.