From b8e4290da17346dbad782fa76ff29ef1f1f16d39 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar <19465051+akumar074@users.noreply.github.com> Date: Fri, 9 Dec 2022 19:49:01 +0530 Subject: [PATCH] KOGITO-7676: Implement Flyway DB schema migration (#1431) * KOGITO-7676: Implement Flyway DB schema migration * Disabled schema generaration in PostgreSQL Integration tests * - Disable auto-DDL in PostgreSQL Data Index - Disabled Flyway migrate-on-start * - Fixed integration tests for Flyway integration * - Fixed integration tests for Flyway integration * - modified ddl zip config * incorporating PR coments * - Cleanup manual DDL scripts genration * - Fixing job-service integration tests * - Updated multi-repo-zip script --- .../DataIndexPostgreSqlResource.java | 1 - .../DataIndexPostgreSqlContainer.java | 2 + .../pom.xml | 11 - .../pom.xml | 11 - ...DataIndexPostgreSqlSpringTestResource.java | 2 + .../test/resources/CompositeTestResource.java | 3 + ...obServiceCompositeQuarkusTestResource.java | 2 +- .../pom.xml | 18 -- .../data-index-service-postgresql/pom.xml | 4 + .../src/main/resources/application.properties | 5 +- .../src/main/resources/data_index_create.sql | 208 --------------- .../migration/V1.32.0__data_index_create.sql | 243 ++++++++++++++++++ .../src/assembly/multi-repo-zip.xml | 6 +- 13 files changed, 262 insertions(+), 254 deletions(-) delete mode 100644 data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql create mode 100644 data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/db/migration/V1.32.0__data_index_create.sql diff --git a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/resources/DataIndexPostgreSqlResource.java b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/resources/DataIndexPostgreSqlResource.java index bc9039a11b..c9d511bd64 100644 --- a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/resources/DataIndexPostgreSqlResource.java +++ b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/resources/DataIndexPostgreSqlResource.java @@ -47,7 +47,6 @@ public void start() { LOGGER.debug("Starting PostgreSQL Quarkus test resource"); properties.clear(); Network network = Network.newNetwork(); - postgresql.withInitScript("data_index_postgresql_create.sql"); postgresql.withNetwork(network); postgresql.withNetworkAliases("postgresql"); postgresql.waitingFor(Wait.forListeningPort()); diff --git a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/testcontainers/DataIndexPostgreSqlContainer.java b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/testcontainers/DataIndexPostgreSqlContainer.java index f6f71d291f..724202b725 100644 --- a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/testcontainers/DataIndexPostgreSqlContainer.java +++ b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-common/src/main/java/org/kie/kogito/index/testcontainers/DataIndexPostgreSqlContainer.java @@ -30,6 +30,8 @@ public void setPostgreSqlURL(String postgreSqlURL, String username, String passw addEnv("QUARKUS_DATASOURCE_JDBC_URL", postgreSqlURL); addEnv("QUARKUS_DATASOURCE_USERNAME", username); addEnv("QUARKUS_DATASOURCE_PASSWORD", password); + addEnv("QUARKUS_FLYWAY_MIGRATE_AT_START", "true"); + addEnv("QUARKUS_FLYWAY_BASELINE_ON_MIGRATE", "true"); } @Override diff --git a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-quarkus/pom.xml b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-quarkus/pom.xml index 79f9e1c330..0b8fab440e 100644 --- a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-quarkus/pom.xml +++ b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-quarkus/pom.xml @@ -117,17 +117,6 @@ com.coderplus.maven.plugins copy-rename-maven-plugin - - copy-postgresql-file - generate-sources - - copy - - - ${project.root.dir}/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql - ${project.basedir}/src/main/resources/data_index_postgresql_create.sql - - rename-oracle-file generate-sources diff --git a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/pom.xml b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/pom.xml index cbb92b21ac..189d882c3d 100644 --- a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/pom.xml +++ b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/pom.xml @@ -128,17 +128,6 @@ com.coderplus.maven.plugins copy-rename-maven-plugin - - copy-postgresql-file - generate-sources - - copy - - - ${project.root.dir}/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql - ${project.basedir}/src/main/resources/data_index_postgresql_create.sql - - rename-oracle-file generate-sources diff --git a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/src/test/java/org/kie/kogito/index/spring/DataIndexPostgreSqlSpringTestResource.java b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/src/test/java/org/kie/kogito/index/spring/DataIndexPostgreSqlSpringTestResource.java index fa4148f178..425f3f75b8 100644 --- a/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/src/test/java/org/kie/kogito/index/spring/DataIndexPostgreSqlSpringTestResource.java +++ b/apps-integration-tests/integration-tests-data-index-service/integration-tests-data-index-service-springboot/src/test/java/org/kie/kogito/index/spring/DataIndexPostgreSqlSpringTestResource.java @@ -35,6 +35,8 @@ protected Map getProperties() { Map properties = new HashMap<>(); properties.put(KOGITO_DATA_INDEX_SERVICE_URL, "http://localhost:" + getTestResource().getMappedPort()); properties.putAll(getTestResource().getProperties()); + properties.put("spring.flyway.enabled", "true"); + properties.put("spring.flyway.baseline-on-migrate", "true"); return properties; } diff --git a/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-common/src/main/java/org/kie/kogito/test/resources/CompositeTestResource.java b/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-common/src/main/java/org/kie/kogito/test/resources/CompositeTestResource.java index 2d2c93409d..7686bb5a1b 100644 --- a/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-common/src/main/java/org/kie/kogito/test/resources/CompositeTestResource.java +++ b/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-common/src/main/java/org/kie/kogito/test/resources/CompositeTestResource.java @@ -163,6 +163,9 @@ protected void configurePostgreSQLToService(Collection> cont service.addEnv("QUARKUS_DATASOURCE_USERNAME", username); service.addEnv("QUARKUS_DATASOURCE_PASSWORD", password); service.addEnv("QUARKUS_DATASOURCE_DB-KIND", "postgresql"); + service.addEnv("QUARKUS_FLYWAY_MIGRATE_AT_START", "true"); + service.addEnv("QUARKUS_FLYWAY_BASELINE_ON_MIGRATE", "true"); + service.addEnv("QUARKUS_FLYWAY_CLEAN_AT_START", "false"); }); }); } diff --git a/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-common-quarkus/src/main/java/org/kie/kogito/test/resources/JobServiceCompositeQuarkusTestResource.java b/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-common-quarkus/src/main/java/org/kie/kogito/test/resources/JobServiceCompositeQuarkusTestResource.java index 7d30deb6d9..ea83a423dc 100644 --- a/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-common-quarkus/src/main/java/org/kie/kogito/test/resources/JobServiceCompositeQuarkusTestResource.java +++ b/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-common-quarkus/src/main/java/org/kie/kogito/test/resources/JobServiceCompositeQuarkusTestResource.java @@ -57,7 +57,7 @@ public void init(JobServiceTestResource annotation) { if (annotation.dataIndexEnabled()) { DataIndexPostgreSqlContainer container = new DataIndexPostgreSqlContainer(); container.addProtoFileFolder(); - KogitoPostgreSqlContainer postgresql = new KogitoPostgreSqlContainer().withInitScript("data_index_postgresql_create.sql"); + KogitoPostgreSqlContainer postgresql = new KogitoPostgreSqlContainer(); resource.withServiceContainer("data-index", container, postgresql); } } diff --git a/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-quarkus-messaging/pom.xml b/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-quarkus-messaging/pom.xml index f8819926ae..8ab5fee6fa 100644 --- a/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-quarkus-messaging/pom.xml +++ b/apps-integration-tests/integration-tests-jobs-service/integration-tests-jobs-service-quarkus/integration-tests-jobs-service-quarkus-messaging/pom.xml @@ -65,24 +65,6 @@ - - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-postgresql-file - generate-sources - - copy - - - ${project.root.dir}/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql - ${project.basedir}/target/classes/data_index_postgresql_create.sql - - - - diff --git a/data-index/data-index-service/data-index-service-postgresql/pom.xml b/data-index/data-index-service/data-index-service-postgresql/pom.xml index 3ad1c837a8..6b4ffb9c2a 100644 --- a/data-index/data-index-service/data-index-service-postgresql/pom.xml +++ b/data-index/data-index-service/data-index-service-postgresql/pom.xml @@ -29,6 +29,10 @@ io.quarkus quarkus-container-image-jib + + io.quarkus + quarkus-flyway + org.kie.kogito diff --git a/data-index/data-index-service/data-index-service-postgresql/src/main/resources/application.properties b/data-index/data-index-service/data-index-service-postgresql/src/main/resources/application.properties index 45f6bd5059..5fdbe8698e 100644 --- a/data-index/data-index-service/data-index-service-postgresql/src/main/resources/application.properties +++ b/data-index/data-index-service/data-index-service-postgresql/src/main/resources/application.properties @@ -8,6 +8,9 @@ quarkus.kafka.health.enabled=true quarkus.kafka.bootstrap-servers=localhost:9092 %dev.kafka.bootstrap.servers=localhost:9092 +#Flyway +quarkus.flyway.baseline-on-migrate=true + mp.messaging.incoming.kogito-processinstances-events.connector=smallrye-kafka mp.messaging.incoming.kogito-processinstances-events.topic=kogito-processinstances-events mp.messaging.incoming.kogito-processinstances-events.group.id=kogito-data-index-processinstances @@ -41,4 +44,4 @@ quarkus.hibernate-orm.physical-naming-strategy=org.hibernate.boot.model.naming.C #Container image quarkus.container-image.build=${quarkus.build.image:true} -quarkus.container-image.group=org.kie.kogito \ No newline at end of file +quarkus.container-image.group=org.kie.kogito diff --git a/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql b/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql deleted file mode 100644 index 1f62c2b9c2..0000000000 --- a/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/data_index_create.sql +++ /dev/null @@ -1,208 +0,0 @@ - - create table attachments ( - id varchar(255) not null, - content varchar(255), - name varchar(255), - updated_at timestamp, - updated_by varchar(255), - task_id varchar(255) not null, - primary key (id) - ); - - create table comments ( - id varchar(255) not null, - content varchar(255), - updated_at timestamp, - updated_by varchar(255), - task_id varchar(255) not null, - primary key (id) - ); - - create table jobs ( - id varchar(255) not null, - callback_endpoint varchar(255), - endpoint varchar(255), - execution_counter int4, - expiration_time timestamp, - last_update timestamp, - node_instance_id varchar(255), - priority int4, - process_id varchar(255), - process_instance_id varchar(255), - repeat_interval int8, - repeat_limit int4, - retries int4, - root_process_id varchar(255), - root_process_instance_id varchar(255), - scheduled_id varchar(255), - status varchar(255), - primary key (id) - ); - - create table milestones ( - id varchar(255) not null, - process_instance_id varchar(255) not null, - name varchar(255), - status varchar(255), - primary key (id, process_instance_id) - ); - - create table nodes ( - id varchar(255) not null, - definition_id varchar(255), - enter timestamp, - exit timestamp, - name varchar(255), - node_id varchar(255), - type varchar(255), - process_instance_id varchar(255) not null, - primary key (id) - ); - - create table processes ( - id varchar(255) not null, - business_key varchar(255), - end_time timestamp, - endpoint varchar(255), - message varchar(255), - node_definition_id varchar(255), - last_update_time timestamp, - parent_process_instance_id varchar(255), - process_id varchar(255), - process_name varchar(255), - root_process_id varchar(255), - root_process_instance_id varchar(255), - start_time timestamp, - state int4, - variables jsonb, - primary key (id) - ); - - create table processes_addons ( - process_id varchar(255) not null, - addon varchar(255) not null, - primary key (process_id, addon) - ); - - create table processes_roles ( - process_id varchar(255) not null, - role varchar(255) not null, - primary key (process_id, role) - ); - - create table tasks ( - id varchar(255) not null, - actual_owner varchar(255), - completed timestamp, - description varchar(255), - endpoint varchar(255), - inputs jsonb, - last_update timestamp, - name varchar(255), - outputs jsonb, - priority varchar(255), - process_id varchar(255), - process_instance_id varchar(255), - reference_name varchar(255), - root_process_id varchar(255), - root_process_instance_id varchar(255), - started timestamp, - state varchar(255), - primary key (id) - ); - - create table tasks_admin_groups ( - task_id varchar(255) not null, - group_id varchar(255) not null, - primary key (task_id, group_id) - ); - - create table tasks_admin_users ( - task_id varchar(255) not null, - user_id varchar(255) not null, - primary key (task_id, user_id) - ); - - create table tasks_excluded_users ( - task_id varchar(255) not null, - user_id varchar(255) not null, - primary key (task_id, user_id) - ); - - create table tasks_potential_groups ( - task_id varchar(255) not null, - group_id varchar(255) not null, - primary key (task_id, group_id) - ); - - create table tasks_potential_users ( - task_id varchar(255) not null, - user_id varchar(255) not null, - primary key (task_id, user_id) - ); - - alter table if exists attachments - add constraint fk_attachments_tasks - foreign key (task_id) - references tasks - on delete cascade; - - alter table if exists comments - add constraint fk_comments_tasks - foreign key (task_id) - references tasks - on delete cascade; - - alter table if exists milestones - add constraint fk_milestones_process - foreign key (process_instance_id) - references processes - on delete cascade; - - alter table if exists nodes - add constraint fk_nodes_process - foreign key (process_instance_id) - references processes - on delete cascade; - - alter table if exists processes_addons - add constraint fk_processes_addons_processes - foreign key (process_id) - references processes - on delete cascade; - - alter table if exists processes_roles - add constraint fk_processes_roles_processes - foreign key (process_id) - references processes - on delete cascade; - - alter table if exists tasks_admin_groups - add constraint fk_tasks_admin_groups_tasks - foreign key (task_id) - references tasks - on delete cascade; - - alter table if exists tasks_admin_users - add constraint fk_tasks_admin_users_tasks - foreign key (task_id) - references tasks - on delete cascade; - - alter table if exists tasks_excluded_users - add constraint fk_tasks_excluded_users_tasks - foreign key (task_id) - references tasks - on delete cascade; - - alter table if exists tasks_potential_groups - add constraint fk_tasks_potential_groups_tasks - foreign key (task_id) - references tasks - on delete cascade; - - alter table if exists tasks_potential_users - add constraint fk_tasks_potential_users_tasks - foreign key (task_id) - references tasks - on delete cascade; diff --git a/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/db/migration/V1.32.0__data_index_create.sql b/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/db/migration/V1.32.0__data_index_create.sql new file mode 100644 index 0000000000..58cbc17bce --- /dev/null +++ b/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/db/migration/V1.32.0__data_index_create.sql @@ -0,0 +1,243 @@ +create table attachments +( + id varchar(255) not null, + content varchar(255), + name varchar(255), + updated_at timestamp, + updated_by varchar(255), + task_id varchar(255) not null, + primary key (id) +); + +create table comments +( + id varchar(255) not null, + content varchar(255), + updated_at timestamp, + updated_by varchar(255), + task_id varchar(255) not null, + primary key (id) +); + +create table jobs +( + id varchar(255) not null, + callback_endpoint varchar(255), + endpoint varchar(255), + execution_counter int4, + expiration_time timestamp, + last_update timestamp, + node_instance_id varchar(255), + priority int4, + process_id varchar(255), + process_instance_id varchar(255), + repeat_interval int8, + repeat_limit int4, + retries int4, + root_process_id varchar(255), + root_process_instance_id varchar(255), + scheduled_id varchar(255), + status varchar(255), + primary key (id) +); + +create table milestones +( + id varchar(255) not null, + process_instance_id varchar(255) not null, + name varchar(255), + status varchar(255), + primary key (id, process_instance_id) +); + +create table nodes +( + id varchar(255) not null, + definition_id varchar(255), + enter timestamp, + exit timestamp, + name varchar(255), + node_id varchar(255), + type varchar(255), + process_instance_id varchar(255) not null, + primary key (id) +); + +create table processes +( + id varchar(255) not null, + business_key varchar(255), + end_time timestamp, + endpoint varchar(255), + message varchar(255), + node_definition_id varchar(255), + last_update_time timestamp, + parent_process_instance_id varchar(255), + process_id varchar(255), + process_name varchar(255), + root_process_id varchar(255), + root_process_instance_id varchar(255), + start_time timestamp, + state int4, + variables jsonb, + primary key (id) +); + +create table processes_addons +( + process_id varchar(255) not null, + addon varchar(255) not null, + primary key (process_id, addon) +); + +create table processes_roles +( + process_id varchar(255) not null, + role varchar(255) not null, + primary key (process_id, role) +); + +create table tasks +( + id varchar(255) not null, + actual_owner varchar(255), + completed timestamp, + description varchar(255), + endpoint varchar(255), + inputs jsonb, + last_update timestamp, + name varchar(255), + outputs jsonb, + priority varchar(255), + process_id varchar(255), + process_instance_id varchar(255), + reference_name varchar(255), + root_process_id varchar(255), + root_process_instance_id varchar(255), + started timestamp, + state varchar(255), + primary key (id) +); + +create table tasks_admin_groups +( + task_id varchar(255) not null, + group_id varchar(255) not null, + primary key (task_id, group_id) +); + +create table tasks_admin_users +( + task_id varchar(255) not null, + user_id varchar(255) not null, + primary key (task_id, user_id) +); + +create table tasks_excluded_users +( + task_id varchar(255) not null, + user_id varchar(255) not null, + primary key (task_id, user_id) +); + +create table tasks_potential_groups +( + task_id varchar(255) not null, + group_id varchar(255) not null, + primary key (task_id, group_id) +); + +create table tasks_potential_users +( + task_id varchar(255) not null, + user_id varchar(255) not null, + primary key (task_id, user_id) +); + +alter table if exists attachments + add constraint fk_attachments_tasks + foreign key (task_id) + references tasks + on +delete +cascade; + +alter table if exists comments + add constraint fk_comments_tasks + foreign key (task_id) + references tasks + on +delete +cascade; + +alter table if exists milestones + add constraint fk_milestones_process + foreign key (process_instance_id) + references processes + on +delete +cascade; + +alter table if exists nodes + add constraint fk_nodes_process + foreign key (process_instance_id) + references processes + on +delete +cascade; + +alter table if exists processes_addons + add constraint fk_processes_addons_processes + foreign key (process_id) + references processes + on +delete +cascade; + +alter table if exists processes_roles + add constraint fk_processes_roles_processes + foreign key (process_id) + references processes + on +delete +cascade; + +alter table if exists tasks_admin_groups + add constraint fk_tasks_admin_groups_tasks + foreign key (task_id) + references tasks + on +delete +cascade; + +alter table if exists tasks_admin_users + add constraint fk_tasks_admin_users_tasks + foreign key (task_id) + references tasks + on +delete +cascade; + +alter table if exists tasks_excluded_users + add constraint fk_tasks_excluded_users_tasks + foreign key (task_id) + references tasks + on +delete +cascade; + +alter table if exists tasks_potential_groups + add constraint fk_tasks_potential_groups_tasks + foreign key (task_id) + references tasks + on +delete +cascade; + +alter table if exists tasks_potential_users + add constraint fk_tasks_potential_users_tasks + foreign key (task_id) + references tasks + on +delete +cascade; diff --git a/persistence-commons/kogito-ddl/src/assembly/multi-repo-zip.xml b/persistence-commons/kogito-ddl/src/assembly/multi-repo-zip.xml index 0b822df668..25d8a52bb1 100644 --- a/persistence-commons/kogito-ddl/src/assembly/multi-repo-zip.xml +++ b/persistence-commons/kogito-ddl/src/assembly/multi-repo-zip.xml @@ -17,10 +17,10 @@ - ${project.root.dir}/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources + ${project.root.dir}/data-index/data-index-storage/data-index-storage-postgresql/src/main/resources/db/migration postgresql - *_create.sql + *.sql @@ -46,4 +46,4 @@ trusty_service_create.sql - \ No newline at end of file +