From e16d0dbb4c3a1efe30b98282961eda5888e96c6e Mon Sep 17 00:00:00 2001 From: Paulius Dambrauskas Date: Sun, 22 Oct 2023 14:49:41 +0300 Subject: [PATCH] Update to Java 17 + Micronaut 4 --- server/build.gradle | 32 +++-- .../application/sessions/Statement.java | 2 +- .../lighter/backend/yarn/YarnProperties.java | 2 +- .../lighter/rest/BatchController.java | 2 +- .../lighter/rest/SessionController.java | 2 +- .../exceptions/ApiErrorResponseProcessor.java | 2 +- .../storage/jdbc/JdbcApplicationStorage.java | 5 +- .../storage/jdbc/JdbcLockProvider.java | 15 ++- .../lighter/storage/jdbc/JdbcLogStorage.java | 2 +- .../storage/jdbc/JdbcStatementStorage.java | 5 +- .../LighterJdbcDatasourceConfiguration.java | 121 ------------------ server/src/main/resources/application.yml | 114 +++++++++-------- .../jdbc/JdbcApplicationStorageTest.groovy | 2 +- .../jdbc/JdbcStatementStorageTest.groovy | 2 +- 14 files changed, 107 insertions(+), 201 deletions(-) delete mode 100644 server/src/main/java/com/exacaster/lighter/storage/jdbc/LighterJdbcDatasourceConfiguration.java diff --git a/server/build.gradle b/server/build.gradle index fa38f482..073c54fb 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -1,8 +1,8 @@ plugins { id "java" id "groovy" - id "io.micronaut.application" version "3.7.10" - id "com.github.ben-manes.versions" version "0.48.0" + id "io.micronaut.application" version "4.1.1" + id "com.github.ben-manes.versions" version "0.49.0" } group "org.exacaster.lighter" @@ -13,11 +13,11 @@ repositories { } java { - sourceCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_17 } micronaut { - version = "3.10.1" + version = "4.1.5" runtime "netty" } mainClassName = "com.exacaster.lighter.Application" @@ -25,22 +25,27 @@ mainClassName = "com.exacaster.lighter.Application" def sparkVersion = project.hasProperty("SPARK_VERSION") ? project.getProperty("SPARK_VERSION") : '3.5.0' def hadoopVersion = '3.3.3' def k8sClientVersion = '6.9.0' -def shedlockVersion = '4.46.0' +def shedlockVersion = '5.9.1' + dependencies { - annotationProcessor "io.micronaut:micronaut-validation" annotationProcessor "io.micronaut.openapi:micronaut-openapi" + annotationProcessor "io.micronaut.validation:micronaut-validation-processor" + annotationProcessor "io.micronaut.data:micronaut-data-processor" + + implementation "io.micronaut.validation:micronaut-validation" - implementation "io.micronaut:micronaut-validation" implementation "io.micronaut.flyway:micronaut-flyway" implementation "io.swagger.core.v3:swagger-annotations" implementation "io.micronaut.sql:micronaut-jdbc-hikari" implementation "io.micronaut.sql:micronaut-jdbi" - implementation "io.micronaut.data:micronaut-data-processor" + implementation "io.micronaut.data:micronaut-data-jdbc" + implementation "io.micronaut:micronaut-http-client" + implementation "io.micronaut:micronaut-jackson-databind" - implementation "org.apache.spark:spark-launcher_2.12:${sparkVersion}" + implementation "org.apache.spark:spark-launcher_2.13:${sparkVersion}" implementation "io.fabric8:kubernetes-client:${k8sClientVersion}" implementation("org.apache.hadoop:hadoop-yarn-client:${hadoopVersion}") { exclude(group: "ch.qos.reload4j", module: "reload4j") @@ -52,17 +57,22 @@ dependencies { } implementation 'net.sf.py4j:py4j:0.10.9.7' - implementation "net.javacrumbs.shedlock:shedlock-micronaut:${shedlockVersion}" - implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-micronaut:${shedlockVersion}" + implementation "net.javacrumbs.shedlock:shedlock-micronaut4:${shedlockVersion}" + implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-template:${shedlockVersion}" +// implementation "net.javacrumbs.shedlock:shedlock-provider-jdbc-micronaut:${shedlockVersion}" runtimeOnly "io.micronaut.micrometer:micronaut-micrometer-core" runtimeOnly "io.micronaut.micrometer:micronaut-micrometer-registry-prometheus" runtimeOnly "io.micronaut:micronaut-management" + runtimeOnly "org.yaml:snakeyaml" runtimeOnly "com.h2database:h2" runtimeOnly "org.postgresql:postgresql" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:5.10.0" + testRuntimeOnly "net.bytebuddy:byte-buddy:1.14.9" + testRuntimeOnly "org.objenesis:objenesis:3.3" + testImplementation "io.micronaut:micronaut-inject-groovy" testImplementation "io.micronaut.test:micronaut-test-spock" testImplementation "io.micronaut:micronaut-http-client" diff --git a/server/src/main/java/com/exacaster/lighter/application/sessions/Statement.java b/server/src/main/java/com/exacaster/lighter/application/sessions/Statement.java index 8e2c996e..7ba63379 100644 --- a/server/src/main/java/com/exacaster/lighter/application/sessions/Statement.java +++ b/server/src/main/java/com/exacaster/lighter/application/sessions/Statement.java @@ -4,7 +4,7 @@ import io.micronaut.core.annotation.Introspected; import io.micronaut.core.annotation.Nullable; -import javax.validation.constraints.NotNull; +import jakarta.validation.constraints.NotNull; import java.time.LocalDateTime; import java.util.StringJoiner; diff --git a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnProperties.java b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnProperties.java index 5209b33e..ee3c6d4e 100644 --- a/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnProperties.java +++ b/server/src/main/java/com/exacaster/lighter/backend/yarn/YarnProperties.java @@ -4,7 +4,7 @@ import io.micronaut.context.annotation.ConfigurationProperties; import io.micronaut.context.annotation.Requires; import io.micronaut.core.annotation.Nullable; -import javax.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; import org.apache.commons.lang3.builder.ToStringBuilder; @ConfigurationProperties("lighter.yarn") diff --git a/server/src/main/java/com/exacaster/lighter/rest/BatchController.java b/server/src/main/java/com/exacaster/lighter/rest/BatchController.java index 93196c8b..28f4bfa5 100644 --- a/server/src/main/java/com/exacaster/lighter/rest/BatchController.java +++ b/server/src/main/java/com/exacaster/lighter/rest/BatchController.java @@ -21,7 +21,7 @@ import io.swagger.v3.oas.annotations.tags.Tags; import java.util.Optional; -import javax.validation.Valid; +import jakarta.validation.Valid; @Tags(@Tag(name = "Batches")) @Validated diff --git a/server/src/main/java/com/exacaster/lighter/rest/SessionController.java b/server/src/main/java/com/exacaster/lighter/rest/SessionController.java index cb8ef830..c1bfd6e7 100644 --- a/server/src/main/java/com/exacaster/lighter/rest/SessionController.java +++ b/server/src/main/java/com/exacaster/lighter/rest/SessionController.java @@ -29,7 +29,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.annotations.tags.Tags; -import javax.validation.Valid; +import jakarta.validation.Valid; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/server/src/main/java/com/exacaster/lighter/rest/exceptions/ApiErrorResponseProcessor.java b/server/src/main/java/com/exacaster/lighter/rest/exceptions/ApiErrorResponseProcessor.java index 1c4c431e..bc06e526 100644 --- a/server/src/main/java/com/exacaster/lighter/rest/exceptions/ApiErrorResponseProcessor.java +++ b/server/src/main/java/com/exacaster/lighter/rest/exceptions/ApiErrorResponseProcessor.java @@ -8,7 +8,7 @@ import io.micronaut.http.server.exceptions.response.ErrorContext; import io.micronaut.http.server.exceptions.response.ErrorResponseProcessor; -import javax.inject.Singleton; +import jakarta.inject.Singleton; @Primary @Singleton diff --git a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorage.java b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorage.java index a572f7c9..e6a705c9 100644 --- a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorage.java +++ b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorage.java @@ -18,13 +18,14 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import javax.sql.DataSource; -import javax.transaction.Transactional; +import jakarta.transaction.Transactional; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; import org.slf4j.Logger; +import javax.sql.DataSource; + @Singleton @Requires(beans = DataSource.class) public class JdbcApplicationStorage implements ApplicationStorage, RowMapper { diff --git a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLockProvider.java b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLockProvider.java index fb0e205b..46d22c7a 100644 --- a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLockProvider.java +++ b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLockProvider.java @@ -2,18 +2,27 @@ import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; +import io.micronaut.data.connection.jdbc.advice.DelegatingDataSource; import jakarta.inject.Singleton; import net.javacrumbs.shedlock.core.LockProvider; -import net.javacrumbs.shedlock.provider.jdbc.micronaut.MicronautJdbcLockProvider; +import net.javacrumbs.shedlock.provider.jdbctemplate.JdbcTemplateLockProvider; +import org.springframework.jdbc.core.JdbcTemplate; import javax.sql.DataSource; + @Factory @Requires(beans = DataSource.class) public class JdbcLockProvider { @Singleton - public LockProvider lockProvider(DataSource datasource) { - return new MicronautJdbcLockProvider(datasource); + public LockProvider lockProvider(DataSource dataSource) { + DataSource unwrappedDataSource = ((DelegatingDataSource) dataSource).getTargetDataSource(); + return new JdbcTemplateLockProvider( + JdbcTemplateLockProvider.Configuration.builder() + .withJdbcTemplate(new JdbcTemplate(unwrappedDataSource)) + .usingDbTime() + .build() + ); } } diff --git a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLogStorage.java b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLogStorage.java index 3a5f7658..975e8ae2 100644 --- a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLogStorage.java +++ b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcLogStorage.java @@ -8,7 +8,7 @@ import java.util.Optional; import jakarta.inject.Singleton; import javax.sql.DataSource; -import javax.transaction.Transactional; +import jakarta.transaction.Transactional; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; diff --git a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcStatementStorage.java b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcStatementStorage.java index 9a60b303..f91974ba 100644 --- a/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcStatementStorage.java +++ b/server/src/main/java/com/exacaster/lighter/storage/jdbc/JdbcStatementStorage.java @@ -14,12 +14,13 @@ import java.util.List; import java.util.Optional; import java.util.UUID; -import javax.sql.DataSource; -import javax.transaction.Transactional; +import jakarta.transaction.Transactional; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; +import javax.sql.DataSource; + @Singleton @Requires(beans = DataSource.class) public class JdbcStatementStorage implements StatementStorage, RowMapper { diff --git a/server/src/main/java/com/exacaster/lighter/storage/jdbc/LighterJdbcDatasourceConfiguration.java b/server/src/main/java/com/exacaster/lighter/storage/jdbc/LighterJdbcDatasourceConfiguration.java deleted file mode 100644 index 8fe14960..00000000 --- a/server/src/main/java/com/exacaster/lighter/storage/jdbc/LighterJdbcDatasourceConfiguration.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.exacaster.lighter.storage.jdbc; - - -import io.micronaut.configuration.jdbc.hikari.DatasourceConfiguration; -import io.micronaut.context.annotation.ConfigurationProperties; -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.annotation.Primary; -import io.micronaut.context.annotation.Requires; -import io.micronaut.core.annotation.Introspected; -import io.micronaut.jdbc.BasicJdbcConfiguration; -import jakarta.inject.Named; -import jakarta.inject.Singleton; -import java.util.Map; - -@Requires(property = "lighter.storage.jdbc.enabled", value = "true") -@Factory -public class LighterJdbcDatasourceConfiguration { - - @Singleton - @Named("default") - public DatasourceConfiguration datasourceConfiguration(JdbcConnectionConfiguration configuration) { - var ds = new DatasourceConfiguration("default"); - ds.setUrl(configuration.url); - ds.setUsername(configuration.username); - ds.setPassword(configuration.password); - ds.setDriverClassName(configuration.driverClassName); - return ds; - } - - @Primary - @Introspected - @ConfigurationProperties("lighter.storage.jdbc") - public static class JdbcConnectionConfiguration implements BasicJdbcConfiguration { - - private String url; - private String password; - private String username; - private String driverClassName; - - @Override - public String getName() { - return null; - } - - @Override - public String getConfiguredUrl() { - return url; - } - - @Override - public String getUrl() { - return url; - } - - @Override - public void setUrl(String url) { - this.url = url; - } - - @Override - public String getConfiguredDriverClassName() { - return driverClassName; - } - - @Override - public String getDriverClassName() { - return driverClassName; - } - - @Override - public void setDriverClassName(String driverClassName) { - this.driverClassName = driverClassName; - } - - @Override - public String getConfiguredUsername() { - return username; - } - - @Override - public String getUsername() { - return username; - } - - @Override - public void setUsername(String username) { - this.username = username; - } - - @Override - public String getConfiguredPassword() { - return password; - } - - @Override - public String getPassword() { - return password; - } - - @Override - public void setPassword(String password) { - this.password = password; - } - - @Override - public String getConfiguredValidationQuery() { - return null; - } - - @Override - public String getValidationQuery() { - return null; - } - - @Override - public void setDataSourceProperties(Map dsProperties) { - } - } - - -} diff --git a/server/src/main/resources/application.yml b/server/src/main/resources/application.yml index 3a7e07a9..fd25054b 100644 --- a/server/src/main/resources/application.yml +++ b/server/src/main/resources/application.yml @@ -1,67 +1,73 @@ lighter: - max-running-jobs: 5 - max-starting-jobs: 5 - spark-history-server-url: http://localhost/spark-history/ - frontend-path: file:${FRONTEND_PATH:../frontend/build} - py-gateway-port: 25333 - url: http://lighter.spark:8080 - session: - timeout-interval: 90m - timeout-active: false - permanent-sessions: [] - schedule-interval: 1m - track-running-interval: 2m - kubernetes: - enabled: false - master: k8s://kubernetes.default.svc.cluster.local:443 - namespace: spark - service-account: spark - max-log-size: 500 - storage: - jdbc: - enabled: true - url: jdbc:h2:mem:lighter;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;MODE=PostgreSQL - username: sa - password: "" - driverClassName: org.h2.Driver + max-running-jobs: 5 + max-starting-jobs: 5 + spark-history-server-url: http://localhost/spark-history/ + frontend-path: file:${FRONTEND_PATH:../frontend/build} + py-gateway-port: 25333 + url: http://lighter.spark:8080 + session: + timeout-interval: 90m + timeout-active: false + permanent-sessions: [ ] + schedule-interval: 1m + track-running-interval: 2m + kubernetes: + enabled: false + master: k8s://kubernetes.default.svc.cluster.local:443 + namespace: spark + service-account: spark + max-log-size: 500 + storage: + jdbc: + url: jdbc:h2:mem:lighter;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;MODE=PostgreSQL + username: sa + password: "" + driverClassName: org.h2.Driver # Micronaut specific configs jackson: - serializationInclusion: ALWAYS - serialization: - writeDatesAsTimestamps: false + serializationInclusion: ALWAYS + serialization: + writeDatesAsTimestamps: false shedlock: - defaults: - lock-at-most-for: 10m + defaults: + lock-at-most-for: 10m flyway: - datasources: - default: - enabled: true + datasources: + default: + enabled: true + +datasources: + default: + url: ${lighter.storage.jdbc.url} + username: ${lighter.storage.jdbc.username} + password: ${lighter.storage.jdbc.password} + driverClassName: ${lighter.storage.jdbc.driverClassName} micronaut: - server: - thread-selection: AUTO - cors: - enabled: true - application: - name: lighter - router: - static-resources: - swagger: - paths: classpath:META-INF/swagger - mapping: /swagger/** - swagger-ui: - enabled: true - paths: classpath:META-INF/swagger/views/swagger-ui - mapping: /swagger-ui/** + server: + thread-selection: AUTO + cors: + enabled: true + application: + name: lighter + router: + static-resources: + swagger: + paths: classpath:META-INF/swagger + mapping: /swagger/** + swagger-ui: + enabled: true + paths: classpath:META-INF/swagger/views/swagger-ui + mapping: /swagger-ui/** - metrics: + metrics: + enabled: true + export: + prometheus: enabled: true - export: - prometheus: - enabled: true - step: PT1M - descriptions: true + step: PT1M + descriptions: true diff --git a/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorageTest.groovy b/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorageTest.groovy index 530dbfa9..e348a528 100644 --- a/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorageTest.groovy +++ b/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcApplicationStorageTest.groovy @@ -6,7 +6,7 @@ import com.exacaster.lighter.application.ApplicationType import com.exacaster.lighter.storage.SortOrder import io.micronaut.test.extensions.spock.annotation.MicronautTest import jakarta.inject.Inject -import javax.transaction.Transactional +import jakarta.transaction.Transactional import spock.lang.Specification import static com.exacaster.lighter.test.Factories.newApplication diff --git a/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcStatementStorageTest.groovy b/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcStatementStorageTest.groovy index 8af42931..802c7851 100644 --- a/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcStatementStorageTest.groovy +++ b/server/src/test/groovy/com/exacaster/lighter/storage/jdbc/JdbcStatementStorageTest.groovy @@ -6,7 +6,7 @@ import io.micronaut.test.extensions.spock.annotation.MicronautTest import jakarta.inject.Inject import spock.lang.Specification -import javax.transaction.Transactional +import jakarta.transaction.Transactional import static com.exacaster.lighter.test.Factories.newSession import static com.exacaster.lighter.test.Factories.statement