[TH2-5179] Implemented value formatter related to JDBC database type

.github/workflows/build-dev-release.yml

@@ -28,6 +28,7 @@ jobs:
       app: ${{ steps.filter.outputs.app }}
       grpc: ${{ steps.filter.outputs.grpc }}
       core: ${{ steps.filter.outputs.core }}
+      oracle: ${{ steps.filter.outputs.oracle }}
     steps:
       - uses: actions/checkout@v3
       - uses: dorny/paths-filter@v2
@@ -41,6 +42,8 @@ jobs:
               - 'grpc/gradle.properties'
             core:
               - 'core/gradle.properties'
+            oracle:
+              - 'oracle/gradle.properties'
   grpc-version:
     name: Extract grpc version
     uses: th2-net/.github/.github/workflows/compound-prebuild-java-workflow.yml@main
@@ -51,6 +54,11 @@ jobs:
     uses: th2-net/.github/.github/workflows/compound-prebuild-java-workflow.yml@main
     with:
       project-path: app
+  oracle-version:
+    name: Extract oracle version
+    uses: th2-net/.github/.github/workflows/compound-prebuild-java-workflow.yml@main
+    with:
+      project-path: oracle
 
   build-and-publish-grpc-java:
     name: Build and publish Java gRPC distributions
@@ -114,6 +122,32 @@ jobs:
           ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
           ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
           ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+
+  build-and-publish-oracle-extension-java:
+    name: Build and publish Java oracle extension distributions
+    runs-on: 'ubuntu-20.04'
+    needs:
+      - build-and-publish-core-java
+      - changes
+    if: |
+      always() &&
+      (needs.changes.outputs.oracle == 'true' || needs.changes.outputs.core == 'true')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 'zulu' '11'
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '11'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+      - name: Build with Gradle
+        run: ./gradlew --info -Pversion_suffix='dev' clean build :read-db-oracle-extension:publish closeAndReleaseSonatypeStagingRepository
+        env:
+          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
+          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
+          ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
+          ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
   owasp-scan:
     needs:
       - build-and-publish-core-java
@@ -125,12 +159,12 @@ jobs:
   publish-docker:
     name: Build and publish docker image
     needs:
-      - build-and-publish-core-java
+      - build-and-publish-oracle-extension-java
       - app-version
       - changes
     if: |
       always() &&
-      (needs.changes.outputs.app == 'true' || needs.changes.outputs.grpc == 'true' || needs.changes.outputs.core == 'true')
+      (needs.changes.outputs.app == 'true' || needs.changes.outputs.grpc == 'true' || needs.changes.outputs.core == 'true' || needs.changes.outputs.oracle == 'true')
     uses: th2-net/.github/.github/workflows/compaund-java-docker-push.yml@main
     with:
       docker-username: ${{ github.actor }}
@@ -159,6 +193,14 @@ jobs:
     with:
       runsOn: ${{ inputs.runsOn }}
       tagName: ${{ needs.grpc-version.outputs.version }}-grpc-dev
+  create-oracle-tag:
+    if: needs.build-and-publish-oracle-extension-java.result == 'success'&& !cancelled()
+    name: Create and push oracle extension tag
+    needs: [ build-and-publish-oracle-extension-java ]
+    uses: th2-net/.github/.github/workflows/compaund-git-tag-push.yml@main
+    with:
+      runsOn: ${{ inputs.runsOn }}
+      tagName: ${{ needs.oracle-version.outputs.version }}-oracle-dev
   create-app-tag:
     if: needs.publish-docker.result == 'success' && needs.build-and-publish-core-java.result == 'success' && !cancelled()
     name: Create and push app tag

.github/workflows/build-release.yml

@@ -8,6 +8,11 @@ jobs:
     uses: th2-net/.github/.github/workflows/compound-prebuild-java-workflow.yml@main
     with:
       project-path: grpc
+  oracle-version:
+    name: Extract app version
+    uses: th2-net/.github/.github/workflows/compound-prebuild-java-workflow.yml@main
+    with:
+      project-path: oracle
   app-version:
     name: Extract app version
     uses: th2-net/.github/.github/workflows/compound-prebuild-java-workflow.yml@main
@@ -20,6 +25,7 @@ jobs:
       app: ${{ steps.filter.outputs.app }}
       grpc: ${{ steps.filter.outputs.grpc }}
       core: ${{ steps.filter.outputs.core }}
+      oracle: ${{ steps.filter.outputs.oracle }}
     steps:
       - uses: dorny/paths-filter@v2
         id: filter
@@ -31,6 +37,8 @@ jobs:
               - 'grpc/gradle.properties'
             core:
               - 'core/gradle.properties'
+            oracle:
+              - 'oracle/gradle.properties'
   build-and-publish-grpc-java:
     name: Build and publish Java gRPC distributions
     needs:
@@ -72,6 +80,22 @@ jobs:
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
       sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
       sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+  build-and-publish-oracle-extension-java:
+    name: Build and publish Java oracle extension distributions
+    needs:
+      - changes
+      - build-and-publish-core-java
+    if: ${{ needs.changes.outputs.oracle == 'true' }}
+    uses: th2-net/.github/.github/workflows/compaund-java-multi-project-build-release.yml@main
+    with:
+      projectPath: oracle
+      release: true
+      additionalTasks: '["publish"]'
+    secrets:
+      sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
+      sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
+      sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
   owasp-scan:
     needs:
       - build-and-publish-core-java
@@ -84,11 +108,12 @@ jobs:
     name: Build and publish docker image
     if: |
       always() &&
-      (needs.changes.outputs.app == 'true' || needs.changes.outputs.grpc == 'true' || needs.changes.outputs.core == 'true')
+      (needs.changes.outputs.app == 'true' || needs.changes.outputs.grpc == 'true' || needs.changes.outputs.core == 'true' || needs.changes.outputs.oracle == 'true')
     needs:
       - build-and-publish-grpc-python
       - build-and-publish-grpc-java
       - build-and-publish-core-java
+      - build-and-publish-oracle-extension-java
       - app-version
     uses: th2-net/.github/.github/workflows/compaund-java-docker-push.yml@main
     with:
@@ -118,6 +143,14 @@ jobs:
     with:
       runsOn: ${{ inputs.runsOn }}
       tagName: ${{ needs.grpc-version.outputs.version }}-grpc
+  create-oracle-tag:
+    if: needs.build-and-publish-oracle-extension-java.result == 'success' && !cancelled()
+    name: Create and push oracle extension tag
+    needs: [ build-and-publish-oracle-extension-java ]
+    uses: th2-net/.github/.github/workflows/compaund-git-tag-push.yml@main
+    with:
+      runsOn: ${{ inputs.runsOn }}
+      tagName: ${{ needs.oracle-version.outputs.version }}-oracle
   create-app-tag:
     if: needs.publish-docker.result == 'success' && needs.build-and-publish-core-java.result == 'success' && !cancelled()
     name: Create and push app tag

.github/workflows/build-sanpshot.yml

@@ -54,9 +54,19 @@ jobs:
       sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
       sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
       sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
+  build-and-publish-oracle-extension-java:
+    name: Build and publish Java oracle extension distributions
+    uses: th2-net/.github/.github/workflows/compaund-java-multi-project-build.yml@main
+    with:
+      projectPath: oracle
+      additionalTasks: '["publish"]'
+      release: false
+    secrets:
+      sonatypeUsername: ${{ secrets.SONATYPE_NEXUS_USERNAME }}
+      sonatypePassword: ${{ secrets.SONATYPE_NEXUS_PASSWORD }}
+      sonatypeSigningKey: ${{ secrets.SONATYPE_GPG_ARMORED_KEY }}
+      sonatypeSigningPassword: ${{ secrets.SONATYPE_SIGNING_PASSWORD }}
   owasp-scan:
-    needs:
-      - build-and-publish-core-java
     uses: th2-net/.github/.github/workflows/owasp-gradle-scan.yml@main
     with:
       multiproject: true

README.md

@@ -1,4 +1,4 @@
-# th2-read-db 0.9.0
+# th2-read-db 0.9.1
 
 The read-db is a component for extracting data from databases using JDBC technology. If database has JDBC driver the read can work with the database
 
@@ -339,6 +339,10 @@ spec:
 
 ## Changes
 
+### 0.9.1
+
++ fixed the pull continuation failure when column with oracle DATE type is used for update query. 
+
 ### 0.9.0
 
 + implemented gRPC backpressure for the `Execute` method

app/build.gradle.kts

@@ -11,16 +11,28 @@ configurations.all {
 dependencies {
     implementation(project(":read-db-core"))
 
-    //region Drivers
+    //region postgresql
     runtimeOnly("org.postgresql:postgresql:42.7.3") {
         because("prostresql support")
     }
+    //endregion
+
+    //region mysql
     runtimeOnly("com.mysql:mysql-connector-j:8.3.0") {
         because("mysql support")
     }
+    //endregion
+
+    //region oracle
     runtimeOnly("com.oracle.database.jdbc:ojdbc11:23.3.0.23.09") {
         because("oracle support")
     }
+    runtimeOnly(project(":read-db-oracle-extension")) {
+        because("oracle support")
+    }
+    //endregion
+
+    //region mssql
     runtimeOnly("com.microsoft.sqlserver:mssql-jdbc:12.4.0.jre11") {
         because("mssql support")
     }

build.gradle.kts

@@ -3,6 +3,7 @@ plugins {
     id("com.exactpro.th2.gradle.publish") version "0.0.4"
 
     kotlin("jvm") version "1.8.22" apply false
+    kotlin("kapt") version "1.8.22" apply false
     id("com.exactpro.th2.gradle.grpc") version "0.0.4" apply false
     id("com.exactpro.th2.gradle.component") version "0.0.4" apply false
 }

core/src/main/kotlin/com/exactpro/th2/read/db/app/DataBaseReader.kt

@@ -28,6 +28,7 @@ import com.exactpro.th2.read.db.core.RowListener
 import com.exactpro.th2.read.db.core.TableRow
 import com.exactpro.th2.read.db.core.TaskId
 import com.exactpro.th2.read.db.core.UpdateListener
+import com.exactpro.th2.read.db.core.ValueTransformProvider
 import com.exactpro.th2.read.db.core.impl.DataBaseMonitorServiceImpl
 import com.exactpro.th2.read.db.core.impl.DataBaseServiceImpl
 import com.exactpro.th2.read.db.core.impl.BaseDataSourceProvider
@@ -210,7 +211,8 @@ class DataBaseReader(
         ): DataBaseReader {
             val sourceProvider: DataSourceProvider = BaseDataSourceProvider(configuration.dataSources)
             val queryProvider: QueryProvider = BaseQueryProvider(configuration.queries)
-            val dataBaseService: DataBaseService = DataBaseServiceImpl(sourceProvider, queryProvider)
+            val transformProvider: ValueTransformProvider = ValueTransformProvider.create(configuration.dataSources)
+            val dataBaseService: DataBaseService = DataBaseServiceImpl(sourceProvider, queryProvider, transformProvider)
             val hashService: HashService = BaseHashServiceImpl(sourceProvider, queryProvider)
             val monitorService: DataBaseMonitorService = DataBaseMonitorServiceImpl(dataBaseService, hashService, clock)
             return DataBaseReader(

core/src/main/kotlin/com/exactpro/th2/read/db/bootstrap/Main.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -84,32 +84,23 @@ fun main(args: Array<String>) {
     // Configure shutdown hook for closing all resources
     // and the lock condition to await termination.
     //
-    // If you use the logic that doesn't require additional threads
+    // If you use the logic that doesn't require additional threads,
     // and you can run everything on main thread
     // you can omit the part with locks (but please keep the resources queue)
-    val resources: Deque<() -> Unit> = ConcurrentLinkedDeque()
+    val resourceRegister = ResourceRegister()
     val lock = ReentrantLock()
     val condition: Condition = lock.newCondition()
-    configureShutdownHook(resources, lock, condition)
+    configureShutdownHook(resourceRegister, lock, condition)
 
     try {
         // You need to initialize the CommonFactory
-        val resourceHandler: (name: String, resource: () -> Unit) -> Unit = { name, resource ->
-            resources += {
-                LOGGER.info { "Closing resource $name" }
-                runCatching(resource).onFailure {
-                    LOGGER.error(it) { "cannot close resource $name" }
-                }
-            }
-        }
-
         // You can use custom paths to each config that is required for the CommonFactory
         // If args are empty the default path will be chosen.
         val factory = CommonFactory.createFromArguments(*args)
         // do not forget to add resource to the resources queue
-        resourceHandler("common factory", factory::close)
+        resourceRegister.add("common factory", factory::close)
 
-        setupApp(factory, resourceHandler)
+        setupApp(factory, resourceRegister::add)
 
         awaitShutdown(lock, condition)
     } catch (ex: Exception) {
@@ -118,7 +109,31 @@ fun main(args: Array<String>) {
     }
 }
 
-internal fun setupApp(
+class ResourceRegister: AutoCloseable {
+    private val resources: Deque<() -> Unit> = ConcurrentLinkedDeque()
+
+    fun add(name: String, resource: () -> Unit) {
+        resources += {
+            LOGGER.info { "Closing resource $name" }
+            runCatching(resource).onFailure {
+                LOGGER.error(it) { "cannot close resource $name" }
+            }
+        }
+    }
+
+    override fun close() {
+        resources.descendingIterator().forEachRemaining { resource ->
+            try {
+                resource()
+            } catch (e: Exception) {
+                LOGGER.error(e) { "Cannot close resource ${resource::class}" }
+            }
+        }
+    }
+
+}
+
+fun setupApp(
     factory: CommonFactory,
     closeResource: (name: String, resource: () -> Unit) -> Unit,
 ) {
@@ -404,7 +419,7 @@ private class TransportPreprocessor(bookName: String) : Preprocessor<RawMessage.
 
 private data class SessionKey<DIRECTION>(val alias: String, val direction: DIRECTION)
 
-private fun configureShutdownHook(resources: Deque<() -> Unit>, lock: ReentrantLock, condition: Condition) {
+private fun configureShutdownHook(resourceRegister: ResourceRegister, lock: ReentrantLock, condition: Condition) {
     Runtime.getRuntime().addShutdownHook(thread(
         start = false,
         name = "Shutdown hook"
@@ -417,13 +432,7 @@ private fun configureShutdownHook(resources: Deque<() -> Unit>, lock: ReentrantL
         } finally {
             lock.unlock()
         }
-        resources.descendingIterator().forEachRemaining { resource ->
-            try {
-                resource()
-            } catch (e: Exception) {
-                LOGGER.error(e) { "Cannot close resource ${resource::class}" }
-            }
-        }
+        resourceRegister.close()
         LIVENESS_MONITOR.disable()
         LOGGER.info { "Shutdown end" }
     })

core/src/main/kotlin/com/exactpro/th2/read/db/core/Model.kt

@@ -37,7 +37,7 @@ data class QueryId(val id: String)
 data class TaskId(val id: String)
 
 data class TableRow(
-    val columns: Map<String, Any?>,
+    val columns: Map<String, String?>,
     val associatedMessageType: String? = null,
     val executionId: Long? = null
 )