[TH2-5268] Added fetchSize option for query configuration.

.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+  - package-ecosystem: gradle
+    directory: /
+    reviewers:
+      - Nikita-Smirnov-Exactpro
+      - OptimumCode
+    labels:
+      - dependencies
+    schedule:
+      interval: daily
+  - package-ecosystem: github-actions
+    directory: /
+    reviewers:
+      - Nikita-Smirnov-Exactpro
+      - OptimumCode
+    labels:
+      - dependencies
+    schedule:
+      interval: daily

.github/workflows/ci-unwelcome-words.yml

@@ -5,6 +5,7 @@ on:
 
 jobs:
   test:
+    if: github.actor != 'dependabot[bot]'
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3

.github/workflows/scan.yml

@@ -0,0 +1,12 @@
+name: Scan licenses and vulnerabilities in java project
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 1'
+
+jobs:
+  build:
+    uses: th2-net/.github/.github/workflows/compound-java-scan.yml@main
+    secrets:
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}

README.md

@@ -1,4 +1,4 @@
-# th2-read-db 0.11.0
+# th2-read-db 0.12.0
 
 The read-db is a component for extracting data from databases using JDBC technology. If database has JDBC driver the read can work with the database
 
@@ -72,6 +72,8 @@ The list of queries that can be executed by read-db.
 + defaultParameters - the default values for parameters. They will be used if the parameter was not specified in the request
 + messageType - the message type that should be associated with this query.
   If it is set the read-db will set a property `th2.csv.override_message_type` with specified value
++ fetchSize - the number of rows that should be fetched from the database when query is executed.
+  Default value is 0 that means all rows are fetched to application instantly as database returns them.
 
 ### startupTasks
 
@@ -339,6 +341,21 @@ spec:
 
 ## Changes
 
+### 0.12.0
++ added `fetchSize` option for query configuration.
++ updated:
+  + th2 gradle plugin: `0.1.6` based on bom: `4.9.0`
+  + common: `5.14.0-dev`
+  + common-utils: `2.3.0-dev`
+  + commons-dbcp2: `2.13.0`
++ updated jdbc:
+  + postgresql: `42.7.4`
+  + mysql-connector-j: `8.4.0`
+  + ojdbc11: `23.6.0.24.10`
+  + mssql-jdbc: `12.8.1.jre11`
++ migrated:
+  + io.github.oshai:kotlin-logging: `5.1.4`
+
 ### 0.11.0
 
 + updated th2 gradle plugin `0.0.8`

app/gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
-release_version=0.11.0
+release_version=0.12.0
 description=read-db component for extracting data from databases using JDBC technology

core/build.gradle.kts

@@ -45,6 +45,7 @@ dependencies {
     testImplementation("org.testcontainers:testcontainers")
     testImplementation("org.testcontainers:mysql")
     testImplementation("org.testcontainers:oracle-xe")
+    testImplementation("com.google.protobuf:protobuf-java")
     testImplementation("io.grpc:grpc-testing")
     testImplementation("io.grpc:grpc-inprocess")
 

core/gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
-release_version=0.11.0
+release_version=0.12.0
 description=core part of read db to create an application with required JDBC drivers in the classpath

core/src/main/kotlin/com/exactpro/th2/read/db/app/DataBaseReader.kt

@@ -29,16 +29,16 @@ import com.exactpro.th2.read.db.core.TableRow
 import com.exactpro.th2.read.db.core.TaskId
 import com.exactpro.th2.read.db.core.UpdateListener
 import com.exactpro.th2.read.db.core.ValueTransformProvider
-import com.exactpro.th2.read.db.core.impl.DataBaseMonitorServiceImpl
-import com.exactpro.th2.read.db.core.impl.DataBaseServiceImpl
 import com.exactpro.th2.read.db.core.impl.BaseDataSourceProvider
 import com.exactpro.th2.read.db.core.impl.BaseHashServiceImpl
 import com.exactpro.th2.read.db.core.impl.BaseQueryProvider
+import com.exactpro.th2.read.db.core.impl.DataBaseMonitorServiceImpl
+import com.exactpro.th2.read.db.core.impl.DataBaseServiceImpl
 import com.exactpro.th2.read.db.core.util.runCatchingException
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.onCompletion
 import kotlinx.coroutines.launch
-import mu.KotlinLogging
 import java.time.Clock
 import java.time.Duration
 

core/src/main/kotlin/com/exactpro/th2/read/db/bootstrap/Main.kt

@@ -50,12 +50,12 @@ import com.exactpro.th2.read.db.impl.grpc.DataBaseReaderGrpcServer
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 import com.google.protobuf.Timestamp
 import com.google.protobuf.util.Timestamps
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.cancel
-import mu.KotlinLogging
 import java.time.Duration
 import java.time.Instant
 import java.util.Deque

core/src/main/kotlin/com/exactpro/th2/read/db/bootstrap/Saver.kt

@@ -17,7 +17,7 @@
 package com.exactpro.th2.read.db.bootstrap
 
 import com.exactpro.th2.read.db.core.util.runCatchingException
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import java.time.Duration
 import java.util.concurrent.BlockingQueue
 import java.util.concurrent.TimeUnit

core/src/main/kotlin/com/exactpro/th2/read/db/core/DataBaseMonitorService.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,8 +17,8 @@
 package com.exactpro.th2.read.db.core
 
 import com.exactpro.th2.read.db.app.ResetState
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineScope
-import mu.KotlinLogging
 import java.time.Duration
 import java.time.Instant
 import java.time.LocalTime

core/src/main/kotlin/com/exactpro/th2/read/db/core/QueryConfiguration.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,4 +23,5 @@ data class QueryConfiguration(
     val query: String,
     val defaultParameters: QueryParametersValues = emptyMap(),
     val messageType: String? = null,
+    val fetchSize: Int = 0
 )

core/src/main/kotlin/com/exactpro/th2/read/db/core/QueryHolder.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ data class QueryHolder(
     val parameters: Map<String, List<ParameterInfo>>,
     val defaultParameters: QueryParametersValues,
     val associatedMessageType: String? = null,
+    val fetchSize: Int = 0
 )
 
 data class ParameterInfo(

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/BaseDataSourceProvider.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import com.exactpro.th2.read.db.core.DataSourceConfiguration
 import com.exactpro.th2.read.db.core.DataSourceHolder
 import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.DataSourceProvider
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.apache.commons.dbcp2.BasicDataSource
 
 class BaseDataSourceProvider(

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/BaseQueryProvider.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ import com.exactpro.th2.read.db.core.QueryConfiguration
 import com.exactpro.th2.read.db.core.QueryHolder
 import com.exactpro.th2.read.db.core.QueryId
 import com.exactpro.th2.read.db.core.QueryProvider
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.apache.commons.text.StringSubstitutor
 import org.apache.commons.text.lookup.StringLookup
 import java.sql.JDBCType
@@ -40,7 +40,7 @@ class BaseQueryProvider(
                 LOGGER.warn { "Default parameter $it was not found in query. Known parameters: ${parameters.keys}" }
             }
         }
-        QueryHolder(processedQuery, parameters, cfg.defaultParameters, cfg.messageType).also {
+        QueryHolder(processedQuery, parameters, cfg.defaultParameters, cfg.messageType, cfg.fetchSize).also {
             LOGGER.trace { "Holder for $id query created: $it" }
         }
     }

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/DataBaseMonitorServiceImpl.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,11 @@
 package com.exactpro.th2.read.db.core.impl
 
 import com.exactpro.th2.read.db.app.ResetState
-import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.DataBaseMonitorService
 import com.exactpro.th2.read.db.core.DataBaseMonitorService.Companion.TH2_PULL_TASK_UPDATE_HASH_PROPERTY
 import com.exactpro.th2.read.db.core.DataBaseMonitorService.Companion.calculateNearestResetDate
 import com.exactpro.th2.read.db.core.DataBaseService
+import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.HashService
 import com.exactpro.th2.read.db.core.HashService.Companion.calculateHash
 import com.exactpro.th2.read.db.core.MessageLoader
@@ -30,6 +30,7 @@ import com.exactpro.th2.read.db.core.QueryParametersValues
 import com.exactpro.th2.read.db.core.TableRow
 import com.exactpro.th2.read.db.core.TaskId
 import com.exactpro.th2.read.db.core.UpdateListener
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.cancelAndJoin
@@ -40,7 +41,6 @@ import kotlinx.coroutines.flow.onCompletion
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
-import mu.KotlinLogging
 import java.time.Clock
 import java.time.Duration
 import java.time.Instant

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/DataBaseServiceImpl.kt

@@ -16,9 +16,9 @@
 
 package com.exactpro.th2.read.db.core.impl
 
+import com.exactpro.th2.read.db.core.DataBaseService
 import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.DataSourceProvider
-import com.exactpro.th2.read.db.core.DataBaseService
 import com.exactpro.th2.read.db.core.QueryHolder
 import com.exactpro.th2.read.db.core.QueryId
 import com.exactpro.th2.read.db.core.QueryParametersValues
@@ -32,10 +32,10 @@ import com.exactpro.th2.read.db.core.util.getColumnValue
 import com.exactpro.th2.read.db.core.util.runCatchingException
 import com.exactpro.th2.read.db.core.util.set
 import com.exactpro.th2.read.db.core.util.setCollection
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.onCompletion
-import mu.KotlinLogging
 import java.sql.Connection
 import java.sql.ResultSet
 import javax.sql.DataSource
@@ -128,6 +128,7 @@ class DataBaseServiceImpl(
                 }
             }
         }
+        fetchSize = holder.fetchSize
         LOGGER.trace { "Execute query: $holder" }
         executeQuery()
     }

core/src/main/kotlin/com/exactpro/th2/read/db/core/util/StatementUtils.kt

@@ -16,7 +16,7 @@
 
 package com.exactpro.th2.read.db.core.util
 
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import java.sql.Array
 import java.sql.Date
 import java.sql.PreparedStatement

core/src/main/kotlin/com/exactpro/th2/read/db/impl/grpc/DataBaseReaderGrpcServer.kt

@@ -43,13 +43,13 @@ import com.exactpro.th2.read.db.impl.grpc.util.toGrpc
 import com.exactpro.th2.read.db.impl.grpc.util.toModel
 import com.google.protobuf.Empty
 import com.google.protobuf.Message
+import io.github.oshai.kotlinlogging.KotlinLogging
 import io.grpc.Status
 import io.grpc.StatusRuntimeException
 import io.grpc.stub.ServerCallStreamObserver
 import io.grpc.stub.StreamObserver
 import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.channels.ClosedReceiveChannelException
-import mu.KotlinLogging
 import java.time.Instant
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicLong

core/src/test/kotlin/com/exactpro/th2/read/db/app/DataBaseReaderIntegrationTest.kt

@@ -34,13 +34,13 @@ import com.exactpro.th2.read.db.core.ValueTransformProvider.Companion.DEFAULT_TR
 import com.exactpro.th2.read.db.core.impl.BaseDataSourceProvider
 import com.exactpro.th2.read.db.core.impl.BaseHashServiceImpl
 import com.exactpro.th2.read.db.core.impl.BaseQueryProvider
+import io.github.oshai.kotlinlogging.KotlinLogging
 import io.netty.buffer.ByteBufUtil.decodeHexDump
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.advanceTimeBy
 import kotlinx.coroutines.test.advanceUntilIdle
 import kotlinx.coroutines.test.runCurrent
 import kotlinx.coroutines.test.runTest
-import mu.KotlinLogging
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertNotNull

core/src/test/kotlin/com/exactpro/th2/read/db/app/DataBaseReaderOracleIntegrationTest.kt

@@ -30,11 +30,11 @@ import com.exactpro.th2.read.db.core.QueryId
 import com.exactpro.th2.read.db.core.RowListener
 import com.exactpro.th2.read.db.core.TableRow
 import com.exactpro.th2.read.db.core.UpdateListener
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.test.advanceTimeBy
 import kotlinx.coroutines.test.advanceUntilIdle
 import kotlinx.coroutines.test.runTest
-import mu.KotlinLogging
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertNotNull

core/src/test/kotlin/com/exactpro/th2/read/db/app/FullReadDbIntegrationTest.kt

@@ -38,7 +38,7 @@ import com.exactpro.th2.test.spec.pin
 import com.exactpro.th2.test.spec.pins
 import com.exactpro.th2.test.spec.publishers
 import com.exactpro.th2.test.spec.server
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.BeforeAll

core/src/test/kotlin/com/exactpro/th2/read/db/impl/grpc/DataBaseReaderGrpcServerIntegrationTest.kt

@@ -41,6 +41,7 @@ import com.exactpro.th2.read.db.grpc.ReadDbGrpc
 import com.exactpro.th2.read.db.impl.grpc.util.toModel
 import com.google.protobuf.ByteString
 import com.google.protobuf.util.Timestamps
+import io.github.oshai.kotlinlogging.KotlinLogging
 import io.grpc.BindableService
 import io.grpc.Context
 import io.grpc.ManagedChannel
@@ -50,7 +51,6 @@ import io.grpc.inprocess.InProcessServerBuilder
 import io.netty.buffer.ByteBufUtil.decodeHexDump
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.runBlocking
-import mu.KotlinLogging
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.BeforeAll