[TH2-5268] Added fetchSize option for query configuration. (#29)

* [TH2-5268] update libraries

.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+  - package-ecosystem: gradle
+    directory: /
+    reviewers:
+      - Nikita-Smirnov-Exactpro
+      - OptimumCode
+    labels:
+      - dependencies
+    schedule:
+      interval: daily
+  - package-ecosystem: github-actions
+    directory: /
+    reviewers:
+      - Nikita-Smirnov-Exactpro
+      - OptimumCode
+    labels:
+      - dependencies
+    schedule:
+      interval: daily

.github/workflows/build-dev-release.yml

@@ -153,6 +153,13 @@ jobs:
       multiproject: true
     secrets:
       nvd-api-key: ${{ secrets.NVD_APIKEY }}
+  license-scan:
+    if: "!cancelled()"
+    uses: th2-net/.github/.github/workflows/license_check.yml@main
+    needs:
+      - app-version
+    with:
+      version: ${{ needs.app-version.outputs.version }}-dev
   publish-docker:
     name: Build and publish docker image
     needs:

.github/workflows/build-release.yml

@@ -103,6 +103,13 @@ jobs:
       multiproject: true
     secrets:
       nvd-api-key: ${{ secrets.NVD_APIKEY }}
+  license-scan:
+    if: "!cancelled()"
+    uses: th2-net/.github/.github/workflows/license_check.yml@main
+    needs:
+      - app-version
+    with:
+      version: ${{ needs.app-version.outputs.version }}
   publish-docker:
     name: Build and publish docker image
     if: |

.github/workflows/build-sanpshot.yml

@@ -73,6 +73,13 @@ jobs:
       multiproject: true
     secrets:
       nvd-api-key: ${{ secrets.NVD_APIKEY }}
+  license-scan:
+    if: "!cancelled()"
+    uses: th2-net/.github/.github/workflows/license_check.yml@main
+    needs:
+      - app-version
+    with:
+      version: ${{ needs.app-version.outputs.version }}
   publish-docker:
     name: Build docker image
     needs:

.github/workflows/ci-unwelcome-words.yml

@@ -5,6 +5,7 @@ on:
 
 jobs:
   test:
+    if: github.actor != 'dependabot[bot]'
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3

.github/workflows/scan.yml

@@ -0,0 +1,14 @@
+name: Scan licenses and vulnerabilities in java project
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 1'
+
+jobs:
+  build:
+    uses: th2-net/.github/.github/workflows/compound-java-scan.yml@main
+    with:
+      multiproject: true
+    secrets:
+      nvd-api-key: ${{ secrets.NVD_APIKEY }}

README.md

@@ -1,4 +1,4 @@
-# th2-read-db 0.11.0
+# th2-read-db 0.12.0
 
 The read-db is a component for extracting data from databases using JDBC technology. If database has JDBC driver the read can work with the database
 
@@ -51,6 +51,12 @@ eventPublication:
 
 ## Parameters
 
+The list of read-db parameters
+
++ defaultQueryFetchSize (default 0) - default value of `fetchSize` option of query parameters defined in `queries` section.
+  if value is 0 that means jdbc driver fetches result row without limitation.
++ useTransport (default false) - use th2 transport or protobuf protocol to publish outgoing messages
+
 ### dataSources
 
 The list of data sources where the read-db can connect.
@@ -72,6 +78,8 @@ The list of queries that can be executed by read-db.
 + defaultParameters - the default values for parameters. They will be used if the parameter was not specified in the request
 + messageType - the message type that should be associated with this query.
   If it is set the read-db will set a property `th2.csv.override_message_type` with specified value
++ fetchSize - the number of rows that should be fetched from the database when query is executed.
+  default value is `defaultQueryFetchSize` from root configuration.
 
 ### startupTasks
 
@@ -225,6 +233,7 @@ spec:
     eventPublication:
       maxBatchSizeInItems: 100
       maxFlushTime: 1000
+    defaultQueryFetchSize: 0
     useTransport: true
   pins:
     - name: client
@@ -302,6 +311,7 @@ spec:
     eventPublication:
       maxBatchSizeInItems: 100
       maxFlushTime: 1000
+    defaultQueryFetchSize: 0
     useTransport: true
   pins:
     mq:
@@ -339,6 +349,22 @@ spec:
 
 ## Changes
 
+### 0.12.0
++ added `fetchSize` option into query configuration.
++ added `defaultQueryFetchSize` option into root configuration.
++ updated:
+  + th2 gradle plugin: `0.1.6` based on bom: `4.9.0`
+  + common: `5.14.0-dev`
+  + common-utils: `2.3.0-dev`
+  + commons-dbcp2: `2.13.0`
++ updated jdbc:
+  + postgresql: `42.7.4`
+  + mysql-connector-j: `8.4.0`
+  + ojdbc11: `23.6.0.24.10`
+  + mssql-jdbc: `12.8.1.jre11`
++ migrated:
+  + io.github.oshai:kotlin-logging: `5.1.4`
+
 ### 0.11.0
 
 + updated th2 gradle plugin `0.0.8`

app/gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
-release_version=0.11.0
+release_version=0.12.0
 description=read-db component for extracting data from databases using JDBC technology

core/gradle.properties

@@ -1,3 +1,3 @@
 kotlin.code.style=official
-release_version=0.11.0
+release_version=0.12.0
 description=core part of read db to create an application with required JDBC drivers in the classpath

core/src/main/kotlin/com/exactpro/th2/read/db/app/DataBaseReader.kt

@@ -29,16 +29,16 @@ import com.exactpro.th2.read.db.core.TableRow
 import com.exactpro.th2.read.db.core.TaskId
 import com.exactpro.th2.read.db.core.UpdateListener
 import com.exactpro.th2.read.db.core.ValueTransformProvider
-import com.exactpro.th2.read.db.core.impl.DataBaseMonitorServiceImpl
-import com.exactpro.th2.read.db.core.impl.DataBaseServiceImpl
 import com.exactpro.th2.read.db.core.impl.BaseDataSourceProvider
 import com.exactpro.th2.read.db.core.impl.BaseHashServiceImpl
 import com.exactpro.th2.read.db.core.impl.BaseQueryProvider
+import com.exactpro.th2.read.db.core.impl.DataBaseMonitorServiceImpl
+import com.exactpro.th2.read.db.core.impl.DataBaseServiceImpl
 import com.exactpro.th2.read.db.core.util.runCatchingException
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.onCompletion
 import kotlinx.coroutines.launch
-import mu.KotlinLogging
 import java.time.Clock
 import java.time.Duration
 
@@ -210,7 +210,7 @@ class DataBaseReader(
             clock: Clock = Clock.systemDefaultZone()
         ): DataBaseReader {
             val sourceProvider: DataSourceProvider = BaseDataSourceProvider(configuration.dataSources)
-            val queryProvider: QueryProvider = BaseQueryProvider(configuration.queries)
+            val queryProvider: QueryProvider = BaseQueryProvider(configuration.queries, configuration.defaultQueryFetchSize)
             val transformProvider: ValueTransformProvider = ValueTransformProvider.create(configuration.dataSources)
             val dataBaseService: DataBaseService = DataBaseServiceImpl(sourceProvider, queryProvider, transformProvider)
             val hashService: HashService = BaseHashServiceImpl(sourceProvider, queryProvider)

core/src/main/kotlin/com/exactpro/th2/read/db/app/DataBaseReaderConfiguration.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,6 +32,7 @@ class DataBaseReaderConfiguration(
     val startupTasks: List<StartupTaskConfiguration> = emptyList(),
     val publication: PublicationConfiguration = PublicationConfiguration(),
     val eventPublication: EventPublicationConfiguration = EventPublicationConfiguration(),
+    val defaultQueryFetchSize: Int = 0,
     val useTransport: Boolean = false
 )
 

core/src/main/kotlin/com/exactpro/th2/read/db/bootstrap/Main.kt

@@ -50,12 +50,12 @@ import com.exactpro.th2.read.db.impl.grpc.DataBaseReaderGrpcServer
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 import com.google.protobuf.Timestamp
 import com.google.protobuf.util.Timestamps
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.cancel
-import mu.KotlinLogging
 import java.time.Duration
 import java.time.Instant
 import java.util.Deque

core/src/main/kotlin/com/exactpro/th2/read/db/bootstrap/Saver.kt

@@ -17,7 +17,7 @@
 package com.exactpro.th2.read.db.bootstrap
 
 import com.exactpro.th2.read.db.core.util.runCatchingException
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import java.time.Duration
 import java.util.concurrent.BlockingQueue
 import java.util.concurrent.TimeUnit

core/src/main/kotlin/com/exactpro/th2/read/db/core/DataBaseMonitorService.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,8 +17,8 @@
 package com.exactpro.th2.read.db.core
 
 import com.exactpro.th2.read.db.app.ResetState
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineScope
-import mu.KotlinLogging
 import java.time.Duration
 import java.time.Instant
 import java.time.LocalTime

core/src/main/kotlin/com/exactpro/th2/read/db/core/QueryConfiguration.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,4 +23,5 @@ data class QueryConfiguration(
     val query: String,
     val defaultParameters: QueryParametersValues = emptyMap(),
     val messageType: String? = null,
+    val fetchSize: Int? = null,
 )

core/src/main/kotlin/com/exactpro/th2/read/db/core/QueryHolder.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ data class QueryHolder(
     val parameters: Map<String, List<ParameterInfo>>,
     val defaultParameters: QueryParametersValues,
     val associatedMessageType: String? = null,
+    val fetchSize: Int = 0
 )
 
 data class ParameterInfo(

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/BaseDataSourceProvider.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import com.exactpro.th2.read.db.core.DataSourceConfiguration
 import com.exactpro.th2.read.db.core.DataSourceHolder
 import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.DataSourceProvider
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.apache.commons.dbcp2.BasicDataSource
 
 class BaseDataSourceProvider(

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/BaseQueryProvider.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,14 +21,15 @@ import com.exactpro.th2.read.db.core.QueryConfiguration
 import com.exactpro.th2.read.db.core.QueryHolder
 import com.exactpro.th2.read.db.core.QueryId
 import com.exactpro.th2.read.db.core.QueryProvider
-import mu.KotlinLogging
+import io.github.oshai.kotlinlogging.KotlinLogging
 import org.apache.commons.text.StringSubstitutor
 import org.apache.commons.text.lookup.StringLookup
 import java.sql.JDBCType
 import java.sql.SQLType
 
 class BaseQueryProvider(
-    queriesConfiguration: Map<QueryId, QueryConfiguration>
+    queriesConfiguration: Map<QueryId, QueryConfiguration>,
+    defaultFetchSize: Int,
 ) : QueryProvider {
     private val queryById: Map<QueryId, QueryHolder> = queriesConfiguration.mapValues { (id, cfg) ->
         LOGGER.trace { "Creating holder for $id query" }
@@ -40,7 +41,13 @@ class BaseQueryProvider(
                 LOGGER.warn { "Default parameter $it was not found in query. Known parameters: ${parameters.keys}" }
             }
         }
-        QueryHolder(processedQuery, parameters, cfg.defaultParameters, cfg.messageType).also {
+        QueryHolder(
+            processedQuery,
+            parameters,
+            cfg.defaultParameters,
+            cfg.messageType,
+            cfg.fetchSize ?: defaultFetchSize
+        ).also {
             LOGGER.trace { "Holder for $id query created: $it" }
         }
     }

core/src/main/kotlin/com/exactpro/th2/read/db/core/impl/DataBaseMonitorServiceImpl.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022-2023 Exactpro (Exactpro Systems Limited)
+ * Copyright 2022-2024 Exactpro (Exactpro Systems Limited)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,11 @@
 package com.exactpro.th2.read.db.core.impl
 
 import com.exactpro.th2.read.db.app.ResetState
-import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.DataBaseMonitorService
 import com.exactpro.th2.read.db.core.DataBaseMonitorService.Companion.TH2_PULL_TASK_UPDATE_HASH_PROPERTY
 import com.exactpro.th2.read.db.core.DataBaseMonitorService.Companion.calculateNearestResetDate
 import com.exactpro.th2.read.db.core.DataBaseService
+import com.exactpro.th2.read.db.core.DataSourceId
 import com.exactpro.th2.read.db.core.HashService
 import com.exactpro.th2.read.db.core.HashService.Companion.calculateHash
 import com.exactpro.th2.read.db.core.MessageLoader
@@ -30,6 +30,7 @@ import com.exactpro.th2.read.db.core.QueryParametersValues
 import com.exactpro.th2.read.db.core.TableRow
 import com.exactpro.th2.read.db.core.TaskId
 import com.exactpro.th2.read.db.core.UpdateListener
+import io.github.oshai.kotlinlogging.KotlinLogging
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.cancelAndJoin
@@ -40,7 +41,6 @@ import kotlinx.coroutines.flow.onCompletion
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
-import mu.KotlinLogging
 import java.time.Clock
 import java.time.Duration
 import java.time.Instant