Merge remote-tracking branch 'refs/remotes/seatunnel-github/dev' into…

… dev
LeonYoah · Aug 15, 2024 · 2c0f5f9 · 2c0f5f9
2 parents e882618 + 16eeb1c
commit 2c0f5f9
Show file tree

Hide file tree

Showing 199 changed files with 6,266 additions and 833 deletions.
diff --git a/.github/workflows/labeler/label-scope-conf.yml b/.github/workflows/labeler/label-scope-conf.yml
@@ -252,6 +252,11 @@ Milvus:
       - changed-files:
           - any-glob-to-any-file: seatunnel-connectors-v2/connector-milvus/**
           - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(milvus)/**'
+activemq:
+  - all:
+      - changed-files:
+          - any-glob-to-any-file: seatunnel-connectors-v2/connector-activemq/**
+          - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(activemq)/**'
 Zeta Rest API:
   - changed-files:
       - any-glob-to-any-file: seatunnel-engine/**/server/rest/**

diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml
@@ -105,4 +105,4 @@ jobs:
                   }
                 }
               }
-            }
+            }
diff --git a/bin/install-plugin.cmd b/bin/install-plugin.cmd
@@ -22,8 +22,8 @@ REM Get seatunnel home
 set "SEATUNNEL_HOME=%~dp0..\"
 echo Set SEATUNNEL_HOME to [%SEATUNNEL_HOME%]
 
-REM Connector default version is 2.3.6, you can also choose a custom version. eg: 2.1.2:  install-plugin.bat 2.1.2
-set "version=2.3.6"
+REM Connector default version is 2.3.7, you can also choose a custom version. eg: 2.1.2:  install-plugin.bat 2.1.2
+set "version=2.3.7"
 if not "%~1"=="" set "version=%~1"
 
 REM Create the lib directory

diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh
@@ -23,8 +23,8 @@
 # get seatunnel home
 SEATUNNEL_HOME=$(cd $(dirname $0);cd ../;pwd)
 
-# connector default version is 2.3.6, you can also choose a custom version. eg: 2.1.2:  sh install-plugin.sh 2.1.2
-version=2.3.6
+# connector default version is 2.3.7, you can also choose a custom version. eg: 2.1.2:  sh install-plugin.sh 2.1.2
+version=2.3.7
 
 if [ -n "$1" ]; then
     version="$1"

diff --git a/config/plugin_config b/config/plugin_config
@@ -86,4 +86,5 @@ connector-rocketmq
 connector-tdengine
 connector-web3j
 connector-milvus
+connector-activemq
 --end--
diff --git a/docs/en/connector-v2/sink/Activemq.md b/docs/en/connector-v2/sink/Activemq.md
@@ -0,0 +1,123 @@
+# Activemq
+
+> Activemq sink connector
+
+## Description
+
+Used to write data to Activemq.
+
+## Key features
+
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+
+## Options
+
+|                name                 |  type   | required | default value |
+|-------------------------------------|---------|----------|---------------|
+| host                                | string  | no       | -             |
+| port                                | int     | no       | -             |
+| virtual_host                        | string  | no       | -             |
+| username                            | string  | no       | -             |
+| password                            | string  | no       | -             |
+| queue_name                          | string  | yes      | -             |
+| uri                                 | string  | yes      | -             |
+| check_for_duplicate                 | boolean | no       | -             |
+| client_id                           | boolean | no       | -             |
+| copy_message_on_send                | boolean | no       | -             |
+| disable_timeStamps_by_default       | boolean | no       | -             |
+| use_compression                     | boolean | no       | -             |
+| always_session_async                | boolean | no       | -             |
+| dispatch_async                      | boolean | no       | -             |
+| nested_map_and_list_enabled         | boolean | no       | -             |
+| warnAboutUnstartedConnectionTimeout | boolean | no       | -             |
+| closeTimeout                        | int     | no       | -             |
+
+### host [string]
+
+the default host to use for connections
+
+### port [int]
+
+the default port to use for connections
+
+### username [string]
+
+the AMQP user name to use when connecting to the broker
+
+### password [string]
+
+the password to use when connecting to the broker
+
+### uri [string]
+
+convenience method for setting the fields in an AMQP URI: host, port, username, password and virtual host
+
+### queue_name [string]
+
+the queue to write the message to
+
+### check_for_duplicate [boolean]
+
+will check for duplucate messages
+
+### client_id [string]
+
+client id
+
+### copy_message_on_send [boolean]
+
+if true, enables new JMS Message object as part of the send method
+
+### disable_timeStamps_by_default [boolean]
+
+disables timestamp for slight performance boost
+
+### use_compression [boolean]
+
+Enables the use of compression on the message’s body.
+
+### always_session_async [boolean]
+
+When true a separate thread is used for dispatching messages for each Session in the Connection.
+
+### always_sync_send [boolean]
+
+When true a MessageProducer will always use Sync sends when sending a Message
+
+### close_timeout [boolean]
+
+Sets the timeout, in milliseconds, before a close is considered complete.
+
+### dispatch_async [boolean]
+
+Should the broker dispatch messages asynchronously to the consumer
+
+### nested_map_and_list_enabled [boolean]
+
+Controls whether Structured Message Properties and MapMessages are supported
+
+### warn_about_unstarted_connection_timeout [int]
+
+The timeout, in milliseconds, from the time of connection creation to when a warning is generated
+
+## Example
+
+simple:
+
+```hocon
+sink {
+      ActiveMQ {
+          uri="tcp://localhost:61616"
+          username = "admin"
+          password = "admin"
+          queue_name = "test1"
+      }
+}
+```
+
+## Changelog
+
+### next version
+
+- Add Activemq Source Connector
+
diff --git a/docs/en/connector-v2/sink/SftpFile.md b/docs/en/connector-v2/sink/SftpFile.md
@@ -36,7 +36,7 @@ By default, we use 2PC commit to ensure `exactly-once`
 |---------------------------------------|---------|----------|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
 | host                                  | string  | yes      | -                                          |                                                                                                                   |
 | port                                  | int     | yes      | -                                          |                                                                                                                   |
-| username                              | string  | yes      | -                                          |                                                                                                                   |
+| user                                  | string  | yes      | -                                          |                                                                                                                   |
 | password                              | string  | yes      | -                                          |                                                                                                                   |
 | path                                  | string  | yes      | -                                          |                                                                                                                   |
 | tmp_path                              | string  | yes      | /tmp/seatunnel                             | The result file will write to a tmp path first and then use `mv` to submit tmp dir to target dir. Need a FTP dir. |
@@ -72,9 +72,9 @@ The target sftp host is required
 
 The target sftp port is required
 
-### username [string]
+### user [string]
 
-The target sftp username is required
+The target sftp user is required
 
 ### password [string]
 
@@ -229,7 +229,7 @@ For text file format with `have_partition` and `custom_filename` and `sink_colum
 SftpFile {
     host = "xxx.xxx.xxx.xxx"
     port = 22
-    username = "username"
+    user = "username"
     password = "password"
     path = "/data/sftp/seatunnel/job1"
     tmp_path = "/data/sftp/seatunnel/tmp"

diff --git a/docs/en/connector-v2/source/Hbase.md b/docs/en/connector-v2/source/Hbase.md
@@ -1,12 +1,12 @@
 # Hbase
 
-> Hbase source connector
+> Hbase Source Connector
 
 ## Description
 
-Read data from Apache Hbase.
+Reads data from Apache Hbase.
 
-## Key features
+## Key Features
 
 - [x] [batch](../../concept/connector-v2-features.md)
 - [ ] [stream](../../concept/connector-v2-features.md)
@@ -17,75 +17,80 @@ Read data from Apache Hbase.
 
 ## Options
 
-|        name        |  type  | required | default value |
-|--------------------|--------|----------|---------------|
-| zookeeper_quorum   | string | yes      | -             |
-| table              | string | yes      | -             |
-| query_columns      | list   | yes      | -             |
-| schema             | config | yes      | -             |
-| hbase_extra_config | string | no       | -             |
-| common-options     |        | no       | -             |
+|        Name        |  Type   | Required | Default |
+|--------------------|---------|----------|---------|
+| zookeeper_quorum   | string  | Yes      | -       |
+| table              | string  | Yes      | -       |
+| schema             | config  | Yes      | -       |
+| hbase_extra_config | string  | No       | -       |
+| caching            | int     | No       | -1      |
+| batch              | int     | No       | -1      |
+| cache_blocks       | boolean | No       | false   |
+| common-options     |         | No       | -       |
 
 ### zookeeper_quorum [string]
 
-The zookeeper cluster host of hbase, example: "hadoop001:2181,hadoop002:2181,hadoop003:2181"
+The zookeeper quorum for Hbase cluster hosts, e.g., "hadoop001:2181,hadoop002:2181,hadoop003:2181".
 
 ### table [string]
 
-The table name you want to write, example: "seatunnel"
-
-### query_columns [list]
-
-The column name which you want to query in the table. If you want to query the rowkey column, please set "rowkey" in query_columns.
-Other column format should be: columnFamily:columnName, example: ["rowkey", "columnFamily1:column1", "columnFamily1:column1", "columnFamily2:column1"]
+The name of the table to write to, e.g., "seatunnel".
 
 ### schema [config]
 
-Hbase uses byte arrays for storage. Therefore, you need to configure data types for each column in a table. For more information, see: [guide](../../concept/schema-feature.md#how-to-declare-type-supported).
+Hbase stores data in byte arrays. Therefore, you need to configure the data types for each column in the table. For more information, see: [guide](../../concept/schema-feature.md#how-to-declare-type-supported).
 
 ### hbase_extra_config [config]
 
-The extra configuration of hbase
+Additional configurations for Hbase.
+
+### caching
+
+The caching parameter sets the number of rows fetched per server trip during scans. This reduces round-trips between client and server, improving scan efficiency. Default: -1.
+
+### batch
+
+The batch parameter sets the maximum number of columns returned per scan. This is useful for rows with many columns to avoid fetching excessive data at once, thus saving memory and improving performance.
 
-### common options
+### cache_blocks
 
-Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details
+The cache_blocks parameter determines whether to cache data blocks during scans. By default, HBase caches data blocks during scans. Setting this to false reduces memory usage during scans. Default in SeaTunnel: false.
 
-## Examples
+### common-options
+
+Common parameters for Source plugins, refer to [Common Source Options](common-options.md).
+
+## Example
 
 ```bash
 source {
   Hbase {
-      zookeeper_quorum = "hadoop001:2181,hadoop002:2181,hadoop003:2181"
-      table = "seatunnel_test"
-      query_columns=["rowkey", "columnFamily1:column1", "columnFamily1:column1", "columnFamily2:column1"]
-      schema = {
-            columns = [
-                  {
-                     name = rowkey
-                     type = string
-                  },
-                  {
-                     name = "columnFamily1:column1"
-                     type = boolean
-                  },
-                  {
-                     name = "columnFamily1:column1"
-                     type = double
-                  },
-                  {
-                     name = "columnFamily2:column1"
-                     type = bigint
-                  }
-            ]
-      }
+    zookeeper_quorum = "hadoop001:2181,hadoop002:2181,hadoop003:2181" 
+    table = "seatunnel_test" 
+    caching = 1000 
+    batch = 100 
+    cache_blocks = false 
+    schema = {
+      columns = [
+        { 
+          name = "rowkey" 
+          type = string 
+        },
+        {
+          name = "columnFamily1:column1"
+          type = boolean
+        },
+        {
+          name = "columnFamily1:column2" 
+          type = double
+        },
+        {
+          name = "columnFamily2:column1"
+          type = bigint
+        }
+      ]
+    }
   }
 }
 ```
 
-## Changelog
-
-### next version
-
-- Add Hbase Source Connector
-
-Original file line number
+Diff line change
@@ Expand Up / @@ -105,4 +105,4 @@ jobs: @@
                       }
                     }
                   }
-                }
+                }