feat: support parameterized query sdk (#248)

* feat: refactor test mode support check logic * feat: add parameterized unit test * cicd: release on tag start with hybridse-v * fix: mini cluster core on mac * feat: add default parameter type * fix: parameterized query engine cache bug fix * feat: support execute sql with parameter * fix: execute request sql bug fix * feat: turn off testing and example build hybridse core * feat: style fix * feat: update hybridse to 0.3.0-0813 * cicd: install hybridse from local source * cicd: install hybridse * feat: run and explain interface rfc * fix: hybridse compile fix * feat: remove session Run() method * fix: insert sdk bug fix for parameterized insert * fix:UninstantiatedParameterizedTestSuite error * feat: add and update changelog * feat: turn on nvl and bitwise function on openmldb * feat: fix bitwise sql by adding ; * feat: renmae interface ExecuteSQLRequest and ExecuteSQLParameterized * feat: support install hybridse from local source ocde * feat: update changelog and fix init env * fix: init env * move setup_thirdparty to openmldb/steps/... * feat: add install thirdsrc in setup thirdparty * feat: change executeSQL with row to ExecuteSQLRequest * feat: update hybridse source with download mode * feat: remove unused log
4paradigm · Aug 20, 2021 · ae71267 · ae71267
1 parent 9d162b4
commit ae71267
Show file tree

Hide file tree

Showing 52 changed files with 996 additions and 397 deletions.
diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml
@@ -11,6 +11,7 @@ on:
 
 env:
   GIT_SUBMODULE_STRATEGY: recursive
+  HYBRIDSE_SOURCE: download
 
 jobs:
   build_and_cpp_ut:
@@ -23,7 +24,7 @@ jobs:
       - uses: actions/checkout@v1
       - name: build
         run: |
-          bash steps/init_env.sh
+          bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }}
           mkdir -p build
           source /root/.bashrc && cd build && cmake -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=OFF -DTESTING_ENABLE=ON .. && make -j$(nproc) && cd ../
           rm -rf thirdparty
@@ -65,7 +66,7 @@ jobs:
       - uses: actions/checkout@v1
       - name: build
         run: |
-          bash steps/init_env.sh
+          bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }}
           mkdir -p build
           source /root/.bashrc && cd build && cmake -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=OFF -DTESTING_ENABLE=ON .. && make -j$(nproc) sql_sdk_test && cd ../
       - name: run sql_sdk_test
@@ -87,7 +88,7 @@ jobs:
       - uses: actions/checkout@v1
       - name: build
         run: |
-          bash steps/init_env.sh
+          bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }}
           mkdir -p build
           source /root/.bashrc && cd build && cmake -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=OFF -DTESTING_ENABLE=ON .. && make -j$(nproc) sql_cluster_test && cd ../
       - name: run sql_sdk_test
@@ -110,7 +111,7 @@ jobs:
 
       - name: build jsdk and package
         run: |
-          bash steps/init_env.sh
+          bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }}
           mkdir -p build
           source /root/.bashrc && cd build && cmake -DSQL_PYSDK_ENABLE=OFF -DSQL_JAVASDK_ENABLE=ON -DTESTING_ENABLE=OFF .. && make -j$(nproc) sql_javasdk_package openmldb && cd ../
 
@@ -140,7 +141,7 @@ jobs:
       - uses: actions/checkout@v1
       - name: build pysdk and sqlalchemy
         run: |
-          bash steps/init_env.sh
+          bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }}
           mkdir -p build
           source /root/.bashrc && cd build && cmake -DSQL_PYSDK_ENABLE=ON -DSQL_JAVASDK_ENABLE=OFF -DTESTING_ENABLE=OFF .. && make -j$(nproc) sqlalchemy_openmldb openmldb && cd ../
       - name: test sqlalchemy
@@ -230,7 +231,7 @@ jobs:
 
       - name: upload to maven
         run: |
-          bash steps/init_env.sh
+          bash steps/init_env.sh ${{ env.HYBRIDSE_SOURCE }}
           VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
           VERSION=$(echo $VERSION | sed -e 's/^v//')
           sh steps/package_openmldb_javasdk.sh $VERSION

diff --git a/.github/workflows/hybridse-ci.yml b/.github/workflows/hybridse-ci.yml
@@ -5,7 +5,7 @@ on:
     branches:
       - main
     tags:
-      - v*
+      - hybridse-v*
   pull_request:
     paths:
       - .github/workflows/hybridse-ci.yml
@@ -86,10 +86,10 @@ jobs:
           name: lib-artifacts
 
       - name: Prepare Maven Release Deploy
-        if: startsWith(github.ref, 'refs/tags/v')
+        if: startsWith(github.ref, 'refs/tags/hybridse-v')
         run: |
           VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          VERSION=$(echo $VERSION | sed -e 's/^v//')
+          VERSION=$(echo $VERSION | sed -e 's/^hybridse-v//')
           ./java/prepare_release.sh $VERSION
 
       - name: Publish Java Library
@@ -106,9 +106,9 @@ jobs:
       - name: Create Archive
         if: ${{ github.event_name == 'push' }}
         run: |
-          if [[ "${{ github.ref }}" == "refs/tags/v"* ]]; then
+          if [[ "${{ github.ref }}" == "refs/tags/hybridse-v"* ]]; then
             VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-            VERSION=$(echo $VERSION | sed -e 's/^v//')
+            VERSION=$(echo $VERSION | sed -e 's/^hybridse-v//')
             HYBRIDSE_VERSION=$VERSION ./tools/hybridse_deploy.sh
           else
             ./tools/hybridse_deploy.sh
@@ -188,10 +188,10 @@ jobs:
           name: lib-artifacts
 
       - name: Prepare Maven Release Deploy
-        if: startsWith(github.ref, 'refs/tags/v')
+        if: startsWith(github.ref, 'refs/tags/hybridse-v')
         run: |
           VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          VERSION=$(echo $VERSION | sed -e 's/^v//')
+          VERSION=$(echo $VERSION | sed -e 's/^hybridse-v//')
           ./java/prepare_release.sh $VERSION
 
       - name: Publish Java Library
@@ -209,9 +209,9 @@ jobs:
       - name: Create Archive
         if: ${{ github.event_name == 'push' }}
         run: |
-          if [[ "${{ github.ref }}" == "refs/tags/v"* ]]; then
+          if [[ "${{ github.ref }}" == "refs/tags/hybridse-v"* ]]; then
             VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-            VERSION=$(echo $VERSION | sed -e 's/^v//')
+            VERSION=$(echo $VERSION | sed -e 's/^hybridse-v//')
             HYBRIDSE_VERSION=$VERSION ./tools/hybridse_deploy.sh
           else
             ./tools/hybridse_deploy.sh
@@ -267,10 +267,10 @@ jobs:
           cp libhybridse_*.dylib java/hybridse-native/src/main/resources/
 
       - name: Prepare Maven Release Deploy
-        if: startsWith(github.ref, 'refs/tags/v')
+        if: startsWith(github.ref, 'refs/tags/hybridse-v')
         run: |
           VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          VERSION=$(echo $VERSION | sed -e 's/^v//')
+          VERSION=$(echo $VERSION | sed -e 's/^hybridse-v//')
           ./java/prepare_release.sh $VERSION
 
       - name: Publish Java Library
@@ -289,15 +289,15 @@ jobs:
     runs-on: ubuntu-latest
     needs: ["linux-build", "macos-build"]
     if: >
-      success() && startsWith(github.ref, 'refs/tags/v')
+      success() && startsWith(github.ref, 'refs/tags/hybridse-v')
     steps:
       - name: Download Release Artifacts
         uses: actions/download-artifact@v2
         with:
           name: release-artifacts
 
       - name: Release
-        if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ startsWith(github.ref, 'refs/tags/hybridse-v') }}
         uses: softprops/action-gh-release@v1
         with:
           files: |

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,47 @@
+# Changelog
+
+## [Unreleased]
+
+### Feature
+- Support parameterized query under BatchMode [#262](https://github.com/4paradigm/OpenMLDB/issues/262), [#168](https://github.com/4paradigm/OpenMLDB/issues/168)
+
+### SQL Syntax
+- `nvl` & `nvl2`: [#238](https://github.com/4paradigm/OpenMLDB/issues/238)
+- bitwise operators: `&`, `|`, `^`, `~` [#244](https://github.com/4paradigm/OpenMLDB/pull/244)
+
+## [0.2.2] - 2021-08-08
+### Feature
++ Add `VARCHAR` Type [#237](https://github.com/4paradigm/OpenMLDB/issues/237)
+
+### Bug Fix
+- Fix invalid back qoute identifier name [#263](https://github.com/4paradigm/OpenMLDB/issues/263)
+  can't write as multiple path style (e.g a.b) now
+- InsertPreparedStatement set month by mistake when use Date type [#200](https://github.com/4paradigm/OpenMLDB/pull/200)
+
+### Note:
+`OPTIONS` can't write as multiple path style (e.g a.b) now
+
+## [0.2.0] - 2021-07-22
+### Features
+
++ Refactor front-end using [zetasql](https://github.com/jingchen2222/zetasql). Thus OpenMLDB can support more SQL syntaxs and provide friendly syntax error message.
++ Better code style and comment
++ Add APIServer module. User can use Rest API access OpenMLDB.[#48](https://github.com/4paradigm/OpenMLDB/issues/48)
+
+### SQL Syntax
+
+Changed
+- `table options` syntax: [#103](https://github.com/4paradigm/HybridSE/issues/103)
+- `lead` method: [#136](https://github.com/4paradigm/HybridSE/pull/136)
+
+Removed
+- `||` and `&&` as logical operator: [#264](https://github.com/4paradigm/OpenMLDB/issues/264)
+- `at` function: [#265](https://github.com/4paradigm/OpenMLDB/issues/265)
+
+### Note
+- openmldb-0.2.0-linux.tar.gz targets on x86_64
+- aarch64 artifacts consider experimental
+
+[Unreleased]: https://github.com/4paradigm/OpenMLDB/compare/0.2.2...HEAD
+[0.2.2]: https://github.com/4paradigm/OpenMLDB/compare/v0.2.0...0.2.2
+[0.2.0]: https://github.com/4paradigm/OpenMLDB/compare/v0.1.5-pre...v0.2.0
diff --git a/cases/function/expression/test_arithmetic.yaml b/cases/function/expression/test_arithmetic.yaml
@@ -595,7 +595,6 @@ cases:
            NULL, NULL, NULL, NULL, NULL]
   - id: bitwise_operators
     desc: bitwise and/or/xor
-    mode: hybridse-only
     inputs:
       - columns: ["c1 int16","c2 int32","c3 bigint", "c6 timestamp"]
         indexs: ["index1:c3:c6"]
@@ -619,72 +618,67 @@ cases:
           - [ 0, 5, 15, 0, 10, 0 ]
   - id: bitwise_operators_fail
     desc: bitwise and/or/xor, fail on non-integral operands
-    mode: hybridse-only
     inputs:
       - columns: [ "c0 int", "c1 bool", "c2 float", "c3 double", "c4 string", "c5 date", "c6 timestamp" ]
         indexs: ["index1:c0:c6"]
         rows:
           - [1, true, 1.0, 2.0, "abc", "2012-8-11", 1590738989000]
     sql: |
-      select d[1] d[0] 10 as r1 from {0}
+      select d[1] d[0] 10 as r1 from {0};
     dataProvider:
       - ['&', '|', '^']
       - [ '{0}.c1', '{0}.c2', '{0}.c3', '{0}.c4', '{0}.c5', '{0}.c6' ]
     expect:
       success: false
   - id: bitwise_operators_not
     desc: bitwise not
-    mode: hybridse-only
     inputs:
       - columns: ["c1 int16","c2 int32","c3 bigint", "c6 timestamp"]
         indexs: ["index1:c3:c6"]
         rows:
           - [3, 6, 12, 1590738989000]
     sql: |
-      select ~c1 as r1, ~c2 as r2, ~c3 as r3 from {0}
+      select ~c1 as r1, ~c2 as r2, ~c3 as r3 from {0};
     expect:
       columns: [ 'r1 int16', 'r2 int32', 'r3 bigint']
       rows:
         - [ -4, -7, -13 ]
   - id: bitwise_not_fail
     desc: bitwise not, fail on non-integral operand
-    mode: hybridse-only
     inputs:
       - columns: [ "c0 int", "c1 bool", "c2 float", "c3 double", "c4 string", "c5 date", "c6 timestamp" ]
         indexs: ["index1:c0:c6"]
         rows:
           - [1, true, 1.0, 2.0, "abc", "2012-8-11", 1590738989000]
     sql: |
-      select d[0] d[1] as r1 from {0}
+      select d[0] d[1] as r1 from {0};
     dataProvider:
       - ['~']
       - [ '{0}.c1', '{0}.c2', '{0}.c3', '{0}.c4', '{0}.c5', '{0}.c6' ]
     expect:
       success: false
   - id: bitwise_null_operands
     desc: bitwise operation return null if any of operands is null
-    mode: hybridse-only
     inputs:
       - columns: ["c1 int16","c2 int32","c3 bigint", "c4 int16", "c6 timestamp"]
         indexs: ["index1:c3:c6"]
         rows:
           - [3, 6, 12, NULL, 1590738989000]
     sql: |
-      select {0}.c1 & {0}.c4 as r1, {0}.c2 | {0}.c4 as r2, {0}.c3 ^ {0}.c4 as r3, ~ {0}.c4 as r4 from {0}
+      select {0}.c1 & {0}.c4 as r1, {0}.c2 | {0}.c4 as r2, {0}.c3 ^ {0}.c4 as r3, ~ {0}.c4 as r4 from {0};
     expect:
       columns: [ 'r1 int16', 'r2 int32', 'r3 int64', 'r4 int16' ]
       rows:
         - [ NULL, NULL, NULL, NULL ]
   - id: bitwise_const_null_operands
     desc: bitwise operation return null if any of operands is null
-    mode: hybridse-only
     inputs:
       - columns: ["c1 int16","c2 int32","c3 bigint", "c4 int", "c6 timestamp"]
         indexs: ["index1:c3:c6"]
         rows:
           - [3, 6, 12, NULL, 1590738989000]
     sql: |
-      select {0}.c1 & NULL as r1, {0}.c2 | NULL as r2, {0}.c3 ^ NULL as r3, ~ NULL as r4 from {0}
+      select {0}.c1 & NULL as r1, {0}.c2 | NULL as r2, {0}.c3 ^ NULL as r3, ~ NULL as r4 from {0};
     expect:
       columns: [ 'r1 int16', 'r2 int32', 'r3 int64', 'r4 bool' ]
       rows:

diff --git a/cases/function/expression/test_condition.yaml b/cases/function/expression/test_condition.yaml
@@ -286,7 +286,6 @@ cases:
       success: false
   - id: 11-2
     desc: NVL is synonyms to ifnull
-    mode: hybridse-only
     inputs:
       - columns: ["col1 int","col2 string", "col4 timestamp"]
         indexs: ["index1:col1:col4"]
@@ -305,7 +304,6 @@ cases:
         - [3, ""]
   - id: 11-3
     desc: NVL-表达式-/0
-    mode: hybridse-only
     inputs:
       - columns: ["col1 int","col2 int", "col4 timestamp"]
         indexs: ["index1:col1:col4"]
@@ -348,7 +346,6 @@ cases:
 
   - id: NVL2-1
     desc: NVL2
-    mode: hybridse-only
     inputs:
       - columns: ["col1 int","col2 int", "col4 timestamp"]
         indexs: ["index1:col1:col4"]
@@ -368,7 +365,6 @@ cases:
 
   - id: NVL2-2
     desc: NVL2, type not match
-    mode: hybridse-only
     inputs:
       - columns: ["col1 int","col2 int", "col4 timestamp"]
         indexs: ["index1:col1:col4"]
@@ -383,7 +379,6 @@ cases:
 
   - id: NVL2-3
     desc: NVL2, sub expression
-    mode: hybridse-only
     inputs:
       - columns: ["col1 int","col2 int", "col4 timestamp"]
         indexs: ["index1:col1:col4"]

diff --git a/cases/query/parameterized_query.yaml b/cases/query/parameterized_query.yaml
@@ -42,7 +42,7 @@ cases:
       data: |
         0, 1, 5, 1.1, 11.1, 1, 1
         0, 2, 5, 2.2, 22.2, 2, 22
-  - id: 1
+  - id: 1-1
     desc: 带参数的Where条件部分命中索引
     mode: request-unsupport, offline-unsupport
     db: db1
@@ -71,6 +71,37 @@ cases:
       data: |
         1, 4, 55, 4.4, 44.4, 2, 4444
         2, 5, 55, 5.5, 55.5, 3, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+  - id: 1-2
+    desc: 带参数的Where条件部分命中索引,参数包含字符串
+    mode: request-unsupport, offline-unsupport
+    db: db1
+    sql: |
+      SELECT col0, col1, col2, col3, col4, col5, col6 FROM {0} where col6=? and col1 < ?;
+    parameters:
+      columns: [ "p1 string" , "p2 bigint" ]
+      rows:
+        - [ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, 7 ]
+    inputs:
+      - schema: col0:string, col1:int64, col2:int16, col3:float, col4:double, col5:int64, col6:string
+        index: index1:col6:col5
+        data: |
+          0, 1, 5, 1.1, 11.1, 1, 1
+          0, 2, 5, 2.2, 22.2, 2, 22
+          1, 3, 55, 3.3, 33.3, 1, 333
+          1, 4, 55, 4.4, 44.4, 2, 4444
+          2, 5, 55, 5.5, 55.5, 3, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+          2, 6, 55, 6.6, 66.6, 4, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+          2, 7, 55, 7.7, 77.7, 5, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+    batch_plan: |
+      SIMPLE_PROJECT(sources=(col0, col1, col2, col3, col4, col5, col6))
+        FILTER_BY(condition=col1 < ?2, left_keys=(), right_keys=(), index_keys=(?1))
+          DATA_PROVIDER(type=Partition, table=auto_t0, index=index1)
+    expect:
+      schema: col0:string, col1:int64, col2:int16, col3:float, col4:double, col5:int64, col6:string
+      order: col1
+      data: |
+        2, 5, 55, 5.5, 55.5, 3, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+        2, 6, 55, 6.6, 66.6, 4, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
   - id: 2-1
     desc: 带参数的Where条件未命中索引
     mode: request-unsupport, offline-unsupport