Prepare for release of 2.0.0 (#186)

* #150: Added S3 intermediate storage layer (#152) Co-authored-by: Christoph Pirkl <[email protected]> * #149: Added s3 import query generator and runner (#156) Fixes #149 Co-authored-by: Christoph Kuhnke <[email protected]> Co-authored-by: KK <[email protected]> * #160: Add support for writing to Exasol database using S3 as intermediate storage (#161) Fixes #160 * Address review suggestion from @tkilias * #159: Added cleanup process to remove intermediate data after job finish (#165) Fixes #159 --------- Co-authored-by: Christoph Kuhnke <[email protected]> Co-authored-by: KK <[email protected]> * #158: Refactored options class (#167) Fixes #158 --------- Co-authored-by: Torsten Kilias <[email protected]> * #168: Refactored to add module setup (#170) Fixes #168 * #171: Refactored artifact packaging and releasing process for module setup (#173) --------- Co-authored-by: Sebastian Bär <[email protected]> * #174: Refactored GitHub workflow actions (#175) Fixes #175 * #176: Fixed sha256sum files upload bug (#179) * #155: Refactored to unify user parameters (#178) Fixes #155 --------- Co-authored-by: Christoph Pirkl <[email protected]> * #164: Added validation check before writing to intermediate location (#180) Fixes #164 --------- Co-authored-by: Christoph Pirkl <[email protected]> * #162: Updated developer guide with staged commit behaviour (#182) Fixes #162 --------- Co-authored-by: Christoph Pirkl <[email protected]> * #183: Updated user guide to include latest changes (#184) Fixes #183 --------- Co-authored-by: Christoph Pirkl <[email protected]> * Updated version to `2.0.0` (#185) * Changed version * Improved default options * Updated changes and user guide docs --------- Co-authored-by: Christoph Pirkl <[email protected]> Co-authored-by: Christoph Kuhnke <[email protected]> Co-authored-by: KK <[email protected]> Co-authored-by: Torsten Kilias <[email protected]> Co-authored-by: Sebastian Bär <[email protected]>
exasol · Jul 14, 2023 · e522e56 · e522e56
1 parent af9c3f6
commit e522e56
Show file tree

Hide file tree

Showing 113 changed files with 7,480 additions and 1,453 deletions.
diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
@@ -12,30 +12,13 @@ concurrency:
 
 jobs:
   build:
-    name: Building with ${{ matrix.mvn.name }} and Exasol ${{ matrix.exasol-docker-version }}
+    name: Building with ${{ matrix.profile }} and Exasol ${{ matrix.exasol-docker-version }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        exasol-docker-version: [ 7.1.18 ]
-        mvn:
-          [
-            {
-              name: 'Spark3.4',
-              profile: '-Pspark3.4',
-              project-keeper-skip: false,
-            },
-            {
-              name: 'Spark3.3',
-              profile: '-Pspark3.3',
-              project-keeper-skip: false,
-            },
-            {
-              name: 'Spark3.3 Scala2.12',
-              profile: '-Pspark3.3-scala2.12',
-              project-keeper-skip: false,
-            },
-          ]
+        exasol-docker-version: [ 8.18.1 ]
+        profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3
@@ -57,17 +40,11 @@ jobs:
         run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties"
       - name: Pull docker Images
         run: docker pull exasol/docker-db:${{ matrix.exasol-docker-version }}
-      - name: Run scalafix linting
-        run: |
-          mvn --batch-mode clean compile test scalafix:scalafix ${{ matrix.mvn.profile }} \
-              -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-              -DtrimStackTrace=false
       - name: Run tests and build with Maven
         run: |
-          mvn --batch-mode verify ${{ matrix.mvn.profile }} \
+          mvn --batch-mode verify ${{ matrix.profile }} \
               -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-              -DtrimStackTrace=false \
-              -Dproject-keeper.skip=${{ matrix.mvn.project-keeper-skip }}
+              -DtrimStackTrace=false
         env:
           EXASOL_DOCKER_VERSION: ${{ matrix.exasol-docker-version }}
       - name: Publish Test Report
@@ -76,7 +53,7 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
       - name: Sonar analysis
-        if: ${{ env.SONAR_TOKEN != null && !matrix.mvn.project-keeper-skip }}
+        if: ${{ env.SONAR_TOKEN != null && matrix.profile == '-Pspark3.4' }}
         run: |
           mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \
               -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
@@ -86,4 +63,4 @@ jobs:
               -Dsonar.login=$SONAR_TOKEN
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
diff --git a/.github/workflows/pk-verify.yml b/.github/workflows/pk-verify.yml
@@ -0,0 +1,35 @@
+name: PK Verify
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Project Keeper Verify
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: 11
+          cache: 'maven'
+      - name: Cache SonarCloud packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.sonar/cache
+          key: ${{ runner.os }}-sonar
+          restore-keys: ${{ runner.os }}-sonar
+      - name: Run Project Keeper Separately
+        run: mvn --batch-mode -DtrimStackTrace=false com.exasol:project-keeper-maven-plugin:2.9.9:verify --projects .
diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml
@@ -6,10 +6,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3
@@ -23,21 +19,13 @@ jobs:
           cache: 'maven'
       - name: Enable testcontainer reuse
         run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties"
-      - name: Install xmlstarlet
-        run: sudo apt install -y --no-install-recommends xmlstarlet
-      - name: Create release pom file
-        run: ./tools/createReleasePom.sh ${{ matrix.profile }}
       - name: Run tests and build with Maven
-        run: |
-          mvn --batch-mode --file release.xml clean verify ${{ matrix.profile }} \
-              -Dproject-keeper.skip=true
-      - name: Remove release pom file
-        run: rm -rf release.xml
+        run: mvn --batch-mode clean verify
       - name: Prepare checksum
-        run: find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + > original_checksum
+        run: find */target -maxdepth 1 -name *.jar -exec sha256sum "{}" + > original_checksum
       - name: Upload checksum to the artifactory
         uses: actions/upload-artifact@v3
         with:
           name: original_checksum
           retention-days: 5
-          path: original_checksum
+          path: original_checksum
diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml
@@ -6,10 +6,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3
@@ -21,16 +17,7 @@ jobs:
           distribution: 'temurin'
           java-version: 11
           cache: 'maven'
-      - name: Install xmlstarlet
-        run: sudo apt install -y --no-install-recommends xmlstarlet
-      - name: Create release pom file
-        run: ./tools/createReleasePom.sh ${{ matrix.profile }}
       - name: Build with Maven skipping tests
-        run: |
-          mvn --batch-mode --file release.xml clean verify ${{ matrix.profile }} \
-              -DskipTests \
-              -Dproject-keeper.skip=true
-      - name: Remove release pom file
-        run: rm -rf release.xml
+        run: mvn --batch-mode clean verify -DskipTests
       - name: Print checksum
-        run: echo 'checksum_start==';find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + | xargs;echo '==checksum_end'
+        run: echo 'checksum_start==';find */target -maxdepth 1 -name *.jar -exec sha256sum "{}" + | xargs;echo '==checksum_end'
diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3
@@ -26,19 +26,9 @@ jobs:
           server-password: MAVEN_PASSWORD
           gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }}
           gpg-passphrase: MAVEN_GPG_PASSPHRASE
-      - name: Install xmlstarlet
-        run: sudo apt install -y --no-install-recommends xmlstarlet
-      - name: Create release pom file
-        run: ./tools/createReleasePom.sh ${{ matrix.profile }}
       - name: Publish to Central Repository
-        run: |
-          mvn --file release.xml clean deploy ${{ matrix.profile }} \
-              -Dgpg.skip=false \
-              -DskipTests \
-              -Dproject-keeper.skip=true
+        run: mvn --batch-mode clean deploy ${{ matrix.profile }} -Dgpg.skip=false -DskipTests
         env:
           MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
           MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }}
           MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }}
-      - name: Remove release pom file
-        run: rm -rf release.xml
diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
+        profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ]
     steps:
       - name: Checkout the repository
         uses: actions/checkout@v3
@@ -25,34 +25,25 @@ jobs:
           distribution: 'temurin'
           java-version: 11
           cache: 'maven'
-      - name: Install xmlstarlet
-        run: sudo apt install -y --no-install-recommends xmlstarlet
-      - name: Create release pom file
-        run: ./tools/createReleasePom.sh ${{ matrix.profile }}
       - name: Build with Maven skipping tests
-        run: |
-          mvn --batch-mode --file release.xml clean verify ${{ matrix.profile }} \
-              -DskipTests \
-              -Dproject-keeper.skip=true
-      - name: Remove release pom file
-        run: rm -rf release.xml
+        run: mvn --batch-mode clean verify ${{ matrix.profile }} -DskipTests
       - name: Generate sha256sum files
-        run: |
-          cd target
-          find . -maxdepth 1 -name \*.jar -exec bash -c 'sha256sum {} > {}.sha256' \;
+        run: find */target -maxdepth 1 -name \*.jar -exec bash -c 'sha256sum {} > {}.sha256' \;
       - name: Upload assets to the GitHub release draft
         uses: shogo82148/actions-upload-release-asset@v1
         with:
           upload_url: ${{ github.event.inputs.upload_url }}
-          asset_path: target/*.jar
+          asset_path: '**/target/*.jar'
       - name: Upload sha256sum files
         uses: shogo82148/actions-upload-release-asset@v1
         with:
           upload_url: ${{ github.event.inputs.upload_url }}
-          asset_path: target/*.sha256
+          asset_path: '**/target/*.sha256'
+      - name: Create a zip file from error code report JSON files
+        run: zip -v error_code_report.zip */target/error_code_report.json
       - name: Upload error-code-report
+        if: ${{ matrix.profile == '-Pspark3.4' }}
         uses: shogo82148/actions-upload-release-asset@v1
-        if: ${{ matrix.profile == '-Pspark3.3' }}
         with:
           upload_url: ${{ github.event.inputs.upload_url }}
-          asset_path: target/error_code_report.json
+          asset_path: error_code_report.zip
diff --git a/.gitignore b/.gitignore
@@ -58,6 +58,9 @@ tmp
 .project
 .scala_dependencies
 *.sc
+**/.settings/org.eclipse.core.resources.prefs
+**/.settings/org.eclipse.jdt.apt.core.prefs
+**/.settings/org.eclipse.m2e.core.prefs
 
 # Ensime
 .ensime

diff --git a/.project-keeper.yml b/.project-keeper.yml
@@ -1,17 +1,38 @@
 sources:
   - type: maven
-    path: pom.xml
+    path: parent-pom/pom.xml
+  - type: maven
+    path: exasol-jdbc/pom.xml
+    modules:
+      - integration_tests
+    parentPom:
+      groupId: "com.exasol"
+      artifactId: "spark-connector-parent-pom"
+      version: "${revision}"
+      relativePath: "../parent-pom/pom.xml"
+  - type: maven
+    path: exasol-s3/pom.xml
     modules:
-      - maven_central
       - integration_tests
+    parentPom:
+      groupId: "com.exasol"
+      artifactId: "spark-connector-parent-pom"
+      version: "${revision}"
+      relativePath: "../parent-pom/pom.xml"
+version:
+  fromSource: parent-pom/pom.xml
 excludes:
   - regex: "(?s)E-PK-CORE-62: The project's README.md does not contain a valid badges block. Please add or replace the following badges:.*"
   - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build-next-java.yml'"
   - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build.yml'"
-  - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_prepare_original_checksum.yml'"
-  - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_print_quick_checksum.yml'"
   - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_upload_github_release_assets.yml'"
   - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_release_on_maven_central.yml'"
+  - "E-PK-CORE-18: Outdated content: '.settings/org.eclipse.jdt.core.prefs'"
+  - regex: "(?s)E-PK-CORE-18: Outdated content: 'exasol-(jdbc|s3)/pk_generated_parent.pom'"
+  - regex: "(?s)E-PK-CORE-104: Invalid pom file 'exasol-(jdbc|s3)/pom.xml':.*"
+  - regex: "(?s)E-PK-CORE-118: Invalid pom file 'exasol-(jdbc|s3)/pom.xml':.*"
+  - "E-PK-CORE-17: Missing required file: '.github/workflows/project-keeper-verify.yml'"
+  - "E-PK-CORE-17: Missing required file: '.github/workflows/project-keeper.sh'"
 linkReplacements:
   - https://netty.io/netty-all/|https://netty.io/index.html
   - http://nexus.sonatype.org/oss-repository-hosting.html/scalatest-maven-plugin|https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin

diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
@@ -10,9 +10,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary=
 org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
 org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=11
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=11
+org.eclipse.jdt.core.compiler.compliance=1.8
 org.eclipse.jdt.core.compiler.debug.lineNumber=generate
 org.eclipse.jdt.core.compiler.debug.localVariable=generate
 org.eclipse.jdt.core.compiler.debug.sourceFile=generate
@@ -113,7 +113,7 @@ org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
 org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
 org.eclipse.jdt.core.compiler.processAnnotations=enabled
 org.eclipse.jdt.core.compiler.release=disabled
-org.eclipse.jdt.core.compiler.source=11
+org.eclipse.jdt.core.compiler.source=1.8
 org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false
 org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647
 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # Spark Exasol Connector
 
 [![Build Status](https://github.com/exasol/spark-connector/actions/workflows/ci-build.yml/badge.svg)](https://github.com/exasol/spark-connector/actions/workflows/ci-build.yml)
-[![Maven Central – The Spark Exasol Connector](https://img.shields.io/maven-central/v/com.exasol/spark-connector)](https://central.sonatype.com/artifact/com.exasol/spark-connector_2.13/1.3.0-spark-3.3.2/versions)
+[![Maven Central – The Spark Exasol Connector](https://img.shields.io/maven-central/v/com.exasol/spark-connector)](https://mvnrepository.com/artifact/com.exasol/spark-connector)
 
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=com.exasol%3Aspark-connector&metric=alert_status)](https://sonarcloud.io/dashboard?id=com.exasol%3Aspark-connector)