diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml index c4ff3be8..f2079ec3 100644 --- a/.github/workflows/broken_links_checker.yml +++ b/.github/workflows/broken_links_checker.yml @@ -22,6 +22,8 @@ jobs: echo '{"aliveStatusCodes": [429, 200], "ignorePatterns": [' \ '{"pattern": "^https?://(www|dev).mysql.com/"},' \ '{"pattern": "^https?://(www.)?opensource.org"}' \ + '{"pattern": "^https?://(www.)?eclipse.org"}' \ + '{"pattern": "^https?://projects.eclipse.org"}' \ ']}' > ./target/broken_links_checker.json - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 2bb943ec..8ecf1525 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -12,30 +12,13 @@ concurrency: jobs: build: - name: Building with ${{ matrix.mvn.name }} and Exasol ${{ matrix.exasol-docker-version }} + name: Building with ${{ matrix.profile }} and Exasol ${{ matrix.exasol-docker-version }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - exasol-docker-version: [ 7.1.18 ] - mvn: - [ - { - name: 'Spark3.4', - profile: '-Pspark3.4', - project-keeper-skip: false, - }, - { - name: 'Spark3.3', - profile: '-Pspark3.3', - project-keeper-skip: false, - }, - { - name: 'Spark3.3 Scala2.12', - profile: '-Pspark3.3-scala2.12', - project-keeper-skip: false, - }, - ] + exasol-docker-version: [ 8.18.1 ] + profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -57,17 +40,11 @@ jobs: run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" - name: Pull docker Images run: docker pull exasol/docker-db:${{ matrix.exasol-docker-version }} - - name: Run scalafix linting - run: | - mvn --batch-mode clean compile test scalafix:scalafix ${{ matrix.mvn.profile }} \ - -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -DtrimStackTrace=false - name: Run tests and build with Maven run: | - mvn --batch-mode verify ${{ matrix.mvn.profile }} \ + mvn --batch-mode verify ${{ matrix.profile }} \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -DtrimStackTrace=false \ - -Dproject-keeper.skip=${{ matrix.mvn.project-keeper-skip }} + -DtrimStackTrace=false env: EXASOL_DOCKER_VERSION: ${{ matrix.exasol-docker-version }} - name: Publish Test Report @@ -76,7 +53,7 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Sonar analysis - if: ${{ env.SONAR_TOKEN != null && !matrix.mvn.project-keeper-skip }} + if: ${{ env.SONAR_TOKEN != null && matrix.profile == '-Pspark3.4' }} run: | mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ @@ -86,4 +63,4 @@ jobs: -Dsonar.login=$SONAR_TOKEN env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/pk-verify.yml b/.github/workflows/pk-verify.yml new file mode 100644 index 00000000..2ee0d3b6 --- /dev/null +++ b/.github/workflows/pk-verify.yml @@ -0,0 +1,35 @@ +name: PK Verify + +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Project Keeper Verify + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 11 + cache: 'maven' + - name: Cache SonarCloud packages + uses: actions/cache@v3 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Run Project Keeper Separately + run: mvn --batch-mode -DtrimStackTrace=false com.exasol:project-keeper-maven-plugin:2.9.9:verify --projects . diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index 5a2d9493..091fd1df 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -6,10 +6,6 @@ on: jobs: build: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -23,21 +19,13 @@ jobs: cache: 'maven' - name: Enable testcontainer reuse run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" - - name: Install xmlstarlet - run: sudo apt install -y --no-install-recommends xmlstarlet - - name: Create release pom file - run: ./tools/createReleasePom.sh ${{ matrix.profile }} - name: Run tests and build with Maven - run: | - mvn --batch-mode --file release.xml clean verify ${{ matrix.profile }} \ - -Dproject-keeper.skip=true - - name: Remove release pom file - run: rm -rf release.xml + run: mvn --batch-mode clean verify - name: Prepare checksum - run: find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + > original_checksum + run: find */target -maxdepth 1 -name *.jar -exec sha256sum "{}" + > original_checksum - name: Upload checksum to the artifactory uses: actions/upload-artifact@v3 with: name: original_checksum retention-days: 5 - path: original_checksum \ No newline at end of file + path: original_checksum diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml index 212a4c71..1062addd 100644 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -6,10 +6,6 @@ on: jobs: build: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -21,16 +17,7 @@ jobs: distribution: 'temurin' java-version: 11 cache: 'maven' - - name: Install xmlstarlet - run: sudo apt install -y --no-install-recommends xmlstarlet - - name: Create release pom file - run: ./tools/createReleasePom.sh ${{ matrix.profile }} - name: Build with Maven skipping tests - run: | - mvn --batch-mode --file release.xml clean verify ${{ matrix.profile }} \ - -DskipTests \ - -Dproject-keeper.skip=true - - name: Remove release pom file - run: rm -rf release.xml + run: mvn --batch-mode clean verify -DskipTests - name: Print checksum - run: echo 'checksum_start==';find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + | xargs;echo '==checksum_end' + run: echo 'checksum_start==';find */target -maxdepth 1 -name *.jar -exec sha256sum "{}" + | xargs;echo '==checksum_end' diff --git a/.github/workflows/release_droid_release_on_maven_central.yml b/.github/workflows/release_droid_release_on_maven_central.yml index b9df24db..dd8a1008 100644 --- a/.github/workflows/release_droid_release_on_maven_central.yml +++ b/.github/workflows/release_droid_release_on_maven_central.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] + profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -26,19 +26,9 @@ jobs: server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Install xmlstarlet - run: sudo apt install -y --no-install-recommends xmlstarlet - - name: Create release pom file - run: ./tools/createReleasePom.sh ${{ matrix.profile }} - name: Publish to Central Repository - run: | - mvn --file release.xml clean deploy ${{ matrix.profile }} \ - -Dgpg.skip=false \ - -DskipTests \ - -Dproject-keeper.skip=true + run: mvn --batch-mode clean deploy ${{ matrix.profile }} -Dgpg.skip=false -DskipTests env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_PASSWORD }} MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} - - name: Remove release pom file - run: rm -rf release.xml diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 37f08677..f78bb28e 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [ '-Pspark3.4', '-Pspark3.3', '-Pspark3.3-scala2.12' ] + profile: [ '-Pspark3.4', '-Pspark3.4-scala2.12', '-Pspark3.3', '-Pspark3.3-scala2.12' ] steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -25,34 +25,25 @@ jobs: distribution: 'temurin' java-version: 11 cache: 'maven' - - name: Install xmlstarlet - run: sudo apt install -y --no-install-recommends xmlstarlet - - name: Create release pom file - run: ./tools/createReleasePom.sh ${{ matrix.profile }} - name: Build with Maven skipping tests - run: | - mvn --batch-mode --file release.xml clean verify ${{ matrix.profile }} \ - -DskipTests \ - -Dproject-keeper.skip=true - - name: Remove release pom file - run: rm -rf release.xml + run: mvn --batch-mode clean verify ${{ matrix.profile }} -DskipTests - name: Generate sha256sum files - run: | - cd target - find . -maxdepth 1 -name \*.jar -exec bash -c 'sha256sum {} > {}.sha256' \; + run: find */target -maxdepth 1 -name \*.jar -exec bash -c 'sha256sum {} > {}.sha256' \; - name: Upload assets to the GitHub release draft uses: shogo82148/actions-upload-release-asset@v1 with: upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/*.jar + asset_path: '**/target/*.jar' - name: Upload sha256sum files uses: shogo82148/actions-upload-release-asset@v1 with: upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/*.sha256 + asset_path: '**/target/*.sha256' + - name: Create a zip file from error code report JSON files + run: zip -v error_code_report.zip */target/error_code_report.json - name: Upload error-code-report + if: ${{ matrix.profile == '-Pspark3.4' }} uses: shogo82148/actions-upload-release-asset@v1 - if: ${{ matrix.profile == '-Pspark3.3' }} with: upload_url: ${{ github.event.inputs.upload_url }} - asset_path: target/error_code_report.json + asset_path: error_code_report.zip diff --git a/.gitignore b/.gitignore index 46b8a441..051f2f4e 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,9 @@ tmp .project .scala_dependencies *.sc +**/.settings/org.eclipse.core.resources.prefs +**/.settings/org.eclipse.jdt.apt.core.prefs +**/.settings/org.eclipse.m2e.core.prefs # Ensime .ensime diff --git a/.project-keeper.yml b/.project-keeper.yml index d1b2d722..9b814d5b 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -1,17 +1,38 @@ sources: - type: maven - path: pom.xml + path: parent-pom/pom.xml + - type: maven + path: exasol-jdbc/pom.xml + modules: + - integration_tests + parentPom: + groupId: "com.exasol" + artifactId: "spark-connector-parent-pom" + version: "${revision}" + relativePath: "../parent-pom/pom.xml" + - type: maven + path: exasol-s3/pom.xml modules: - - maven_central - integration_tests + parentPom: + groupId: "com.exasol" + artifactId: "spark-connector-parent-pom" + version: "${revision}" + relativePath: "../parent-pom/pom.xml" +version: + fromSource: parent-pom/pom.xml excludes: - regex: "(?s)E-PK-CORE-62: The project's README.md does not contain a valid badges block. Please add or replace the following badges:.*" - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build-next-java.yml'" - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build.yml'" - - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_prepare_original_checksum.yml'" - - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_print_quick_checksum.yml'" - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_upload_github_release_assets.yml'" - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_release_on_maven_central.yml'" + - "E-PK-CORE-18: Outdated content: '.settings/org.eclipse.jdt.core.prefs'" + - regex: "(?s)E-PK-CORE-18: Outdated content: 'exasol-(jdbc|s3)/pk_generated_parent.pom'" + - regex: "(?s)E-PK-CORE-104: Invalid pom file 'exasol-(jdbc|s3)/pom.xml':.*" + - regex: "(?s)E-PK-CORE-118: Invalid pom file 'exasol-(jdbc|s3)/pom.xml':.*" + - "E-PK-CORE-17: Missing required file: '.github/workflows/project-keeper-verify.yml'" + - "E-PK-CORE-17: Missing required file: '.github/workflows/project-keeper.sh'" linkReplacements: - https://netty.io/netty-all/|https://netty.io/index.html - http://nexus.sonatype.org/oss-repository-hosting.html/scalatest-maven-plugin|https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 8b5a9aaa..f362cccd 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -10,9 +10,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -113,7 +113,7 @@ org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false diff --git a/README.md b/README.md index 75698b5d..4cd461f5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Spark Exasol Connector [![Build Status](https://github.com/exasol/spark-connector/actions/workflows/ci-build.yml/badge.svg)](https://github.com/exasol/spark-connector/actions/workflows/ci-build.yml) -[![Maven Central – The Spark Exasol Connector](https://img.shields.io/maven-central/v/com.exasol/spark-connector)](https://central.sonatype.com/artifact/com.exasol/spark-connector_2.13/1.3.0-spark-3.3.2/versions) +[![Maven Central – The Spark Exasol Connector](https://img.shields.io/maven-central/v/com.exasol/spark-connector)](https://mvnrepository.com/artifact/com.exasol/spark-connector) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=com.exasol%3Aspark-connector&metric=alert_status)](https://sonarcloud.io/dashboard?id=com.exasol%3Aspark-connector) diff --git a/dependencies.md b/dependencies.md index 01171e75..82552583 100644 --- a/dependencies.md +++ b/dependencies.md @@ -1,160 +1,234 @@ # Dependencies -## Compile Dependencies - -| Dependency | License | -| ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Scala Library][0] | [Apache-2.0][1] | -| [EXASolution JDBC Driver][2] | [EXAClient License][3] | -| [Exasol SQL Statement Builder][4] | [MIT License][5] | -| [error-reporting-java8][6] | [MIT License][7] | -| [Spark Project Core][8] | [Apache 2.0 License][9] | -| [Spark Project SQL][8] | [Apache 2.0 License][9] | -| [Guava: Google Core Libraries for Java][10] | [Apache License, Version 2.0][11] | -| [Netty/All-in-One][12] | [Apache License, Version 2.0][1] | -| [jackson-databind][13] | [The Apache Software License, Version 2.0][14] | -| [jersey-core-common][15] | [EPL 2.0][16]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][17]; [Apache License, 2.0][9]; [Public Domain][18] | -| [jersey-media-jaxb][19] | [EPL 2.0][16]; [GPL2 w/ CPE][17]; [EDL 1.0][20]; [BSD 2-Clause][21]; [Apache License, 2.0][9]; [Public Domain][18]; [Modified BSD][22]; [jQuery license][23]; [MIT license][24]; [W3C license][25] | -| [jersey-core-server][26] | [EPL 2.0][16]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][17]; [Apache License, 2.0][9]; [Modified BSD][22] | -| [jersey-core-client][27] | [EPL 2.0][16]; [GPL2 w/ CPE][17]; [EDL 1.0][20]; [BSD 2-Clause][21]; [Apache License, 2.0][9]; [Public Domain][18]; [Modified BSD][22]; [jQuery license][23]; [MIT license][24]; [W3C license][25] | -| [Apache Avro Mapred API][28] | [Apache License, Version 2.0][14] | -| Apache Hadoop Client Aggregator | [Apache License, Version 2.0][14] | -| [Protocol Buffers [Core]][29] | [BSD-3-Clause][30] | -| [Apache Commons Text][31] | [Apache License, Version 2.0][14] | -| [Woodstox][32] | [The Apache License, Version 2.0][11] | - -## Test Dependencies +## Spark Exasol Connector Parent pom + +### Plugin Dependencies + +| Dependency | License | +| ------------------------------------------------------ | --------------------------------------------- | +| [SonarQube Scanner for Maven][0] | [GNU LGPL 3][1] | +| [Apache Maven Compiler Plugin][2] | [Apache-2.0][3] | +| [Apache Maven Enforcer Plugin][4] | [Apache-2.0][3] | +| [Maven Flatten Plugin][5] | [Apache Software Licenese][3] | +| [Apache Maven Deploy Plugin][6] | [Apache-2.0][3] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][7] | [ASL2][8] | +| [Maven Surefire Plugin][9] | [Apache-2.0][3] | +| [Versions Maven Plugin][10] | [Apache License, Version 2.0][3] | +| [duplicate-finder-maven-plugin Maven Mojo][11] | [Apache License 2.0][12] | +| [JaCoCo :: Maven Plugin][13] | [Eclipse Public License 2.0][14] | +| [error-code-crawler-maven-plugin][15] | [MIT License][16] | +| [Reproducible Build Maven Plugin][17] | [Apache 2.0][8] | +| [OpenFastTrace Maven Plugin][18] | [GNU General Public License v3.0][19] | +| [Maven Clean Plugin][20] | [The Apache Software License, Version 2.0][8] | +| [Maven Install Plugin][21] | [The Apache Software License, Version 2.0][8] | +| [Maven Site Plugin 3][22] | [The Apache Software License, Version 2.0][8] | + +## Spark Exasol Connector With Jdbc + +### Compile Dependencies + +| Dependency | License | +| ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [EXASolution JDBC Driver][23] | [EXAClient License][24] | +| [Exasol SQL Statement Builder][25] | [MIT License][26] | +| [error-reporting-java8][27] | [MIT License][28] | +| [spark-connector-common-java][29] | [MIT License][30] | +| [Spark Project Core][31] | [Apache 2.0 License][12] | +| [Spark Project SQL][31] | [Apache 2.0 License][12] | +| [Guava: Google Core Libraries for Java][32] | [Apache License, Version 2.0][8] | +| [Netty/All-in-One][33] | [Apache License, Version 2.0][34] | +| [jackson-databind][35] | [The Apache Software License, Version 2.0][3] | +| Apache Hadoop Client Aggregator | [Apache License, Version 2.0][3] | +| [jersey-core-common][36] | [EPL 2.0][37]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][38]; [Apache License, 2.0][12]; [Public Domain][39] | +| [jersey-media-jaxb][40] | [EPL 2.0][37]; [GPL2 w/ CPE][38]; [EDL 1.0][41]; [BSD 2-Clause][42]; [Apache License, 2.0][12]; [Public Domain][39]; [Modified BSD][43]; [jQuery license][44]; [MIT license][45]; [W3C license][46] | +| [jersey-core-server][47] | [EPL 2.0][37]; [The GNU General Public License (GPL), Version 2, With Classpath Exception][38]; [Apache License, 2.0][12]; [Modified BSD][43] | +| [jersey-core-client][48] | [EPL 2.0][37]; [GPL2 w/ CPE][38]; [EDL 1.0][41]; [BSD 2-Clause][42]; [Apache License, 2.0][12]; [Public Domain][39]; [Modified BSD][43]; [jQuery license][44]; [MIT license][45]; [W3C license][46] | +| [Apache Avro Mapred API][49] | [Apache-2.0][3] | + +### Test Dependencies | Dependency | License | | ------------------------------------------ | ----------------------------------------- | -| [scalatest][33] | [the Apache License, ASL Version 2.0][34] | -| [scalatestplus-mockito][35] | [Apache-2.0][34] | -| [mockito-core][36] | [The MIT License][37] | -| [Apache Log4j API][38] | [Apache License, Version 2.0][14] | -| [Apache Log4j 1.x Compatibility API][39] | [Apache License, Version 2.0][14] | -| [Apache Log4j Core][40] | [Apache License, Version 2.0][14] | -| [Test Database Builder for Java][41] | [MIT License][42] | -| [Matcher for SQL Result Sets][43] | [MIT License][44] | -| [Test containers for Exasol on Docker][45] | [MIT License][46] | - -## Plugin Dependencies - -| Dependency | License | -| ------------------------------------------------------- | ---------------------------------------------- | -| [SonarQube Scanner for Maven][47] | [GNU LGPL 3][48] | -| [scala-maven-plugin][49] | [Public domain (Unlicense)][50] | -| [Apache Maven Compiler Plugin][51] | [Apache-2.0][14] | -| [Apache Maven Enforcer Plugin][52] | [Apache-2.0][14] | -| [Maven Flatten Plugin][53] | [Apache Software Licenese][14] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][54] | [ASL2][11] | -| [Maven Surefire Plugin][55] | [Apache-2.0][14] | -| [Versions Maven Plugin][56] | [Apache License, Version 2.0][14] | -| [duplicate-finder-maven-plugin Maven Mojo][57] | [Apache License 2.0][9] | -| [Apache Maven Deploy Plugin][58] | [Apache-2.0][14] | -| [Apache Maven GPG Plugin][59] | [Apache License, Version 2.0][14] | -| [Apache Maven Source Plugin][60] | [Apache License, Version 2.0][14] | -| [Apache Maven Javadoc Plugin][61] | [Apache-2.0][14] | -| [Nexus Staging Maven Plugin][62] | [Eclipse Public License][63] | -| [ScalaTest Maven Plugin][64] | [the Apache License, ASL Version 2.0][34] | -| [Apache Maven JAR Plugin][65] | [Apache License, Version 2.0][14] | -| [Apache Maven Shade Plugin][66] | [Apache License, Version 2.0][14] | -| [Maven Failsafe Plugin][67] | [Apache-2.0][14] | -| [JaCoCo :: Maven Plugin][68] | [Eclipse Public License 2.0][69] | -| [error-code-crawler-maven-plugin][70] | [MIT License][71] | -| [Reproducible Build Maven Plugin][72] | [Apache 2.0][11] | -| [Project keeper maven plugin][73] | [The MIT License][74] | -| [Artifact reference checker and unifier][75] | [MIT License][76] | -| [OpenFastTrace Maven Plugin][77] | [GNU General Public License v3.0][78] | -| [spotless-maven-plugin][79] | [The Apache Software License, Version 2.0][14] | -| [scalafix-maven-plugin][80] | [BSD-3-Clause][30] | -| [Maven Clean Plugin][81] | [The Apache Software License, Version 2.0][11] | -| [Maven Resources Plugin][82] | [The Apache Software License, Version 2.0][11] | -| [Maven Install Plugin][83] | [The Apache Software License, Version 2.0][11] | -| [Maven Site Plugin 3][84] | [The Apache Software License, Version 2.0][11] | - -[0]: https://www.scala-lang.org/ -[1]: https://www.apache.org/licenses/LICENSE-2.0 -[2]: http://www.exasol.com -[3]: https://repo1.maven.org/maven2/com/exasol/exasol-jdbc/7.1.19/exasol-jdbc-7.1.19-license.txt -[4]: https://github.com/exasol/sql-statement-builder/ -[5]: https://github.com/exasol/sql-statement-builder/blob/main/LICENSE -[6]: https://github.com/exasol/error-reporting-java/ -[7]: https://github.com/exasol/error-reporting-java/blob/main/LICENSE -[8]: https://spark.apache.org/ -[9]: http://www.apache.org/licenses/LICENSE-2.0.html -[10]: https://github.com/google/guava -[11]: http://www.apache.org/licenses/LICENSE-2.0.txt -[12]: https://netty.io/index.html -[13]: https://github.com/FasterXML/jackson -[14]: https://www.apache.org/licenses/LICENSE-2.0.txt -[15]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-common -[16]: http://www.eclipse.org/legal/epl-2.0 -[17]: https://www.gnu.org/software/classpath/license.html -[18]: https://creativecommons.org/publicdomain/zero/1.0/ -[19]: https://eclipse-ee4j.github.io/jersey/ -[20]: http://www.eclipse.org/org/documents/edl-v10.php -[21]: https://opensource.org/licenses/BSD-2-Clause -[22]: https://asm.ow2.io/license.html -[23]: https://github.com/jquery/jquery/blob/main/LICENSE.txt -[24]: http://www.opensource.org/licenses/mit-license.php -[25]: https://www.w3.org/Consortium/Legal/copyright-documents-19990405 -[26]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-server -[27]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-client -[28]: https://avro.apache.org -[29]: https://developers.google.com/protocol-buffers/docs/javatutorial -[30]: https://opensource.org/licenses/BSD-3-Clause -[31]: https://commons.apache.org/proper/commons-text -[32]: https://github.com/FasterXML/woodstox -[33]: http://www.scalatest.org -[34]: http://www.apache.org/licenses/LICENSE-2.0 -[35]: https://github.com/scalatest/scalatestplus-mockito -[36]: https://github.com/mockito/mockito -[37]: https://github.com/mockito/mockito/blob/main/LICENSE -[38]: https://logging.apache.org/log4j/2.x/log4j-api/ -[39]: https://logging.apache.org/log4j/2.x/ -[40]: https://logging.apache.org/log4j/2.x/log4j-core/ -[41]: https://github.com/exasol/test-db-builder-java/ -[42]: https://github.com/exasol/test-db-builder-java/blob/main/LICENSE -[43]: https://github.com/exasol/hamcrest-resultset-matcher/ -[44]: https://github.com/exasol/hamcrest-resultset-matcher/blob/main/LICENSE -[45]: https://github.com/exasol/exasol-testcontainers/ -[46]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE -[47]: http://sonarsource.github.io/sonar-scanner-maven/ -[48]: http://www.gnu.org/licenses/lgpl.txt -[49]: http://github.com/davidB/scala-maven-plugin -[50]: http://unlicense.org/ -[51]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[52]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[53]: https://www.mojohaus.org/flatten-maven-plugin/ -[54]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[55]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[56]: https://www.mojohaus.org/versions/versions-maven-plugin/ -[57]: https://github.com/basepom/duplicate-finder-maven-plugin -[58]: https://maven.apache.org/plugins/maven-deploy-plugin/ -[59]: https://maven.apache.org/plugins/maven-gpg-plugin/ -[60]: https://maven.apache.org/plugins/maven-source-plugin/ -[61]: https://maven.apache.org/plugins/maven-javadoc-plugin/ -[62]: http://www.sonatype.com/public-parent/nexus-maven-plugins/nexus-staging/nexus-staging-maven-plugin/ -[63]: http://www.eclipse.org/legal/epl-v10.html -[64]: https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin -[65]: https://maven.apache.org/plugins/maven-jar-plugin/ -[66]: https://maven.apache.org/plugins/maven-shade-plugin/ -[67]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[68]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[69]: https://www.eclipse.org/legal/epl-2.0/ -[70]: https://github.com/exasol/error-code-crawler-maven-plugin/ -[71]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE -[72]: http://zlika.github.io/reproducible-build-maven-plugin -[73]: https://github.com/exasol/project-keeper/ -[74]: https://github.com/exasol/project-keeper/blob/main/LICENSE -[75]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ -[76]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE -[77]: https://github.com/itsallcode/openfasttrace-maven-plugin -[78]: https://www.gnu.org/licenses/gpl-3.0.html -[79]: https://github.com/diffplug/spotless -[80]: https://github.com/evis/scalafix-maven-plugin -[81]: http://maven.apache.org/plugins/maven-clean-plugin/ -[82]: http://maven.apache.org/plugins/maven-resources-plugin/ -[83]: http://maven.apache.org/plugins/maven-install-plugin/ -[84]: http://maven.apache.org/plugins/maven-site-plugin/ +| [scalatest][50] | [the Apache License, ASL Version 2.0][51] | +| [scalatestplus-mockito][52] | [Apache-2.0][51] | +| [mockito-core][53] | [The MIT License][54] | +| [mockito-junit-jupiter][53] | [The MIT License][54] | +| [Apache Log4j API][55] | [Apache License, Version 2.0][3] | +| [Apache Log4j 1.x Compatibility API][56] | [Apache License, Version 2.0][3] | +| [Apache Log4j Core][57] | [Apache License, Version 2.0][3] | +| [Test Database Builder for Java][58] | [MIT License][59] | +| [Matcher for SQL Result Sets][60] | [MIT License][61] | +| [Test containers for Exasol on Docker][62] | [MIT License][63] | + +### Plugin Dependencies + +| Dependency | License | +| ------------------------------------------------------ | --------------------------------------------- | +| [SonarQube Scanner for Maven][0] | [GNU LGPL 3][1] | +| [scala-maven-plugin][64] | [Public domain (Unlicense)][65] | +| [Apache Maven Compiler Plugin][2] | [Apache-2.0][3] | +| [Apache Maven Enforcer Plugin][4] | [Apache-2.0][3] | +| [Maven Flatten Plugin][5] | [Apache Software Licenese][3] | +| [Apache Maven Javadoc Plugin][66] | [Apache-2.0][3] | +| [ScalaTest Maven Plugin][67] | [the Apache License, ASL Version 2.0][51] | +| [Apache Maven JAR Plugin][68] | [Apache License, Version 2.0][3] | +| [Apache Maven Deploy Plugin][6] | [Apache-2.0][3] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][7] | [ASL2][8] | +| [Maven Surefire Plugin][9] | [Apache-2.0][3] | +| [Versions Maven Plugin][10] | [Apache License, Version 2.0][3] | +| [Apache Maven Shade Plugin][69] | [Apache License, Version 2.0][3] | +| [duplicate-finder-maven-plugin Maven Mojo][70] | [Apache License 2.0][12] | +| [Maven Failsafe Plugin][71] | [Apache-2.0][3] | +| [JaCoCo :: Maven Plugin][13] | [Eclipse Public License 2.0][14] | +| [error-code-crawler-maven-plugin][15] | [MIT License][16] | +| [Reproducible Build Maven Plugin][17] | [Apache 2.0][8] | +| [OpenFastTrace Maven Plugin][18] | [GNU General Public License v3.0][19] | +| [spotless-maven-plugin][72] | [The Apache Software License, Version 2.0][3] | +| [scalafix-maven-plugin][73] | [BSD-3-Clause][74] | +| [Maven Clean Plugin][20] | [The Apache Software License, Version 2.0][8] | +| [Maven Resources Plugin][75] | [The Apache Software License, Version 2.0][8] | +| [Maven Install Plugin][21] | [The Apache Software License, Version 2.0][8] | +| [Maven Site Plugin 3][22] | [The Apache Software License, Version 2.0][8] | + +## Spark Exasol Connector With s3 + +### Compile Dependencies + +| Dependency | License | +| ------------------------------------------- | --------------------------------- | +| [Scala Library][76] | [Apache-2.0][34] | +| [spark-connector-common-java][29] | [MIT License][30] | +| [Spark Project Core][31] | [Apache 2.0 License][12] | +| [Spark Project SQL][31] | [Apache 2.0 License][12] | +| Apache Hadoop Client Aggregator | [Apache License, Version 2.0][3] | +| [Netty/All-in-One][33] | [Apache License, Version 2.0][34] | +| [AWS Java SDK :: Services :: Amazon S3][77] | [Apache License, Version 2.0][78] | +| Apache Hadoop Amazon Web Services support | [Apache License, Version 2.0][3] | +| [wildfly-openssl][79] | [Apache License 2.0][80] | + +### Test Dependencies + +| Dependency | License | +| ----------------------------------------------- | --------------------------------- | +| [JUnit Jupiter (Aggregator)][81] | [Eclipse Public License v2.0][82] | +| [JUnit Jupiter API][81] | [Eclipse Public License v2.0][82] | +| [Test Database Builder for Java][58] | [MIT License][59] | +| [Matcher for SQL Result Sets][60] | [MIT License][61] | +| [Test containers for Exasol on Docker][62] | [MIT License][63] | +| [Testcontainers :: JUnit Jupiter Extension][83] | [MIT][84] | +| [mockito-junit-jupiter][53] | [The MIT License][54] | +| [Testcontainers :: Localstack][83] | [MIT][84] | +| [AWS Java SDK for Amazon S3][77] | [Apache License, Version 2.0][78] | + +### Plugin Dependencies + +| Dependency | License | +| ------------------------------------------------------ | --------------------------------------------- | +| [SonarQube Scanner for Maven][0] | [GNU LGPL 3][1] | +| [Apache Maven Compiler Plugin][2] | [Apache-2.0][3] | +| [Apache Maven Enforcer Plugin][4] | [Apache-2.0][3] | +| [Maven Flatten Plugin][5] | [Apache Software Licenese][3] | +| [Apache Maven Deploy Plugin][6] | [Apache-2.0][3] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][7] | [ASL2][8] | +| [Maven Surefire Plugin][9] | [Apache-2.0][3] | +| [Versions Maven Plugin][10] | [Apache License, Version 2.0][3] | +| [Apache Maven Shade Plugin][69] | [Apache License, Version 2.0][3] | +| [duplicate-finder-maven-plugin Maven Mojo][70] | [Apache License 2.0][12] | +| [Maven Failsafe Plugin][71] | [Apache-2.0][3] | +| [JaCoCo :: Maven Plugin][13] | [Eclipse Public License 2.0][14] | +| [error-code-crawler-maven-plugin][15] | [MIT License][16] | +| [Reproducible Build Maven Plugin][17] | [Apache 2.0][8] | +| [OpenFastTrace Maven Plugin][18] | [GNU General Public License v3.0][19] | +| [Maven Clean Plugin][20] | [The Apache Software License, Version 2.0][8] | +| [Maven Resources Plugin][75] | [The Apache Software License, Version 2.0][8] | +| [Maven JAR Plugin][85] | [The Apache Software License, Version 2.0][8] | +| [Maven Install Plugin][21] | [The Apache Software License, Version 2.0][8] | +| [Maven Site Plugin 3][22] | [The Apache Software License, Version 2.0][8] | + +[0]: http://sonarsource.github.io/sonar-scanner-maven/ +[1]: http://www.gnu.org/licenses/lgpl.txt +[2]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[3]: https://www.apache.org/licenses/LICENSE-2.0.txt +[4]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[5]: https://www.mojohaus.org/flatten-maven-plugin/ +[6]: https://maven.apache.org/plugins/maven-deploy-plugin/ +[7]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[8]: http://www.apache.org/licenses/LICENSE-2.0.txt +[9]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[10]: https://www.mojohaus.org/versions/versions-maven-plugin/ +[11]: https://basepom.github.io/duplicate-finder-maven-plugin +[12]: http://www.apache.org/licenses/LICENSE-2.0.html +[13]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[14]: https://www.eclipse.org/legal/epl-2.0/ +[15]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[16]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[17]: http://zlika.github.io/reproducible-build-maven-plugin +[18]: https://github.com/itsallcode/openfasttrace-maven-plugin +[19]: https://www.gnu.org/licenses/gpl-3.0.html +[20]: http://maven.apache.org/plugins/maven-clean-plugin/ +[21]: http://maven.apache.org/plugins/maven-install-plugin/ +[22]: http://maven.apache.org/plugins/maven-site-plugin/ +[23]: http://www.exasol.com +[24]: https://repo1.maven.org/maven2/com/exasol/exasol-jdbc/7.1.20/exasol-jdbc-7.1.20-license.txt +[25]: https://github.com/exasol/sql-statement-builder/ +[26]: https://github.com/exasol/sql-statement-builder/blob/main/LICENSE +[27]: https://github.com/exasol/error-reporting-java/ +[28]: https://github.com/exasol/error-reporting-java/blob/main/LICENSE +[29]: https://github.com/exasol/spark-connector-common-java/ +[30]: https://github.com/exasol/spark-connector-common-java/blob/main/LICENSE +[31]: https://spark.apache.org/ +[32]: https://github.com/google/guava +[33]: https://netty.io/index.html +[34]: https://www.apache.org/licenses/LICENSE-2.0 +[35]: https://github.com/FasterXML/jackson +[36]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-common +[37]: http://www.eclipse.org/legal/epl-2.0 +[38]: https://www.gnu.org/software/classpath/license.html +[39]: https://creativecommons.org/publicdomain/zero/1.0/ +[40]: https://eclipse-ee4j.github.io/jersey/ +[41]: http://www.eclipse.org/org/documents/edl-v10.php +[42]: https://opensource.org/licenses/BSD-2-Clause +[43]: https://asm.ow2.io/license.html +[44]: https://github.com/jquery/jquery/blob/main/LICENSE.txt +[45]: http://www.opensource.org/licenses/mit-license.php +[46]: https://www.w3.org/Consortium/Legal/copyright-documents-19990405 +[47]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-server +[48]: https://projects.eclipse.org/projects/ee4j.jersey/jersey-client +[49]: https://avro.apache.org +[50]: http://www.scalatest.org +[51]: http://www.apache.org/licenses/LICENSE-2.0 +[52]: https://github.com/scalatest/scalatestplus-mockito +[53]: https://github.com/mockito/mockito +[54]: https://github.com/mockito/mockito/blob/main/LICENSE +[55]: https://logging.apache.org/log4j/2.x/log4j-api/ +[56]: https://logging.apache.org/log4j/2.x/ +[57]: https://logging.apache.org/log4j/2.x/log4j-core/ +[58]: https://github.com/exasol/test-db-builder-java/ +[59]: https://github.com/exasol/test-db-builder-java/blob/main/LICENSE +[60]: https://github.com/exasol/hamcrest-resultset-matcher/ +[61]: https://github.com/exasol/hamcrest-resultset-matcher/blob/main/LICENSE +[62]: https://github.com/exasol/exasol-testcontainers/ +[63]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE +[64]: http://github.com/davidB/scala-maven-plugin +[65]: http://unlicense.org/ +[66]: https://maven.apache.org/plugins/maven-javadoc-plugin/ +[67]: https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin +[68]: https://maven.apache.org/plugins/maven-jar-plugin/ +[69]: https://maven.apache.org/plugins/maven-shade-plugin/ +[70]: https://github.com/basepom/duplicate-finder-maven-plugin +[71]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[72]: https://github.com/diffplug/spotless +[73]: https://github.com/evis/scalafix-maven-plugin +[74]: https://opensource.org/licenses/BSD-3-Clause +[75]: http://maven.apache.org/plugins/maven-resources-plugin/ +[76]: https://www.scala-lang.org/ +[77]: https://aws.amazon.com/sdkforjava +[78]: https://aws.amazon.com/apache2.0 +[79]: http://www.jboss.org/wildfly-openssl-parent/wildfly-openssl +[80]: http://repository.jboss.org/licenses/apache-2.0.txt +[81]: https://junit.org/junit5/ +[82]: https://www.eclipse.org/legal/epl-v20.html +[83]: https://testcontainers.org +[84]: http://opensource.org/licenses/MIT +[85]: http://maven.apache.org/plugins/maven-jar-plugin/ diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index d9edf6e2..41c1595b 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,6 @@ # Changes -* [1.4.1](changes_1.4.1.md) +* [2.0.0](changes_2.0.0.md) * [1.4.0](changes_1.4.0.md) * [1.3.0](changes_1.3.0.md) * [1.2.5](changes_1.2.5.md) diff --git a/doc/changes/changes_1.4.1.md b/doc/changes/changes_1.4.1.md deleted file mode 100644 index 2742c5dc..00000000 --- a/doc/changes/changes_1.4.1.md +++ /dev/null @@ -1,21 +0,0 @@ -# The Spark Exasol Connector 1.4.1, released 2023-??-?? - -Code name: - -## Summary - -## Features - -## Security - -* #151: Fixed vulnerability `CVE-2023-26048` coming with `jetty-util` transitive dependency - -## Dependency Updates - -### Compile Dependency Updates - -* Updated `com.google.protobuf:protobuf-java:3.22.3` to `3.22.4` - -### Test Dependency Updates - -* Updated `org.mockito:mockito-core:5.3.0` to `5.3.1` diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md new file mode 100644 index 00000000..44dc5752 --- /dev/null +++ b/doc/changes/changes_2.0.0.md @@ -0,0 +1,154 @@ +# The Spark Exasol Connector 2.0.0, released 2023-07-14 + +Code name: Support S3 intermediate storage + +## Summary + +In this release we added support to use AWS S3 bucket as an intermediate storage layer when accessing Exasol database from Spark cluster. + +With this release, we separated the connector into two variants, `S3` and `JDBC`. We recommend to use this new S3 variant instead of JDBC variant. It improves the stability of the connector. + +Please refer to the user guide for updated usage instructions. + +## Features + +* #149: Added s3 import query generator and runner +* #150: Added S3 intermediate storage layer +* #159: Added cleanup process to remove intermediate data after job finish +* #160: Add support for writing to Exasol database using S3 as intermediate storage +* #168: Refactored to add module setup + +## Security + +* #151: Fixed vulnerability `CVE-2023-26048` coming with `jetty-util` transitive dependency + +## Bugs + +* #176: Fixed artifact upload of `sha256sum` files + +## Refactoring + +* #155: Unified user options +* #158: Refactored common options class +* #164: Validated that write directory is empty +* #171: Refactored artifact packaging and releasing for module setup +* #174: Refactored Github `.github/workflow/` action files +* #183: Updated user guide and prepared for release + +## Dependency Updates + +### Spark Exasol Connector Parent POM + +#### Plugin Dependency Updates + +* Added `com.exasol:error-code-crawler-maven-plugin:1.3.0` +* Added `io.github.zlika:reproducible-build-maven-plugin:0.16` +* Added `org.apache.maven.plugins:maven-clean-plugin:2.5` +* Added `org.apache.maven.plugins:maven-compiler-plugin:3.11.0` +* Added `org.apache.maven.plugins:maven-deploy-plugin:3.1.1` +* Added `org.apache.maven.plugins:maven-enforcer-plugin:3.3.0` +* Added `org.apache.maven.plugins:maven-install-plugin:2.4` +* Added `org.apache.maven.plugins:maven-site-plugin:3.3` +* Added `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` +* Added `org.basepom.maven:duplicate-finder-maven-plugin:2.0.1` +* Added `org.codehaus.mojo:flatten-maven-plugin:1.5.0` +* Added `org.codehaus.mojo:versions-maven-plugin:2.16.0` +* Added `org.itsallcode:openfasttrace-maven-plugin:1.6.2` +* Added `org.jacoco:jacoco-maven-plugin:0.8.10` +* Added `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` +* Added `org.sonatype.ossindex.maven:ossindex-maven-plugin:3.2.0` + +### Spark Exasol Connector With JDBC + +#### Compile Dependency Updates + +* Added `com.exasol:error-reporting-java8:1.0.1` +* Added `com.exasol:exasol-jdbc:7.1.20` +* Added `com.exasol:spark-connector-common-java:1.1.1` +* Added `com.exasol:sql-statement-builder-java8:4.5.4` + +#### Test Dependency Updates + +* Added `com.exasol:exasol-testcontainers:6.6.1` +* Added `com.exasol:hamcrest-resultset-matcher:1.6.0` +* Added `com.exasol:test-db-builder-java:3.4.2` +* Added `org.apache.logging.log4j:log4j-1.2-api:2.20.0` +* Added `org.apache.logging.log4j:log4j-api:2.20.0` +* Added `org.apache.logging.log4j:log4j-core:2.20.0` +* Added `org.mockito:mockito-core:5.4.0` +* Added `org.mockito:mockito-junit-jupiter:5.4.0` +* Added `org.scalatestplus:scalatestplus-mockito_2.13:1.0.0-M2` +* Added `org.scalatest:scalatest_2.13:3.2.9` + +#### Plugin Dependency Updates + +* Added `com.diffplug.spotless:spotless-maven-plugin:2.37.0` +* Added `com.exasol:error-code-crawler-maven-plugin:1.2.3` +* Added `io.github.evis:scalafix-maven-plugin_2.13:0.1.4_0.9.31` +* Added `io.github.zlika:reproducible-build-maven-plugin:0.16` +* Added `net.alchim31.maven:scala-maven-plugin:4.8.1` +* Added `org.apache.maven.plugins:maven-clean-plugin:2.5` +* Added `org.apache.maven.plugins:maven-compiler-plugin:3.11.0` +* Added `org.apache.maven.plugins:maven-deploy-plugin:3.1.1` +* Added `org.apache.maven.plugins:maven-enforcer-plugin:3.3.0` +* Added `org.apache.maven.plugins:maven-failsafe-plugin:3.0.0` +* Added `org.apache.maven.plugins:maven-install-plugin:2.4` +* Added `org.apache.maven.plugins:maven-jar-plugin:3.3.0` +* Added `org.apache.maven.plugins:maven-javadoc-plugin:3.5.0` +* Added `org.apache.maven.plugins:maven-resources-plugin:2.6` +* Added `org.apache.maven.plugins:maven-shade-plugin:3.4.1` +* Added `org.apache.maven.plugins:maven-site-plugin:3.3` +* Added `org.apache.maven.plugins:maven-surefire-plugin:3.0.0` +* Added `org.basepom.maven:duplicate-finder-maven-plugin:1.5.1` +* Added `org.codehaus.mojo:flatten-maven-plugin:1.4.1` +* Added `org.codehaus.mojo:versions-maven-plugin:2.15.0` +* Added `org.itsallcode:openfasttrace-maven-plugin:1.6.2` +* Added `org.jacoco:jacoco-maven-plugin:0.8.9` +* Added `org.scalatest:scalatest-maven-plugin:2.2.0` +* Added `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` +* Added `org.sonatype.ossindex.maven:ossindex-maven-plugin:3.2.0` + +### Spark Exasol Connector With S3 + +#### Compile Dependency Updates + +* Added `com.exasol:spark-connector-common-java:1.1.1` +* Added `org.apache.hadoop:hadoop-aws:3.3.4` +* Added `org.scala-lang:scala-library:2.13.11` +* Added `org.wildfly.openssl:wildfly-openssl:2.2.5.Final` +* Added `software.amazon.awssdk:s3:2.20.100` + +#### Test Dependency Updates + +* Added `com.amazonaws:aws-java-sdk-s3:1.12.503` +* Added `com.exasol:exasol-testcontainers:6.6.1` +* Added `com.exasol:hamcrest-resultset-matcher:1.6.0` +* Added `com.exasol:test-db-builder-java:3.4.2` +* Added `org.junit.jupiter:junit-jupiter-api:5.9.3` +* Added `org.junit.jupiter:junit-jupiter:5.9.3` +* Added `org.mockito:mockito-junit-jupiter:5.4.0` +* Added `org.testcontainers:junit-jupiter:1.18.3` +* Added `org.testcontainers:localstack:1.18.3` + +#### Plugin Dependency Updates + +* Added `com.exasol:error-code-crawler-maven-plugin:1.2.3` +* Added `io.github.zlika:reproducible-build-maven-plugin:0.16` +* Added `org.apache.maven.plugins:maven-clean-plugin:2.5` +* Added `org.apache.maven.plugins:maven-compiler-plugin:3.11.0` +* Added `org.apache.maven.plugins:maven-deploy-plugin:3.1.1` +* Added `org.apache.maven.plugins:maven-enforcer-plugin:3.3.0` +* Added `org.apache.maven.plugins:maven-failsafe-plugin:3.0.0` +* Added `org.apache.maven.plugins:maven-install-plugin:2.4` +* Added `org.apache.maven.plugins:maven-jar-plugin:2.4` +* Added `org.apache.maven.plugins:maven-resources-plugin:2.6` +* Added `org.apache.maven.plugins:maven-shade-plugin:3.4.1` +* Added `org.apache.maven.plugins:maven-site-plugin:3.3` +* Added `org.apache.maven.plugins:maven-surefire-plugin:3.0.0` +* Added `org.basepom.maven:duplicate-finder-maven-plugin:1.5.1` +* Added `org.codehaus.mojo:flatten-maven-plugin:1.4.1` +* Added `org.codehaus.mojo:versions-maven-plugin:2.15.0` +* Added `org.itsallcode:openfasttrace-maven-plugin:1.6.2` +* Added `org.jacoco:jacoco-maven-plugin:0.8.9` +* Added `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` +* Added `org.sonatype.ossindex.maven:ossindex-maven-plugin:3.2.0` diff --git a/doc/development/developer_guide.md b/doc/development/developer_guide.md index 9a149116..b741c521 100644 --- a/doc/development/developer_guide.md +++ b/doc/development/developer_guide.md @@ -1,28 +1,37 @@ # Developer Guide -Please read the general [developer guide for the Scala projects][dev-guide]. +## S3 Write Path Validation -## Integration Tests +When using S3 storage as intermediate layer, we generate a S3 bucket path for intermediate data. The generated path is checked that it is empty before writing data. -The integration tests are run using [Docker][docker] containers. The tests use -[exasol-testcontainers][exa-testcontainers] and -[spark-testing-base][spark-testing-base]. +The path layout: -[docker]: https://www.docker.com/ -[exa-testcontainers]: https://github.com/exasol/exasol-testcontainers/ -[spark-testing-base]: https://github.com/holdenk/spark-testing-base -[dev-guide]: https://github.com/exasol/import-export-udf-common-scala/blob/master/doc/development/developer_guide.md +``` +userProvidedS3Bucket/ +└── -/ + └── / +``` -## Release +The generated intermediate write path `-//` is validated that it is empty before write. And it is cleaned up after the write query finishes. -Currently [release-droid](https://github.com/exasol/release-droid) has some troubles releasing the spark connector. +## S3 Staging Commit Process -* **Validation**: When using the local folder with `release-droid -l .` then validation failes with error message `NumberFormatException for input string: "v0"`.
-Please validate spark-connector without option `-l .`.
-See also [release-droid/issue/245](https://github.com/exasol/release-droid/issues/245). -* **Language Java**: Although spark-connector contains scala code as well, there also is a pom file, though.
-In order to publish spark-connector to maven central using language "Java" is correct. -* For **releasing** spark-connector you should use -``` -java -jar path/to/release-droid-*.jar" -n spark-connector -lg Java -g release -``` +The Spark job that writes data to Exasol uses an AWS S3 bucket as intermediate storage. In this process, the `ExasolS3Table` API implementation uses Spark [`CSVTable`](https://github.com/apache/spark/blob/master/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVTable.scala) writer to create files in S3. + +The write process continues as following: + +1. We ask Spark's `CSVTable` to commit data into S3 bucket +1. We commit to import this data into Exasol database using Exasol's `CSV` loader +1. And finally we ask our `ExasolS3Table` API implementation to commit the write process + +If any failure occurs, each step will trigger the `abort` method and S3 bucket locations will be cleaned up. If job finishes successfully, the Spark job end listener will trigger the cleanup process. + +## S3 Maximum Number of Files + +For the write Spark jobs, we allow maximum of `1000` CSV files to be written as intermediate data into S3 bucket. The main reason for this is that S3 SDK `listObjects` command returns up to 1000 objects from a bucket path per each request. + +Even though we could improve it to list more objects from S3 bucket with multiple requests, we wanted to keep this threshold for now. + +## Integration Tests + +The integration tests are run using [Docker](https://www.docker.com) and [exasol-testcontainers](https://github.com/exasol/exasol-testcontainers/) diff --git a/doc/images/databricks-deployment.png b/doc/images/databricks-deployment.png deleted file mode 100644 index e14c1059..00000000 Binary files a/doc/images/databricks-deployment.png and /dev/null differ diff --git a/doc/user_guide/user_guide.md b/doc/user_guide/user_guide.md index 5c93689c..af95a417 100644 --- a/doc/user_guide/user_guide.md +++ b/doc/user_guide/user_guide.md @@ -7,7 +7,9 @@ Exasol tables. ## Table of Contents - [Getting Started](#getting-started) -- [Spark Exasol Connector as Dependency](#spark-exasol-connector-as-dependency) +- [Versioning](#versioning) +- [Format](#format) +- [Using as Dependency](#using-as-dependency) - [Configuration Parameters](#configuration-options) - [Creating a Spark DataFrame From Exasol Query](#creating-a-spark-dataframe-from-exasol-query) - [Saving Spark DataFrame to an Exasol Table](#saving-spark-dataframe-to-an-exasol-table) @@ -15,215 +17,215 @@ Exasol tables. ## Getting Started -To use the connector, we assume you have an Exasol cluster running with a -version `6.0` or above. Similarly, you have a Spark cluster running with a -version `2.3.0` or later. +The Spark Exasol connector has two variants, one for using with Exasol `JDBC` sub-connections and one for using with AWS `S3` as intermediate storage. -Please make sure that there is an access connection between the two clusters via -internal network addresses. The Spark Exasol Connector makes parallel -connections to the Exasol data nodes and assumes that data nodes have sequential -IPv4 addresses. For example, the first data node has an address `10.0.0.11`, the -second one is assigned `10.0.0.12`, and so on. +We recommend to use the connector with `S3` storage layer, because this version improves several shortcomings of `JDBC` variant. The JDBC sub-connections version are not reliably working when Spark uses dynamic resource scheduling. In other words, when there are limited resource on Spark clusters, using S3 version of connector is better. -In addition, please make sure that the Exasol nodes are reachable from the Spark -cluster on the JDBC port (`8563`) and port range `20000-21000`. The port range -is used for the parallel connections from the Spark tasks to Exasol data nodes. +Depending on the variant you are using, some usage requirements may change. For example, for JDBC the `host` parameter should be the first datanode address. -## Spark Exasol Connector as Dependency +### JDBC -The Spark Exasol Connector is released to the Maven Central Repository. You can -find all the releases at [com.exasol/spark-connector][maven-link] page. +When using the connector with JDBC, please make sure that there is an access connection between the two clusters via internal network addresses. The Spark Exasol Connector makes parallel connections to the Exasol data nodes and assumes that data nodes have sequential IPv4 addresses. For example, the first data node has an address `10.0.0.11`, the second one is assigned `10.0.0.12`, and so on. -[maven-link]: https://central.sonatype.com/search?q=%2522spark-connector_2.13%2522%2520%2522com.exasol%2522 +Additionally, please make sure that the Exasol nodes are reachable from the Spark cluster on the JDBC port (`8563`) and port range `20000-21000`. The port range is used for the parallel connections from the Spark tasks to Exasol data nodes. -There are several options to include the connector as a dependency to your -projects. +### S3 -### Spark Exasol Connector as Maven Dependency +When using with S3 intermediate storage please make sure that there is access to an S3 bucket. And please prepare AWS access and secret keys with enough permissions for the S3 bucket. + +## Versioning + +The Spark Exasol connector is released for specific Spark versions. + +Each version has parts for Scala version, connector version and target Spark runtime version. + +For example, `spark-connector-s3_2.13:2.0.0-spark-3.4.1` artifact shows that it is for S3 variant, released with Scala 2.13 version, connector release version is `2.0.0` and it is released for Spark `3.4.1` runtime. + +## Format + +For creating a Spark dataframe, you should provide the `format`. + +For example, when reading from a source `spark.read.format()...`. + +Depending on the connector variant, use the following formats: + +- JDBC — `"exasol-jdbc"` +- S3 — `"exasol-s3"` + +## Using as Dependency + +We publish the Spark Exasol Connector to the Maven Central Repository. With that, you could include it as a dependency in your Spark applications. + +Here we show `Maven` dependency as an example, but you can find other ways of using the artifact on the [Maven Central Release](https://mvnrepository.com/artifact/com.exasol/spark-connector) page. + +### Using as Maven Dependency You can provide the connector as a dependency to your Spark Java applications. +#### S3 + ```xml com.exasol - spark-connector_2.13 + spark-connector-s3_2.13 ``` -Please do not forget to update the `` placeholder with one of the -latest Spark Exasol Connector releases. - -### Spark Exasol Connector as SBT Dependency +#### JDBC -To provide the connector to your Spark Scala projects, add the following lines -to your `build.sbt` file: - -```scala -libraryDependencies += "com.exasol" % "spark-connector" %% "" +```xml + + + com.exasol + spark-connector-jdbc_2.13 + + + ``` -Similarly, please do not forget to update the `` placeholder. - -### Spark Exasol Connector as Databricks Cluster Dependency +Please do not forget to update the `` placeholder with the latest Spark Exasol Connector releases. -Similar to using maven, you should provide maven artifact coordinates to the -[Databricks Workspace Library][databricks-install]. +### Using as Databricks Cluster Dependency -[databricks-install]: https://docs.databricks.com/libraries/workspace-libraries.html#maven-libraries - -databricks-deployment - -Go to your cluster, then to `Libraries`, and click `Install New`: +You can upload the assembled jar file or provide maven artifact coordinates to the [Databricks Workspace Library](https://docs.databricks.com/libraries/workspace-libraries.html#maven-libraries). +- Go to Azure → Databricks → Workspace → Clusters +- Go to Libraries → and click Install New - Select Maven as a `Library Source` -- In the `Coordinates` field, enter artifact coordinates - `com.exasol:spark-connector_2.13:`. Please note that we use the - Scala version 2.13. Please contact us know if you require Scala version 2.12. +- In the `Coordinates` field, enter artifact coordinates `com.exasol:spark-connector-s3_2.13:`. Please note that we use the Scala version 2.13. Please refer to releases if you require Scala version 2.12. This should match the Spark cluster runtime Scala version. - Click `Install` -Please change the `` to one of the latest Spark Exasol Connector -releases. +Similar to above steps you could also upload the assembled jar file. -### Spark Exasol Connector With Spark Shell +- Select Upload as `Library Source` +- Jar as a `Library Type` +- Drop the **assembled** jar file with `-assembly` suffix +- Click to `Install` -You can also integrate the Spark Exasol Connector to the Spark Shell. Provide -the artifact coordinates `--packages` parameters: +### Using With Spark Shell + +You can also integrate the Spark Exasol Connector to the Spark Shell. Provide the artifact coordinates using the `--packages` parameter: ```sh -spark-shell --packages com.exasol:spark-connector_2.13: +spark-shell --packages com.exasol:spark-connector-s3_2.13: ``` -The `spark-shell` provides Read-Eval-Print-Loop (REPL) to interactively learn -and experiment with the API. +The `spark-shell` provides a Read-Eval-Print-Loop (REPL) to interactively learn and experiment with the API. -### Spark Exasol Connector With Spark Submit +### Using With Spark Submit -Additionally, you can provide the connector when submitting a packaged -application into the Spark cluster. +Additionally, you can provide the connector when submitting a packaged application into the Spark cluster. Use the `spark-submit` command: ```sh spark-submit \ --master spark://spark-master-url:7077 \ - --packages com.exasol:spark-connector_2.13: \ - --class com.organization.SparkApplication \ + --packages com.exasol:spark-connector-jdbc_2.13: \ + --class com.example.SparkApplication \ path/to/spark_application.jar ``` -The `--packages` parameter can be omitted if your Spark application -JAR already includes the connector as a dependency (e.g, `jar-with-dependencies`). +The `--packages` parameter can be omitted if your Spark application JAR already includes the connector as a dependency (e.g, `jar-with-dependencies`). -Like `spark-shell` and `spark-submit`, you can also use `pyspark` and `sparkR` -commands. +Like `spark-shell` and `spark-submit`, you can also use `pyspark` and `sparkR` commands. -### Spark Exasol Connector as JAR Dependency +### Using as JAR Dependency -Please check out the -[releases](https://github.com/exasol/spark-connector/releases) page for already -assembled jar file. Each release contains a jar file with `-assembly` suffix to -that respective version. +Please check out the [releases](https://github.com/exasol/spark-connector/releases) page for already assembled jar file. Each release contains a jar file with `-assembly` suffix to that respective version. This jar file contains all required dependencies as a "shaded jar". -You can also build an assembled jar from the source. This way, you use the -latest commits that may not be released yet. +Then you can use this jar file with `spark-submit`, `spark-shell` or `pyspark` commands. -Clone the repository: +For example, S3 variant with version `2.0.0-spark-3.4.1`: ```sh -git clone https://github.com/exasol/spark-connector - -cd spark-connector/ +spark-shell --jars spark-connector-s3_2.13-2.0.0-spark-3.4.1-assembly.jar ``` -To create an assembled jar file, run the command: +## Configuration Options -```sh -mvn package -DskipTests=true -``` +In this section, we describe the common configuration parameters that are used for both JDBC and S3 variants to facilitate the integration between Spark and Exasol clusters. -The assembled jar file with the `-assembly` suffix should be located in the `target/` -folder. +List of common required and optional parameters: -If you want different version of Spark, you can use profiles `-Pspark3.2` or -`-Pspark3.1` for Spark `3.2` or `3.1` versions, respectively. +| Configuration | Default | Description | +| :------------- | :---------- | :----------------------------------------------------------------------------------------------- | +| `query` | __ | An Exasol SQL query string to send to Exasol cluster | +| `table` | __ | A table name (with schema, e.g. schema.table) to save dataframe into | +| `host` | `10.0.0.11` | A host ip address to the Exasol node | +| `port` | `8563` | A JDBC port number to connect to Exasol database | +| `username` | `sys` | A username for connecting to the Exasol database | +| `password` | `exasol` | A password for connecting to the Exasol database | +| `fingerprint` | `""` | A Exasol connection certificate fingerprint value | +| `jdbc_options` | `""` | A string to specify the list of Exasol JDBC options using format `key1=value1;key2=value2` | -For example, +The `query` parameter is required when you are reading data from Exasol database. Likewise, the `table` parameter is required when you are writing to an Exasol table. -```sh -mvn package -Pspark3.2 -DskipTests=true -``` +### JDBC Options -Then you can use this jar file with `spark-submit`, `spark-shell` or `pyspark` -commands. +The Spark Exasol Connector uses Exasol JDBC Driver to connect to the Exasol cluster from the Spark cluster. You can use this configuration parameter to enrich the JDBC connection. -```sh -spark-shell --jars path/to/assemled-jar +For example, to enable debugging with a log directory: + +``` +.option("jdbc_options", "debug=1;logdir=/tmp/") ``` -## Configuration Options +Please make sure that the `jdbc_options` value does not start or end with a semicolon (`;`). -In this section, we describe the configuration parameters that are used to -facilitate the integration between Spark and Exasol clusters. +For more JDBC options please check the [Exasol JDBC documentation](https://docs.exasol.com/db/latest/connect_exasol/drivers/jdbc.htm). -List of required and optional parameters: +### JDBC Related Configuration Options -| Spark Configuration | Configuration | Default | Description | -| :-------------------------- | :------------- | :---------- | :----------------------------------------------------------------------------------------------- | -| | `query` | __ | An Exasol SQL query string to send to Exasol cluster | -| | `table` | __ | A table name (with schema, e.g. schema.table) to save dataframe into | -| `spark.exasol.host` | `host` | `10.0.0.11` | A host ip address to the **first** Exasol node | -| `spark.exasol.port` | `port` | `8563` | A JDBC port number to connect to Exasol database | -| `spark.exasol.username` | `username` | `sys` | A username for connecting to the Exasol database | -| `spark.exasol.password` | `password` | `exasol` | A password for connecting to the Exasol database | -| `spark.exasol.fingerprint` | `fingerprint` | `""` | A Exasol connection certificate fingerprint value | -| `spark.exasol.max_nodes` | `max_nodes` | `200` | The number of data nodes in the Exasol cluster | -| `spark.exasol.batch_size` | `batch_size` | `1000` | The number of records batched before running an execute statement when saving dataframe | -| `spark.exasol.create_table` | `create_table` | `false` | A permission to create a table if it does not exist in the Exasol database when saving dataframe | -| `spark.exasol.drop_table` | `drop_table` | `false` | A permission to drop the table if it exists in the Exasol database when saving dataframe | -| `spark.exasol.jdbc_options` | `jdbc_options` | `""` | A string to specify the list of Exasol JDBC options using a `key1=value1;key2=value2` format | +When using the `JDBC` variant you can additionally set these parameters: -### Max Nodes +| Configuration | Default | Description | +| :------------- | :---------- | :----------------------------------------------------------------------------------------------- | +| `max_nodes` | `200` | The number of data nodes in the Exasol cluster | +| `batch_size` | `1000` | The number of records batched before running an execute statement when saving dataframe | +| `create_table` | `false` | A permission to create a table if it does not exist in the Exasol database when saving dataframe | +| `drop_table` | `false` | A permission to drop the table if it exists in the Exasol database when saving dataframe | -Setting the `max_nodes` value to a large number does not increase the connector -parallelism. The number of parallel connections is always limited to the -number of Exasol data nodes. +When saving a dataframe, you can provide two optional parameters: `drop_table` and `create_table`. -However, you can use this configuration to decrease the parallelism. This can be -helpful when debugging an issue. For example, you can set it to one and check if -the behavior changes. +If you set the `drop_table` configuration to `true`, then just before saving the Spark dataframe, the connector drops the Exasol table if it exists. -### JDBC Options +If you set the `create_table` configuration to `true`, the connector will eagerly try to create an Exasol table from a Spark dataframe schema before saving the contents of the dataframe. Depending on your use case, you can provide both of these parameters at the same time. -The Spark Exasol Connector uses Exasol JDBC Driver to connect to the Exasol -cluster from the Spark cluster. You can use this configuration parameter to enrich -the JDBC connection. +#### Max Nodes -For example, to enable debugging with a log directory: +Setting the `max_nodes` value to a large number does not increase the connector parallelism. The number of parallel connections is always limited to the number of Exasol data nodes. -``` -.option("jdbc_options", "debug=1;logdir=/tmp/") -``` +However, you can use this configuration to decrease the parallelism. This can be helpful when debugging an issue. For example, you can set it to `1` and check if the behavior changes. + +### S3 Related Configuration Options -Please make sure that the `jdbc_options` value does not start or end with a -semicolon (`;`). +When using the `S3` variant of the connector you should provide the following additional required or optional parameters. -For more JDBC options please check the [Exasol JDBC documentation][exasol-jdbc]. +| Parameter | Default | Required | Description | +|-----------------------|:------------------:|:--------:|-------------------------------------------------------------------- | +| `s3Bucket` | | ✓ | A bucket name for intermediate storage | +| `awsAccessKeyId` | | ✓ | AWS Access Key for accessing bucket | +| `awsSecretAccessKey` | | ✓ | AWS Secret Key for accessing bucket | +| `numPartitions` | `8` | | Number of partitions that will match number of files in `S3` bucket | +| `awsRegion` | `us-east-1` | | AWS Region for provided bucket | +| `awsEndpointOverride` | (default endpoint) | | AWS S3 Endpoint for bucket, set this value for custom endpoints | +| `s3PathStyleAccess` | `false` | | Path style access for bucket, set this value for custom S3 buckets | +| `useSsl` | `true` | | Enable HTTPS client access for S3 bucket | ### Providing Configuration Settings in DataFrame Load or Save -These are required coonfiguration parameters so that the connector can -authenticate itself with the Exasol database. +These are required configuration parameters so that the connector can authenticate itself with the Exasol database. -Provide the configuration options when creating a dataframe from an Exasol -query: +Provide the configuration options when creating a dataframe from an Exasol query: ```scala val exasolDF = sparkSession .read - .format("exasol") + .format("exasol-jdbc") .option("host", "10.0.0.11") .option("port", "8563") .option("username", "") @@ -232,14 +234,11 @@ val exasolDF = sparkSession .load() ``` -Similarly, you can set these configurations when saving a dataframe to an Exasol -table. +Similarly, you can set these configurations when saving a dataframe to an Exasol table. ### Providing Configuration Settings in SparkConf -You can set the configurations in the -[SparkConf](https://spark.apache.org/docs/latest/api/java/org/apache/spark/SparkConf.html) -object using the `spark.exasol.` prefix. +You can set the configurations in the [SparkConf](https://spark.apache.org/docs/latest/api/java/org/apache/spark/SparkConf.html) object using the `spark.exasol.` prefix. ```scala // Create a configuration object @@ -257,15 +256,13 @@ val sparkSession = SparkSession .getOrCreate() ``` -This way you can use the SparkSession with pre-configured settings when reading -or saving a dataframe from the Exasol database. +This way you can use the SparkSession with previously configured settings when reading or saving a dataframe from the Exasol database. Please note that configuration values set on SparkConf have higher precedence. ### Providing Configuration Settings With `spark-submit` -Like SparkConf, you can configure the Exasol key-value settings from outside using, -for example, `--conf key=value` syntax at startup. +Like SparkConf, you can configure the Exasol key-value settings from outside using, for example, `--conf key=value` syntax at startup. Provide configurations with `spark-submit`: @@ -278,18 +275,15 @@ spark-submit \ path/to/spark_application.jar ``` -This allows you to avoid hardcoding the credentials in your Spark applications. +This allows you to avoid hard-coding the credentials in your Spark applications. -Providing configurations parameters with `spark-submit` has a higher precedence -than the SparkConf configurations. +Providing configurations parameters with `spark-submit` has a higher precedence than the SparkConf configurations. ## Creating a Spark DataFrame From Exasol Query -You can query the Exasol database and load the results of the query into a Spark -dataframe. +You can query the Exasol database and load the results of the query into a Spark dataframe. -For that specify the data source format as `"exasol"` and provide the required -configurations. +For that specify the data source format and provide the required configurations. As an example we query two retail tables from the Exasol database: @@ -303,17 +297,14 @@ val exasolQueryString = """ """ ``` -Please combine your Exasol queries into a single query string and load the -result into the Spark DataFrame. This helps to reduce the additional network -overhead. At the moment, it is not possible to create many dataframes with -several separate queries. +Please combine your Exasol queries into a single query and load the result into the Spark DataFrame. This helps to reduce the additional network overhead. -Create a dataframe from the query result: +Create a dataframe from the query result using JDBC variant: ```scala val df = sparkSession .read - .format("exasol") + .format("exasol-jdbc") .option("host", "10.0.0.11") .option("port", "8563") .option("username", "") @@ -322,6 +313,24 @@ val df = sparkSession .load() ``` +Using S3 variant: + +```scala +val df = spark + .read + .format("exasol-s3") + .option("host", "10.10.0.2") + .option("port", "8563") + .option("username", "") + .option("password", "") + .option("query", exasolQueryString) + .option("awsAccessKeyId", "") + .option("awsSecretAccessKey", "") + .option("s3Bucket", "spark-s3-bucket") + .option("awsRegion", "eu-central-1") + .load() +``` + Now you can inspect the schema of the dataframe: ```scala @@ -334,8 +343,7 @@ If you are only interested in some columns, you can select them: df.select("MARKET_ID", "AMOUNT") ``` -Run other Spark related data analytics, run transformations and aggregations on the -dataframe: +Run other Spark related data analytic queries, run transformations and aggregations on the dataframe: ```scala val transformedDF = df @@ -355,34 +363,19 @@ You can also save the Spark dataframe into an Exasol table: groupedDF .write .mode("overwrite") - .format("exasol") + .format("exasol-jdbc") .option("host", "10.0.0.11") .option("port", "8563") .option("username", "") .option("password", "") - .option("create_table", "true") .option("table", ".") .save() ``` -Please notice that we create the table if it is not already available in the -Exasol database. +Please notice that we create the table if it is not already available in the Exasol database. ### Spark Save Modes -When saving a dataframe, you can provide two optional parameters: `drop_table` -and `create_table`. - -If you set the `drop_table` configuration to `true`, then just before saving the -Spark dataframe, the connector drops the Exasol table if it exists. - -If you set the `create_table` configuration to `true`, the connector will -eagerly try to create an Exasol table from a Spark dataframe schema before saving -the contents of the dataframe. - -Depending on your use case, you can provide both of these parameters at the same -time. - Additionally, a Spark save operation takes optional `SaveMode` configurations. | Spark Save Mode | Description | @@ -392,48 +385,25 @@ Additionally, a Spark save operation takes optional `SaveMode` configurations. | `"overwrite"` | If the table exists, it is truncated first and then the contents of the dataframe are saved to the table. | | `"ignore"` | If the table exists, the save operation is skipped, and nothing is changed in the existing table. | -Please keep in mind that Spark Save Modes do not use any locking mechanisms, -thus they are not atomic. +Please keep in mind that Spark Save Modes do not use any locking mechanisms, thus they are not atomic. ## Troubleshooting -In this section, we explain common issues and pitfalls when using Spark Exasol -Connector and provide instructions on how to solve them. +In this section, we explain common issues and pitfalls when using Spark Exasol Connector and provide instructions on how to solve them. ### Exasol JDBC Sub Connections -The Spark Exasol connector uses [Exasol JDBC Sub -Connections][jdbc-subconnections] underneath. The sub-connections are static by -design. You can use them after all the connections have been established. - -[jdbc-subconnections]: https://community.exasol.com/t5/database-features/parallel-connections-with-jdbc/ta-p/1779 - -However, this causes problems in certain situations since Spark tasks are very -dynamic. Depending on the available Spark cluster resource, tasks can be -scheduled dynamically, not all at once. - -In these cases, not all of the sub-connections will be consumed, and the -connector will throw an exception. - -A similar issue occurs when the number of parallel connections (the number of -Exasol data nodes) is more than the Spark tasks. This can happen when the Spark -cluster does not have enough resources to schedule parallel tasks. +The Spark Exasol connector uses [Exasol JDBC Sub Connections](https://community.exasol.com/t5/database-features/parallel-connections-with-jdbc/ta-p/1779) underneath. The sub-connections are static by design. You can use them after all the connections have been established. -For instance, an Exasol cluster has three data nodes, and a Spark cluster has -only two (virtual) CPUs. In this case, the Spark cluster can only schedule two -tasks at a time. In such situations, you can decrease the JDBC sub-connections -by setting `max_nodes` parameters to a lower number. +However, this causes problems in certain situations since Spark tasks are very dynamic. Depending on the available Spark cluster resource, tasks can be scheduled dynamically, not all at once. In these cases, not all of the sub-connections will be consumed, and the connector will throw an exception. A similar issue occurs when the number of parallel connections (the number of Exasol data nodes) is more than the Spark tasks. This can happen when the Spark cluster does not have enough resources to schedule parallel tasks. -To resolve these issues we are waiting for the [Spark Barrier Execution -Mode](https://issues.apache.org/jira/browse/SPARK-24374). +For instance, an Exasol cluster has three data nodes, and a Spark cluster has only two (virtual) CPUs. In this case, the Spark cluster can only schedule two tasks at a time. In such situations, you can decrease the JDBC sub-connections by setting `max_nodes` parameters to a lower number. ### Spark DataFrame `.show()` Action -The spark dataframe `.show()` action is one of the operations that fails because -of the problems described in the previous section. +The spark dataframe `.show()` action is one of the operations that fails because of the problems described in the previous section. -We recommend using the `collect()` operation with combination SQL -`LIMIT` clause instead. +We recommend using the `collect()` operation in combination with the SQL `LIMIT` clause instead. For example, add a limit clause to the Exasol query: @@ -474,19 +444,12 @@ This error occurs because of the issues we described above. [error] at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) ``` -It can be mitigated by submitting a Spark application with enough resources so -that it can start parallel tasks that are more or equal to the number of -parallel Exasol connections. +It can be mitigated by submitting a Spark application with enough resources so that it can start parallel tasks that are more or equal to the number of parallel Exasol connections. -Additionally, you can limit the Exasol parallel connections using the -`max_nodes` parameter. However, we do not advise to limit this value in the -production environment. +Additionally, you can limit the Exasol parallel connections using the `max_nodes` parameter. However, we do not advise to limit this value in the production environment. ### Connection Refused -This usually occurs when the Spark connector cannot reach Exasol data nodes. -Please make sure that the Exasol data nodes are reachable on port `8563` and on -port ranges `20000-21000`. +This usually occurs when the Spark connector cannot reach Exasol data nodes. Please make sure that the Exasol data nodes are reachable on port `8563` and on port ranges `20000-21000`. -Also, please make sure that the `host` parameter value is set to the first -Exasol data node address, for example, `10.0.0.11`. +Also, please make sure that the `host` parameter value is set to the first Exasol data node address, for example, `10.0.0.11`. diff --git a/error_code_config.yml b/error_code_config.yml index 31e65b61..f66154a5 100644 --- a/error_code_config.yml +++ b/error_code_config.yml @@ -2,4 +2,4 @@ error-tags: SEC: packages: - com.exasol.spark - highest-index: 11 + highest-index: 27 diff --git a/exasol-dist/error_code_config.yml b/exasol-dist/error_code_config.yml new file mode 100644 index 00000000..8c2fda39 --- /dev/null +++ b/exasol-dist/error_code_config.yml @@ -0,0 +1,5 @@ +error-tags: + SEC: + packages: + - com.exasol.spark + highest-index: 26 diff --git a/pk_generated_parent.pom b/exasol-dist/pk_generated_parent.pom similarity index 92% rename from pk_generated_parent.pom rename to exasol-dist/pk_generated_parent.pom index a04c6d1e..7fe19d8a 100644 --- a/pk_generated_parent.pom +++ b/exasol-dist/pk_generated_parent.pom @@ -3,8 +3,14 @@ 4.0.0com.exasolspark-connector-generated-parent - 1.4.1 + ${revision}pom + + com.exasol + spark-connector-parent-pom + ${revision} + ../pom.xml + UTF-8 UTF-8 @@ -255,27 +261,6 @@ - - org.apache.maven.plugins - maven-failsafe-plugin - 3.0.0 - - - -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} - - ${test.excludeTags} - - - - verify - - integration-test - verify - - - - org.jacoco jacoco-maven-plugin diff --git a/exasol-dist/src/test/resources/logging.properties b/exasol-dist/src/test/resources/logging.properties new file mode 100644 index 00000000..8c97abe9 --- /dev/null +++ b/exasol-dist/src/test/resources/logging.properties @@ -0,0 +1,6 @@ +handlers=java.util.logging.ConsoleHandler +.level=INFO +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%1$tF %1$tT.%1$tL [%4$-7s] %5$s %n +com.exasol.level=ALL diff --git a/exasol-dist/versionsMavenPluginRules.xml b/exasol-dist/versionsMavenPluginRules.xml new file mode 100644 index 00000000..35bd03d2 --- /dev/null +++ b/exasol-dist/versionsMavenPluginRules.xml @@ -0,0 +1,18 @@ + + + + + (?i).*Alpha(?:-?[\d.]+)? + (?i).*a(?:-?[\d.]+)? + (?i).*Beta(?:-?[\d.]+)? + (?i).*-B(?:-?[\d.]+)? + (?i).*-b(?:-?[\d.]+)? + (?i).*RC(?:-?[\d.]+)? + (?i).*CR(?:-?[\d.]+)? + (?i).*M(?:-?[\d.]+)? + + + + \ No newline at end of file diff --git a/exasol-jdbc/.scalafmt.conf b/exasol-jdbc/.scalafmt.conf new file mode 100644 index 00000000..74fc0126 --- /dev/null +++ b/exasol-jdbc/.scalafmt.conf @@ -0,0 +1,46 @@ +version = 3.7.3 +project.git = true +runner.dialect = scala213 + +maxColumn = 120 +docstrings.style = Asterisk +docstrings.wrap = "no" + +align = true +align.openParenCallSite = false +align.openParenDefnSite = false +align.stripMargin = true +assumeStandardLibraryStripMargin = true +continuationIndent.defnSite = 2 +danglingParentheses.preset = true + +rewrite.rules = [ + AsciiSortImports, + AvoidInfix, + PreferCurlyFors, + RedundantBraces, + RedundantParens, + SortModifiers +] +rewrite.neverInfix.excludeFilters = [ + at + exclude + excludeAll + in + to + until +] +rewrite.redundantBraces.generalExpressions = false +rewrite.redundantBraces.ifElseExpressions = false +rewrite.redundantBraces.stringInterpolation = true +rewrite.sortModifiers.order = [ + "`override`" + "`private`" + "`protected`" + "`sealed`" + "`abstract`" + "`lazy`" + "`implicit`" + "`final`" +] +spaces.inImportCurlyBraces = false diff --git a/exasol-jdbc/.settings/org.eclipse.jdt.core.prefs b/exasol-jdbc/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..f362cccd --- /dev/null +++ b/exasol-jdbc/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,502 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullable.secondary= +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.APILeak=warning +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.terminalDeprecation=warning +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=ignore +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentType=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentTypeStrict=disabled +org.eclipse.jdt.core.compiler.problem.unlikelyEqualsArgumentType=info +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unstableAutoModuleName=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false +org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647 +org.eclipse.jdt.core.formatter.align_type_members_on_columns=false +org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns=false +org.eclipse.jdt.core.formatter.align_with_spaces=false +org.eclipse.jdt.core.formatter.alignment_for_additive_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_enum_constant=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter=0 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type=49 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_assertion_message=0 +org.eclipse.jdt.core.formatter.alignment_for_assignment=0 +org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_loops=16 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain=0 +org.eclipse.jdt.core.formatter.alignment_for_enum_constants=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header=0 +org.eclipse.jdt.core.formatter.alignment_for_logical_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 +org.eclipse.jdt.core.formatter.alignment_for_module_statements=16 +org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references=0 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_record_components=16 +org.eclipse.jdt.core.formatter.alignment_for_relational_operator=0 +org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 +org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_shift_operator=0 +org.eclipse.jdt.core.formatter.alignment_for_string_concatenation=16 +org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_record_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_type_annotations=0 +org.eclipse.jdt.core.formatter.alignment_for_type_arguments=0 +org.eclipse.jdt.core.formatter.alignment_for_type_parameters=0 +org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 +org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_after_package=1 +org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 +org.eclipse.jdt.core.formatter.blank_lines_before_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 +org.eclipse.jdt.core.formatter.blank_lines_before_package=0 +org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 +org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch=0 +org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 +org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_lambda_body=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_record_constructor=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_record_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped=true +org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position=false +org.eclipse.jdt.core.formatter.comment.format_block_comments=true +org.eclipse.jdt.core.formatter.comment.format_header=false +org.eclipse.jdt.core.formatter.comment.format_html=true +org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true +org.eclipse.jdt.core.formatter.comment.format_line_comments=true +org.eclipse.jdt.core.formatter.comment.format_source_code=true +org.eclipse.jdt.core.formatter.comment.indent_parameter_description=false +org.eclipse.jdt.core.formatter.comment.indent_root_tags=false +org.eclipse.jdt.core.formatter.comment.indent_tag_description=false +org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags=do not insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=do not insert +org.eclipse.jdt.core.formatter.comment.line_length=120 +org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true +org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true +org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false +org.eclipse.jdt.core.formatter.compact_else_if=true +org.eclipse.jdt.core.formatter.continuation_indentation=2 +org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off +org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on +org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=false +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_record_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true +org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_empty_lines=false +org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false +org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_additive_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default=insert +org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_record_components=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert +org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow=insert +org.eclipse.jdt.core.formatter.insert_space_after_logical_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_not_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_relational_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert +org.eclipse.jdt.core.formatter.insert_space_after_shift_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation=insert +org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_additive_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case=insert +org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default=insert +org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_record_components=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow=insert +org.eclipse.jdt.core.formatter.insert_space_before_logical_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_constructor=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert +org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_relational_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_shift_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation=insert +org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.join_lines_in_comments=true +org.eclipse.jdt.core.formatter.join_wrapped_lines=true +org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_code_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_method_body_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_record_constructor_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_record_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line=false +org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.lineSplit=120 +org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false +org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false +org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block=0 +org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_record_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause=common_lines +org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.jdt.core.formatter.tabulation.char=space +org.eclipse.jdt.core.formatter.tabulation.size=4 +org.eclipse.jdt.core.formatter.text_block_indentation=0 +org.eclipse.jdt.core.formatter.use_on_off_tags=false +org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=true +org.eclipse.jdt.core.formatter.wrap_before_additive_operator=true +org.eclipse.jdt.core.formatter.wrap_before_assertion_message_operator=true +org.eclipse.jdt.core.formatter.wrap_before_assignment_operator=false +org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator=true +org.eclipse.jdt.core.formatter.wrap_before_conditional_operator=true +org.eclipse.jdt.core.formatter.wrap_before_logical_operator=true +org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator=true +org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true +org.eclipse.jdt.core.formatter.wrap_before_relational_operator=true +org.eclipse.jdt.core.formatter.wrap_before_shift_operator=true +org.eclipse.jdt.core.formatter.wrap_before_string_concatenation=true +org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true +org.eclipse.jdt.core.javaFormatter=org.eclipse.jdt.core.defaultJavaFormatter diff --git a/exasol-jdbc/.settings/org.eclipse.jdt.ui.prefs b/exasol-jdbc/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 00000000..1add06a7 --- /dev/null +++ b/exasol-jdbc/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,205 @@ +cleanup.add_default_serial_version_id=true +cleanup.add_generated_serial_version_id=false +cleanup.add_missing_annotations=true +cleanup.add_missing_deprecated_annotations=true +cleanup.add_missing_methods=false +cleanup.add_missing_nls_tags=false +cleanup.add_missing_override_annotations=true +cleanup.add_missing_override_annotations_interface_methods=true +cleanup.add_serial_version_id=false +cleanup.always_use_blocks=true +cleanup.always_use_parentheses_in_expressions=false +cleanup.always_use_this_for_non_static_field_access=true +cleanup.always_use_this_for_non_static_method_access=false +cleanup.convert_functional_interfaces=true +cleanup.convert_to_enhanced_for_loop=true +cleanup.correct_indentation=true +cleanup.format_source_code=true +cleanup.format_source_code_changes_only=false +cleanup.insert_inferred_type_arguments=false +cleanup.make_local_variable_final=true +cleanup.make_parameters_final=true +cleanup.make_private_fields_final=true +cleanup.make_type_abstract_if_missing_method=false +cleanup.make_variable_declarations_final=true +cleanup.never_use_blocks=false +cleanup.never_use_parentheses_in_expressions=true +cleanup.organize_imports=false +cleanup.qualify_static_field_accesses_with_declaring_class=false +cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +cleanup.qualify_static_member_accesses_with_declaring_class=true +cleanup.qualify_static_method_accesses_with_declaring_class=false +cleanup.remove_private_constructors=true +cleanup.remove_redundant_modifiers=false +cleanup.remove_redundant_semicolons=true +cleanup.remove_redundant_type_arguments=true +cleanup.remove_trailing_whitespaces=true +cleanup.remove_trailing_whitespaces_all=true +cleanup.remove_trailing_whitespaces_ignore_empty=false +cleanup.remove_unnecessary_casts=true +cleanup.remove_unnecessary_nls_tags=true +cleanup.remove_unused_imports=true +cleanup.remove_unused_local_variables=false +cleanup.remove_unused_private_fields=true +cleanup.remove_unused_private_members=true +cleanup.remove_unused_private_methods=true +cleanup.remove_unused_private_types=true +cleanup.sort_members=false +cleanup.sort_members_all=false +cleanup.use_anonymous_class_creation=false +cleanup.use_blocks=true +cleanup.use_blocks_only_for_return_and_throw=false +cleanup.use_lambda=true +cleanup.use_parentheses_in_expressions=true +cleanup.use_this_for_non_static_field_access=true +cleanup.use_this_for_non_static_field_access_only_if_necessary=false +cleanup.use_this_for_non_static_method_access=false +cleanup.use_this_for_non_static_method_access_only_if_necessary=true +cleanup_profile=_Exasol +cleanup_settings_version=2 +eclipse.preferences.version=1 +editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true +formatter_profile=_Exasol +formatter_settings_version=21 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=java;javax;org;com; +org.eclipse.jdt.ui.ondemandthreshold=3 +org.eclipse.jdt.ui.staticondemandthreshold=3 +sp_cleanup.add_all=false +sp_cleanup.add_default_serial_version_id=true +sp_cleanup.add_generated_serial_version_id=false +sp_cleanup.add_missing_annotations=true +sp_cleanup.add_missing_deprecated_annotations=true +sp_cleanup.add_missing_methods=false +sp_cleanup.add_missing_nls_tags=false +sp_cleanup.add_missing_override_annotations=true +sp_cleanup.add_missing_override_annotations_interface_methods=true +sp_cleanup.add_serial_version_id=false +sp_cleanup.always_use_blocks=true +sp_cleanup.always_use_parentheses_in_expressions=true +sp_cleanup.always_use_this_for_non_static_field_access=true +sp_cleanup.always_use_this_for_non_static_method_access=false +sp_cleanup.array_with_curly=false +sp_cleanup.arrays_fill=false +sp_cleanup.bitwise_conditional_expression=false +sp_cleanup.boolean_literal=false +sp_cleanup.boolean_value_rather_than_comparison=false +sp_cleanup.break_loop=false +sp_cleanup.collection_cloning=false +sp_cleanup.comparing_on_criteria=false +sp_cleanup.comparison_statement=false +sp_cleanup.controlflow_merge=false +sp_cleanup.convert_functional_interfaces=true +sp_cleanup.convert_to_enhanced_for_loop=true +sp_cleanup.convert_to_enhanced_for_loop_if_loop_var_used=false +sp_cleanup.convert_to_switch_expressions=false +sp_cleanup.correct_indentation=true +sp_cleanup.do_while_rather_than_while=false +sp_cleanup.double_negation=false +sp_cleanup.else_if=false +sp_cleanup.embedded_if=false +sp_cleanup.evaluate_nullable=false +sp_cleanup.extract_increment=false +sp_cleanup.format_source_code=true +sp_cleanup.format_source_code_changes_only=false +sp_cleanup.hash=false +sp_cleanup.if_condition=false +sp_cleanup.insert_inferred_type_arguments=false +sp_cleanup.instanceof=false +sp_cleanup.instanceof_keyword=false +sp_cleanup.invert_equals=false +sp_cleanup.join=false +sp_cleanup.lazy_logical_operator=false +sp_cleanup.make_local_variable_final=true +sp_cleanup.make_parameters_final=true +sp_cleanup.make_private_fields_final=true +sp_cleanup.make_type_abstract_if_missing_method=false +sp_cleanup.make_variable_declarations_final=true +sp_cleanup.map_cloning=false +sp_cleanup.merge_conditional_blocks=false +sp_cleanup.multi_catch=false +sp_cleanup.never_use_blocks=false +sp_cleanup.never_use_parentheses_in_expressions=false +sp_cleanup.no_string_creation=false +sp_cleanup.no_super=false +sp_cleanup.number_suffix=false +sp_cleanup.objects_equals=false +sp_cleanup.on_save_use_additional_actions=true +sp_cleanup.one_if_rather_than_duplicate_blocks_that_fall_through=false +sp_cleanup.operand_factorization=false +sp_cleanup.organize_imports=true +sp_cleanup.overridden_assignment=false +sp_cleanup.plain_replacement=false +sp_cleanup.precompile_regex=false +sp_cleanup.primitive_comparison=false +sp_cleanup.primitive_parsing=false +sp_cleanup.primitive_rather_than_wrapper=false +sp_cleanup.primitive_serialization=false +sp_cleanup.pull_out_if_from_if_else=false +sp_cleanup.pull_up_assignment=false +sp_cleanup.push_down_negation=false +sp_cleanup.qualify_static_field_accesses_with_declaring_class=false +sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_with_declaring_class=true +sp_cleanup.qualify_static_method_accesses_with_declaring_class=false +sp_cleanup.reduce_indentation=false +sp_cleanup.redundant_comparator=false +sp_cleanup.redundant_falling_through_block_end=false +sp_cleanup.remove_private_constructors=true +sp_cleanup.remove_redundant_modifiers=false +sp_cleanup.remove_redundant_semicolons=true +sp_cleanup.remove_redundant_type_arguments=true +sp_cleanup.remove_trailing_whitespaces=true +sp_cleanup.remove_trailing_whitespaces_all=true +sp_cleanup.remove_trailing_whitespaces_ignore_empty=false +sp_cleanup.remove_unnecessary_array_creation=false +sp_cleanup.remove_unnecessary_casts=true +sp_cleanup.remove_unnecessary_nls_tags=true +sp_cleanup.remove_unused_imports=true +sp_cleanup.remove_unused_local_variables=false +sp_cleanup.remove_unused_private_fields=true +sp_cleanup.remove_unused_private_members=false +sp_cleanup.remove_unused_private_methods=true +sp_cleanup.remove_unused_private_types=true +sp_cleanup.return_expression=false +sp_cleanup.simplify_lambda_expression_and_method_ref=false +sp_cleanup.single_used_field=false +sp_cleanup.sort_members=false +sp_cleanup.sort_members_all=false +sp_cleanup.standard_comparison=false +sp_cleanup.static_inner_class=false +sp_cleanup.strictly_equal_or_different=false +sp_cleanup.stringbuffer_to_stringbuilder=false +sp_cleanup.stringbuilder=false +sp_cleanup.stringbuilder_for_local_vars=false +sp_cleanup.substring=false +sp_cleanup.switch=false +sp_cleanup.system_property=false +sp_cleanup.system_property_boolean=false +sp_cleanup.system_property_file_encoding=false +sp_cleanup.system_property_file_separator=false +sp_cleanup.system_property_line_separator=false +sp_cleanup.system_property_path_separator=false +sp_cleanup.ternary_operator=false +sp_cleanup.try_with_resource=false +sp_cleanup.unlooped_while=false +sp_cleanup.unreachable_block=false +sp_cleanup.use_anonymous_class_creation=false +sp_cleanup.use_autoboxing=false +sp_cleanup.use_blocks=true +sp_cleanup.use_blocks_only_for_return_and_throw=false +sp_cleanup.use_directly_map_method=false +sp_cleanup.use_lambda=true +sp_cleanup.use_parentheses_in_expressions=true +sp_cleanup.use_string_is_blank=false +sp_cleanup.use_this_for_non_static_field_access=true +sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=false +sp_cleanup.use_this_for_non_static_method_access=false +sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true +sp_cleanup.use_unboxing=false +sp_cleanup.use_var=false +sp_cleanup.useless_continue=false +sp_cleanup.useless_return=false +sp_cleanup.valueof_rather_than_instantiation=false diff --git a/exasol-jdbc/error_code_config.yml b/exasol-jdbc/error_code_config.yml new file mode 100644 index 00000000..8c2fda39 --- /dev/null +++ b/exasol-jdbc/error_code_config.yml @@ -0,0 +1,5 @@ +error-tags: + SEC: + packages: + - com.exasol.spark + highest-index: 26 diff --git a/exasol-jdbc/pk_generated_parent.pom b/exasol-jdbc/pk_generated_parent.pom new file mode 100644 index 00000000..960b10cf --- /dev/null +++ b/exasol-jdbc/pk_generated_parent.pom @@ -0,0 +1,262 @@ + + + 4.0.0 + com.exasol + spark-connector-jdbc-generated-parent + ${revision} + pom + + com.exasol + spark-connector-parent-pom + ${revision} + ../parent-pom/pom.xml + + + UTF-8 + UTF-8 + 11 + + + + + Apache License + https://github.com/exasol/spark-connector/blob/main/LICENSE + repo + + + + + Exasol + opensource@exasol.com + Exasol AG + https://www.exasol.com/ + + + + scm:git:https://github.com/exasol/spark-connector.git + scm:git:https://github.com/exasol/spark-connector.git + https://github.com/exasol/spark-connector/ + + + + + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.9.1.2184 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + + + enforce-maven + + enforce + + + + + [3.8.7,3.9.0) + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.4.1 + + true + oss + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.sonatype.ossindex.maven + ossindex-maven-plugin + 3.2.0 + + + audit + package + + audit + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + ${test.excludeTags} + + + + org.codehaus.mojo + versions-maven-plugin + 2.15.0 + + + display-updates + package + + display-plugin-updates + display-dependency-updates + + + + + file:///${project.basedir}/versionsMavenPluginRules.xml + + + + org.basepom.maven + duplicate-finder-maven-plugin + 1.5.1 + + + default + verify + + check + + + + + true + true + true + true + true + true + false + true + true + false + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + + ${test.excludeTags} + + + + verify + + integration-test + verify + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.9 + + + prepare-agent + + prepare-agent + + + + merge-results + verify + + merge + + + + + ${project.build.directory}/ + + jacoco*.exec + + + + ${project.build.directory}/aggregate.exec + + + + report + verify + + report + + + ${project.build.directory}/aggregate.exec + + + + + + com.exasol + error-code-crawler-maven-plugin + 1.2.3 + + + verify + + verify + + + + + + io.github.zlika + reproducible-build-maven-plugin + 0.16 + + + strip-jar + package + + strip-jar + + + + + + + diff --git a/exasol-jdbc/pom.xml b/exasol-jdbc/pom.xml new file mode 100644 index 00000000..f7031db2 --- /dev/null +++ b/exasol-jdbc/pom.xml @@ -0,0 +1,416 @@ + + + 4.0.0 + spark-connector-jdbc_${scala.compat.version} + ${revision}-spark-${spark.version} + jar + Spark Exasol Connector with JDBC + A connector for Apache Spark to access Exasol + https://github.com/exasol/spark-connector/ + + spark-connector-jdbc-generated-parent + com.exasol + ${revision} + pk_generated_parent.pom + + + + . + src/main/** + + + + com.exasol + exasol-jdbc + + + com.exasol + sql-statement-builder-java8 + + + com.exasol + error-reporting-java8 + + + com.exasol + spark-connector-common-java + + + org.apache.spark + spark-core_${scala.compat.version} + + + org.apache.spark + spark-sql_${scala.compat.version} + + + com.google.guava + guava + + + io.netty + netty-all + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.hadoop + hadoop-client + + + + org.glassfish.jersey.core + jersey-common + ${jersey.version} + provided + + + + org.glassfish.jersey.media + jersey-media-jaxb + ${jersey.version} + provided + + + + org.glassfish.jersey.core + jersey-server + ${jersey.version} + provided + + + + org.glassfish.jersey.core + jersey-client + ${jersey.version} + provided + + + + org.apache.avro + avro-mapred + 1.11.2 + provided + + + javax.servlet + * + + + org.eclipse.jetty + jetty-server + + + + org.eclipse.jetty + jetty-util + + + + org.xerial.snappy + snappy-java + + + + + + org.scalatest + scalatest_${scala.compat.version} + 3.2.9 + test + + + org.scalatestplus + scalatestplus-mockito_${scala.compat.version} + 1.0.0-M2 + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + org.apache.logging.log4j + log4j-api + test + + + org.apache.logging.log4j + log4j-1.2-api + test + + + org.apache.logging.log4j + log4j-core + test + + + com.exasol + test-db-builder-java + test + + + com.exasol + hamcrest-resultset-matcher + test + + + com.exasol + exasol-testcontainers + test + + + + + + net.alchim31.maven + scala-maven-plugin + 4.8.1 + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + attach-scaladocs + verify + + doc + doc-jar + + + + + ${scala.version} + ${scala.compat.version} + true + true + incremental + + -unchecked + -deprecation + -feature + -explaintypes + -Xcheckinit + -Xfatal-warnings + -Xlint:_ + -Ywarn-dead-code + -Ywarn-numeric-widen + -Ywarn-value-discard + -Ywarn-extra-implicit + -Ywarn-unused:_ + + + -source + ${java.version} + -target + ${java.version} + -deprecation + -parameters + -Xlint:all + + + -Xmx2048m + -Xss64m + + + + org.scalameta + semanticdb-scalac_${scala.version} + 4.8.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.5.0 + + ${java.version} + + + + org.scalatest + scalatest-maven-plugin + 2.2.0 + + . + TestSuite.txt + -Djava.util.logging.config.file=src/test/resources/logging.properties + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + ${argLine} + ${project.build.directory}/surefire-reports + + + + test + + test + + + (?<!IT) + + + + integration-test + integration-test + + test + + + (?<=IT) + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + default-jar + package + + jar + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.1 + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${project.artifactId}-${project.version}-assembly + false + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + reference.conf + + + + + + + + org.basepom.maven + duplicate-finder-maven-plugin + + false + false + false + + git.properties + arrow-git.properties + mime.types + .*\.proto$ + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.37.0 + + + + ${project.basedir}/.scalafmt.conf + + + + + + + check + + + + + + io.github.evis + scalafix-maven-plugin_${scala.compat.version} + 0.1.4_0.9.31 + + + com.geirsson + metaconfig-pprint_${scala.compat.version} + 0.11.1 + + + com.github.liancheng + organize-imports_${scala.compat.version} + 0.6.0 + + + com.github.vovapolu + scaluzzi_${scala.compat.version} + 0.1.23 + + + + CHECK + + + + + diff --git a/exasol-jdbc/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/exasol-jdbc/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..342ce441 --- /dev/null +++ b/exasol-jdbc/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1 @@ +com.exasol.spark.DefaultSource diff --git a/src/main/scala/com/exasol/spark/DefaultSource.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/DefaultSource.scala similarity index 76% rename from src/main/scala/com/exasol/spark/DefaultSource.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/DefaultSource.scala index 64a93b2f..214e575e 100644 --- a/src/main/scala/com/exasol/spark/DefaultSource.scala +++ b/exasol-jdbc/src/main/scala/com/exasol/spark/DefaultSource.scala @@ -3,10 +3,13 @@ package com.exasol.spark import org.apache.spark.sql._ import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.CaseInsensitiveStringMap import com.exasol.errorreporting.ExaError -import com.exasol.spark.util.ExasolConfiguration +import com.exasol.spark.common.ExasolOptions +import com.exasol.spark.util.Constants._ import com.exasol.spark.util.ExasolConnectionManager +import com.exasol.spark.util.ExasolOptionsProvider import com.exasol.spark.util.Types import com.exasol.spark.writer.ExasolWriter import com.exasol.sql.StatementFactory @@ -39,13 +42,10 @@ class DefaultSource * required for read * @return An [[ExasolRelation]] relation */ - override def createRelation( - sqlContext: SQLContext, - parameters: Map[String, String] - ): BaseRelation = { - val queryString = getKeyValue("query", parameters) - val manager = createManager(parameters, sqlContext) - new ExasolRelation(sqlContext, queryString, None, manager) + override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { + val options = createOptions(parameters, sqlContext) + val manager = ExasolConnectionManager(options) + new ExasolRelation(sqlContext, options.getQuery(), None, manager) } /** @@ -64,9 +64,9 @@ class DefaultSource parameters: Map[String, String], schema: StructType ): BaseRelation = { - val queryString = getKeyValue("query", parameters) - val manager = createManager(parameters, sqlContext) - new ExasolRelation(sqlContext, queryString, Option(schema), manager) + val options = createOptions(parameters, sqlContext) + val manager = ExasolConnectionManager(options) + new ExasolRelation(sqlContext, options.getQuery(), Option(schema), manager) } /** @@ -87,9 +87,10 @@ class DefaultSource parameters: Map[String, String], data: DataFrame ): BaseRelation = { - val tableName = getKeyValue("table", parameters) - val manager = createManager(parameters, sqlContext) - if (manager.config.drop_table) { + val options = createOptions(parameters, sqlContext) + val tableName = options.getTable() + val manager = ExasolConnectionManager(options) + if (options.hasEnabled(DROP_TABLE)) { manager.dropTable(tableName) } val isTableExist = manager.tableExists(tableName) @@ -97,16 +98,16 @@ class DefaultSource mode match { case SaveMode.Overwrite => if (!isTableExist) { - createExasolTable(data, tableName, manager) + createExasolTable(data, tableName, options, manager) } manager.truncateTable(tableName) - saveDataFrame(sqlContext, data, tableName, manager) + saveDataFrame(sqlContext, data, tableName, options, manager) case SaveMode.Append => if (!isTableExist) { - createExasolTable(data, tableName, manager) + createExasolTable(data, tableName, options, manager) } - saveDataFrame(sqlContext, data, tableName, manager) + saveDataFrame(sqlContext, data, tableName, options, manager) case SaveMode.ErrorIfExists => if (isTableExist) { @@ -123,13 +124,13 @@ class DefaultSource .toString() ) } - createExasolTable(data, tableName, manager) - saveDataFrame(sqlContext, data, tableName, manager) + createExasolTable(data, tableName, options, manager) + saveDataFrame(sqlContext, data, tableName, options, manager) case SaveMode.Ignore => if (!isTableExist) { - createExasolTable(data, tableName, manager) - saveDataFrame(sqlContext, data, tableName, manager) + createExasolTable(data, tableName, options, manager) + saveDataFrame(sqlContext, data, tableName, options, manager) } } @@ -150,9 +151,10 @@ class DefaultSource sqlContext: SQLContext, df: DataFrame, tableName: String, + options: ExasolOptions, manager: ExasolConnectionManager ): Unit = { - val writer = new ExasolWriter(sqlContext.sparkContext, tableName, df.schema, manager) + val writer = new ExasolWriter(sqlContext.sparkContext, tableName, df.schema, options, manager) val exaNodesCnt = writer.startParallel() val newDF = repartitionPerNode(df, exaNodesCnt) @@ -163,9 +165,10 @@ class DefaultSource private[this] def createExasolTable( df: DataFrame, tableName: String, + options: ExasolOptions, manager: ExasolConnectionManager ): Unit = - if (manager.config.create_table || manager.config.drop_table) { + if (options.hasEnabled(CREATE_TABLE) || options.hasEnabled(DROP_TABLE)) { manager.createTable(tableName, Types.createTableSchema(df.schema)) } else { throw new UnsupportedOperationException( @@ -211,26 +214,12 @@ class DefaultSource } } - private[this] def getKeyValue(key: String, parameters: Map[String, String]): String = - parameters.get(key) match { - case Some(str) => str - case None => - throw new UnsupportedOperationException( - ExaError - .messageBuilder("E-SEC-1") - .message("Parameter {{PARAMETER}} is missing.", key) - .mitigation("Please provide required parameter.") - .toString() - ) + private[this] def createOptions(parameters: Map[String, String], sqlContext: SQLContext): ExasolOptions = { + val hashMap = new java.util.HashMap[String, String]() + mergeConfigurations(parameters, sqlContext.getAllConfs).foreach { case (key, value) => + hashMap.put(key, value) } - - // Creates an ExasolConnectionManager with merged configuration values. - private[this] def createManager( - parameters: Map[String, String], - sqlContext: SQLContext - ): ExasolConnectionManager = { - val config = ExasolConfiguration(mergeConfigurations(parameters, sqlContext.getAllConfs)) - ExasolConnectionManager(config) + ExasolOptionsProvider(new CaseInsensitiveStringMap(hashMap)) } // Merges user provided parameters with `spark.exasol.*` runtime diff --git a/src/main/scala/com/exasol/spark/ExasolQueryEnricher.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/ExasolQueryEnricher.scala similarity index 100% rename from src/main/scala/com/exasol/spark/ExasolQueryEnricher.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/ExasolQueryEnricher.scala diff --git a/src/main/scala/com/exasol/spark/ExasolRelation.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/ExasolRelation.scala similarity index 100% rename from src/main/scala/com/exasol/spark/ExasolRelation.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/ExasolRelation.scala diff --git a/src/main/scala/com/exasol/spark/ExasolWriter.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/ExasolWriter.scala similarity index 93% rename from src/main/scala/com/exasol/spark/ExasolWriter.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/ExasolWriter.scala index 8702501c..b0d4ecf1 100644 --- a/src/main/scala/com/exasol/spark/ExasolWriter.scala +++ b/exasol-jdbc/src/main/scala/com/exasol/spark/ExasolWriter.scala @@ -11,6 +11,8 @@ import org.apache.spark.sql.types.StructType import com.exasol.errorreporting.ExaError import com.exasol.jdbc.EXAConnection +import com.exasol.spark.common.ExasolOptions +import com.exasol.spark.util.Constants._ import com.exasol.spark.util.Converter import com.exasol.spark.util.ExasolConnectionManager import com.exasol.spark.util.Types @@ -21,6 +23,7 @@ class ExasolWriter( @transient val sc: SparkContext, tableName: String, rddSchema: StructType, + options: ExasolOptions, manager: ExasolConnectionManager ) extends Serializable { @@ -77,7 +80,7 @@ class ExasolWriter( val nullTypes = rddSchema.fields.map(f => Types.jdbcTypeFromSparkDataType(f.dataType)) val fieldCnt = rddSchema.fields.length - val batchSize = manager.config.batch_size + val batchSize = if (options.containsKey(BATCH_SIZE)) options.get(BATCH_SIZE).toInt else DEFAULT_BATCH_SIZE try { var rowCnt = 0 diff --git a/src/main/scala/com/exasol/spark/rdd/ExasolRDD.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/rdd/ExasolRDD.scala similarity index 100% rename from src/main/scala/com/exasol/spark/rdd/ExasolRDD.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/rdd/ExasolRDD.scala diff --git a/src/main/scala/com/exasol/spark/rdd/ExasolRDDPartition.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/rdd/ExasolRDDPartition.scala similarity index 100% rename from src/main/scala/com/exasol/spark/rdd/ExasolRDDPartition.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/rdd/ExasolRDDPartition.scala diff --git a/exasol-jdbc/src/main/scala/com/exasol/spark/util/Constants.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/Constants.scala new file mode 100644 index 00000000..b96f951b --- /dev/null +++ b/exasol-jdbc/src/main/scala/com/exasol/spark/util/Constants.scala @@ -0,0 +1,25 @@ +package com.exasol.spark.util + +/** + * An object for providing contants. + */ +object Constants { + + /** Boolean {@code create_table} parameter for creating a table if it does not exist. */ + val CREATE_TABLE = "create_table" + + /** Boolean {@code drop_table} parameter for dropping a table if it exists. */ + val DROP_TABLE = "drop_table" + + /** Integer {@code batch_size} parameter for batching write queries. */ + val BATCH_SIZE = "batch_size" + + /** Default value for {@code batch_size} parameter. */ + val DEFAULT_BATCH_SIZE = 1000 + + /** Integer {@code max_nodes} parameter for connecting to Exasol data nodes. */ + val MAX_NODES = "max_nodes" + + /** Default value for {@code max_nodes} parameter. */ + val DEFAULT_MAX_NODES = 200 +} diff --git a/src/main/scala/com/exasol/spark/util/Converter.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/Converter.scala similarity index 100% rename from src/main/scala/com/exasol/spark/util/Converter.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/util/Converter.scala diff --git a/src/main/scala/com/exasol/spark/util/ExasolConfiguration.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolConfiguration.scala similarity index 100% rename from src/main/scala/com/exasol/spark/util/ExasolConfiguration.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolConfiguration.scala diff --git a/src/main/scala/com/exasol/spark/util/ExasolConnectionManager.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolConnectionManager.scala similarity index 85% rename from src/main/scala/com/exasol/spark/util/ExasolConnectionManager.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolConnectionManager.scala index 07a8e055..8ca3ad4d 100644 --- a/src/main/scala/com/exasol/spark/util/ExasolConnectionManager.scala +++ b/exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolConnectionManager.scala @@ -11,6 +11,9 @@ import com.exasol.errorreporting.ExaError import com.exasol.jdbc.EXAConnection import com.exasol.jdbc.EXAResultSet import com.exasol.jdbc.EXAStatement +import com.exasol.spark.common.ExasolOptions +import com.exasol.spark.common.Option +import com.exasol.spark.util.Constants._ /** * A class that provides and manages Exasol connections. @@ -21,35 +24,28 @@ import com.exasol.jdbc.EXAStatement * @param config An [[ExasolConfiguration]] with user provided or runtime * configuration parameters */ -final case class ExasolConnectionManager(config: ExasolConfiguration) { +final case class ExasolConnectionManager(options: ExasolOptions) { - private[this] val DO_NOT_VALIDATE_CERTIFICATE = "validateservercertificate=0" - private[this] val MAIN_CONNECTION_PREFIX = "jdbc:exa" private[this] val WORKER_CONNECTION_PREFIX = "jdbc:exa-worker" + private[this] val USERNAME = options.getUsername() + private[this] val PASSWORD = options.getPassword() /** A regular Exasol jdbc connection string */ - def getJdbcConnectionString(): String = { - val host = getHostWithFingerprint(config.host) - val url = s"$MAIN_CONNECTION_PREFIX:$host:${config.port}" - getConnectionStringWithOptions(url) - } + def getJdbcConnectionString(): String = + options.getJdbcUrl() private[this] def getHostWithFingerprint(host: String): String = - if (!config.fingerprint.isEmpty() && !config.jdbc_options.contains(DO_NOT_VALIDATE_CERTIFICATE)) { - host + "/" + config.fingerprint + if (options.hasFingerprint()) { + host + "/" + options.getFingerprint() } else { host } def mainConnection(): EXAConnection = - ExasolConnectionManager.makeConnection(getJdbcConnectionString(), config.username, config.password) + ExasolConnectionManager.makeConnection(getJdbcConnectionString(), USERNAME, PASSWORD) def writerMainConnection(): EXAConnection = - ExasolConnectionManager.makeConnection( - s"${getJdbcConnectionString()};autocommit=0", - config.username, - config.password - ) + ExasolConnectionManager.makeConnection(s"${getJdbcConnectionString()};autocommit=0", USERNAME, PASSWORD) /** * A single non-pooled [[com.exasol.jdbc.EXAConnection]] connection. @@ -58,7 +54,7 @@ final case class ExasolConnectionManager(config: ExasolConfiguration) { * the user. */ def getConnection(): EXAConnection = - ExasolConnectionManager.createConnection(getJdbcConnectionString(), config.username, config.password) + ExasolConnectionManager.createConnection(getJdbcConnectionString(), USERNAME, PASSWORD) /** * Starts a parallel sub-connections from the main JDBC connection. @@ -66,8 +62,10 @@ final case class ExasolConnectionManager(config: ExasolConfiguration) { * @param mainConnection the main connection * @return the number of parallel connections */ - def initParallel(mainConnection: EXAConnection): Int = - mainConnection.EnterParallel(config.max_nodes) + def initParallel(mainConnection: EXAConnection): Int = { + val max_nodes = if (options.containsKey(MAX_NODES)) options.get(MAX_NODES).toInt else DEFAULT_MAX_NODES + mainConnection.EnterParallel(max_nodes) + } /** * Returns the list of all parallel sub-connection URLs. @@ -98,7 +96,7 @@ final case class ExasolConnectionManager(config: ExasolConfiguration) { * @return a JDBC connection on the separate parallel connection */ def subConnection(subConnectionUrl: String): EXAConnection = - ExasolConnectionManager.makeConnection(subConnectionUrl, config.username, config.password) + ExasolConnectionManager.makeConnection(subConnectionUrl, USERNAME, PASSWORD) /** * A method to run with a new connection. @@ -172,9 +170,6 @@ final case class ExasolConnectionManager(config: ExasolConfiguration) { /** * Checks if table already exists, if so should return true otherwise false. * - * TODO: This should be changed to Exasol specific checks. For example, by - * using EXA_USER_TABLES. - * * @param tableName A Exasol table name including schema, e.g. * `schema.tableName` * @return `true` if table exists, otherwise return `false` @@ -222,14 +217,12 @@ final case class ExasolConnectionManager(config: ExasolConfiguration) { () } - private[this] def getConnectionStringWithOptions(url: String): String = { - val jdbcOptions = config.jdbc_options - if (jdbcOptions == null || jdbcOptions.isEmpty) { + private[this] def getConnectionStringWithOptions(url: String): String = + if (!options.containsKey(Option.JDBC_OPTIONS.key())) { url } else { - s"$url;$jdbcOptions" + s"$url;${options.get(Option.JDBC_OPTIONS.key())}" } - } } diff --git a/exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolOptionsProvider.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolOptionsProvider.scala new file mode 100644 index 00000000..14975c5a --- /dev/null +++ b/exasol-jdbc/src/main/scala/com/exasol/spark/util/ExasolOptionsProvider.scala @@ -0,0 +1,129 @@ +package com.exasol.spark.util + +import java.net.InetAddress + +import scala.util.matching.Regex + +import com.exasol.errorreporting.ExaError +import com.exasol.spark.common.ExasolOptions +import com.exasol.spark.common.Option + +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +/** + * A companion object that creates {@link ExasolOptions}. + * + * It creates the configuration parameters for Spark Exasol connector. + * + * These can be user provided when loading or defined in Spark configurations. + * For example, user provided: + * + * {{{ + * df = sparkSession + * .read + * .format("exasol") + * .option("host", "127.0.0.1") + * .option("port", "8888") + * // ... + * .load() + * }}} + * + * From Spark configuration: + * + * {{{ + * val sparkConf = new SparkConf() + * .setMaster("local[*]") + * .setAppName("spark-exasol-connector") + * .set("spark.exasol.host", "localhost") + * .set("spark.exasol.port", "1234") + * // ... + * + * val sparkSession = SparkSession + * .builder() + * .config(sparkConf) + * .getOrCreate() + * }}} + * + * If both are defined, spark configs are used. If nothing is defined, then + * default values are used. + */ +object ExasolOptionsProvider { + + private[this] val IPv4_DIGITS: String = "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" + private[this] val IPv4_REGEX: Regex = raw"""^$IPv4_DIGITS\.$IPv4_DIGITS\.$IPv4_DIGITS\.$IPv4_DIGITS$$""".r + + /** + * Returns {@link ExasolConfiguration} from key-value options. + * + * @param map key value map + * @return an instance of {@link ExasolConfiguration} + */ + def apply(map: Map[String, String]): ExasolOptions = { + val javaMap = new java.util.HashMap[String, String]() + map.foreach { case (key, value) => javaMap.put(key, value) } + ExasolOptionsProvider(new CaseInsensitiveStringMap(javaMap)) + } + + /** + * Returns {@link ExasolConfiguration} from key-value options. + * + * It also validates key-value parameters. + * + * @param map key value case-insensitive map + * @return an instance of {@link ExasolConfiguration} + */ + def apply(map: CaseInsensitiveStringMap): ExasolOptions = { + val host = map.getOrDefault(Option.HOST.key(), getLocalHost()) + val jdbc_options = map.getOrDefault(Option.JDBC_OPTIONS.key(), "") + checkHost(host) + checkJdbcOptions(jdbc_options) + ExasolOptions.from(map) + } + + private[this] def getLocalHost(): String = InetAddress.getLocalHost().getHostAddress() + + private[util] def checkHost(host: String): String = host match { + case IPv4_REGEX(_*) => host + case _ => + throw new IllegalArgumentException( + ExaError + .messageBuilder("E-SEC-4") + .message("The host value is not an IPv4 address.") + .mitigation("The host value should be an IPv4 address of the first Exasol datanode.") + .toString() + ) + } + + private[this] def checkJdbcOptions(options: String): Unit = { + checkStartsOrEndsWith(options, ";") + if (!options.isEmpty()) { + val keyValuePairs = options.split(";") + checkContainsKeyValuePairs(keyValuePairs, "=") + } + } + + private[this] def checkStartsOrEndsWith(input: String, pattern: String): Unit = + if (input.endsWith(pattern) || input.startsWith(pattern)) { + throw new IllegalArgumentException( + ExaError + .messageBuilder("E-SEC-5") + .message("JDBC options should not start or end with semicolon.") + .mitigation("Please remove from beginning or end of JDBC options.") + .toString() + ) + } + + private[this] def checkContainsKeyValuePairs(options: Array[String], pattern: String): Unit = + options.foreach { case keyValue => + if (keyValue.split(pattern).length != 2) { + throw new IllegalArgumentException( + ExaError + .messageBuilder("E-SEC-6") + .message("Parameter {{PARAMETER}} does not have key=value format.", keyValue) + .mitigation("Please make sure parameters are encoded as key=value pairs.") + .toString() + ) + } + } + +} diff --git a/src/main/scala/com/exasol/spark/util/Filters.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/Filters.scala similarity index 100% rename from src/main/scala/com/exasol/spark/util/Filters.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/util/Filters.scala diff --git a/src/main/scala/com/exasol/spark/util/Types.scala b/exasol-jdbc/src/main/scala/com/exasol/spark/util/Types.scala similarity index 100% rename from src/main/scala/com/exasol/spark/util/Types.scala rename to exasol-jdbc/src/main/scala/com/exasol/spark/util/Types.scala diff --git a/exasol-jdbc/src/test/java/com/exasol/spark/SparkSessionProvider.java b/exasol-jdbc/src/test/java/com/exasol/spark/SparkSessionProvider.java new file mode 100644 index 00000000..0c66632b --- /dev/null +++ b/exasol-jdbc/src/test/java/com/exasol/spark/SparkSessionProvider.java @@ -0,0 +1,41 @@ +package com.exasol.spark; + +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; + +/** + * A class that provides Spark session for tests. + */ +public final class SparkSessionProvider { + private static volatile SparkSession sparkSession; + + private SparkSessionProvider() { + // Intentionally private + } + + public static SparkSession getSparkSession(final SparkConf sparkConf) { + final SparkSession result = sparkSession; + if (result != null && !isSparkContextStopped()) { + return result; + } + synchronized (SparkSessionProvider.class) { + if (sparkSession == null || isSparkContextStopped()) { + sparkSession = createSparkSession(sparkConf); + } + return sparkSession; + } + } + + private static boolean isSparkContextStopped() { + return sparkSession.sparkContext().isStopped(); + } + + private static SparkSession createSparkSession(final SparkConf sparkConf) { + SparkSession.Builder sparkSessionBuilder = SparkSession.builder(); + if (sparkConf != null) { + sparkSessionBuilder = sparkSessionBuilder.config(sparkConf); + } + return sparkSessionBuilder.getOrCreate(); + } + +} diff --git a/src/test/java/org/apache/log4j/MDC.java b/exasol-jdbc/src/test/java/org/apache/log4j/MDC.java similarity index 100% rename from src/test/java/org/apache/log4j/MDC.java rename to exasol-jdbc/src/test/java/org/apache/log4j/MDC.java diff --git a/src/test/java/org/apache/log4j/README.md b/exasol-jdbc/src/test/java/org/apache/log4j/README.md similarity index 100% rename from src/test/java/org/apache/log4j/README.md rename to exasol-jdbc/src/test/java/org/apache/log4j/README.md diff --git a/src/test/java/org/apache/log4j/helpers/ThreadLocalMap.java b/exasol-jdbc/src/test/java/org/apache/log4j/helpers/ThreadLocalMap.java similarity index 100% rename from src/test/java/org/apache/log4j/helpers/ThreadLocalMap.java rename to exasol-jdbc/src/test/java/org/apache/log4j/helpers/ThreadLocalMap.java diff --git a/exasol-jdbc/src/test/resources/logging.properties b/exasol-jdbc/src/test/resources/logging.properties new file mode 100644 index 00000000..8c97abe9 --- /dev/null +++ b/exasol-jdbc/src/test/resources/logging.properties @@ -0,0 +1,6 @@ +handlers=java.util.logging.ConsoleHandler +.level=INFO +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%1$tF %1$tT.%1$tL [%4$-7s] %5$s %n +com.exasol.level=ALL diff --git a/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/exasol-jdbc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker similarity index 100% rename from src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker rename to exasol-jdbc/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/src/test/scala/com/exasol/spark/DefaultSourceSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/DefaultSourceSuite.scala similarity index 84% rename from src/test/scala/com/exasol/spark/DefaultSourceSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/DefaultSourceSuite.scala index 77138e39..b8db8014 100644 --- a/src/test/scala/com/exasol/spark/DefaultSourceSuite.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/DefaultSourceSuite.scala @@ -6,6 +6,8 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.SQLContext import org.apache.spark.sql.SaveMode +import com.exasol.spark.common.ExasolValidationException + import org.mockito.Mockito.when import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -15,10 +17,11 @@ class DefaultSourceSuite extends AnyFunSuite with Matchers with MockitoSugar { test("when reading should throw an Exception if no `query` parameter is provided") { val sqlContext = mock[SQLContext] - val thrown = intercept[UnsupportedOperationException] { + when(sqlContext.getAllConfs).thenReturn(Map.empty[String, String]) + val thrown = intercept[ExasolValidationException] { new DefaultSource().createRelation(sqlContext, Map[String, String]()) } - assertErrors(thrown.getMessage(), "E-SEC-1", "Parameter 'query' is missing") + assert(thrown.getMessage().startsWith("E-SCCJ-10")) } test("throws an Exception if host parameter is not an ip address") { @@ -28,22 +31,18 @@ class DefaultSourceSuite extends AnyFunSuite with Matchers with MockitoSugar { val thrown = intercept[IllegalArgumentException] { new DefaultSource().createRelation(sqlContext, parameters) } - assertErrors(thrown.getMessage(), "E-SEC-4", "host value should be an IPv4 address of the first Exasol datanode") + assert(thrown.getMessage().startsWith("E-SEC-4")) + assert(thrown.getMessage().contains("host value should be an IPv4 address of the first Exasol datanode")) } test("when saving should throw an Exception if no `table` parameter is provided") { val df = mock[DataFrame] val sqlContext = mock[SQLContext] - val thrown = intercept[UnsupportedOperationException] { + when(sqlContext.getAllConfs).thenReturn(Map.empty[String, String]) + val thrown = intercept[ExasolValidationException] { new DefaultSource().createRelation(sqlContext, SaveMode.Append, Map[String, String](), df) } - assertErrors(thrown.getMessage(), "E-SEC-1", "Parameter 'table' is missing") - } - - private[this] def assertErrors(message: String, errorCode: String, containedString: String): Unit = { - assert(message.startsWith(errorCode)) - assert(message.contains(containedString)) - () + assert(thrown.getMessage().startsWith("E-SCCJ-10")) } test("`repartitionPerNode` should reduce dataframe partitions number") { diff --git a/src/test/scala/com/exasol/spark/ExasolQueryEnricherSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/ExasolQueryEnricherSuite.scala similarity index 100% rename from src/test/scala/com/exasol/spark/ExasolQueryEnricherSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/ExasolQueryEnricherSuite.scala diff --git a/src/test/scala/com/exasol/spark/ExasolRelationSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/ExasolRelationSuite.scala similarity index 100% rename from src/test/scala/com/exasol/spark/ExasolRelationSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/ExasolRelationSuite.scala diff --git a/src/test/scala/com/exasol/spark/SparkSessionSetup.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/SparkSessionSetup.scala similarity index 61% rename from src/test/scala/com/exasol/spark/SparkSessionSetup.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/SparkSessionSetup.scala index b854d2cb..1f33faf6 100644 --- a/src/test/scala/com/exasol/spark/SparkSessionSetup.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/SparkSessionSetup.scala @@ -13,19 +13,18 @@ import org.scalatest.Suite * A trait that provides Spark session setup across tests. */ trait SparkSessionSetup extends BeforeAndAfterAll { self: Suite => - @transient lazy val spark: SparkSession = SparkSessionProvider.getSparkSession() - @transient lazy val sqlContext: SQLContext = SparkSessionProvider.getSQLContext() + @transient lazy val spark: SparkSession = SparkSessionProvider.getSparkSession(getSparkConf()) + @transient lazy val sqlContext: SQLContext = SparkSession.builder().getOrCreate().sqlContext @transient private var sparkContext: SparkContext = _ override def beforeAll(): Unit = { super.beforeAll() setupSparkContext() - setupSparkSession() } override def afterAll(): Unit = { stopSparkContext() - SparkSessionProvider.setSparkSession(null) + spark.stop() super.afterAll() } @@ -39,23 +38,12 @@ trait SparkSessionSetup extends BeforeAndAfterAll { self: Suite => sparkContext = SparkContext.getOrCreate(getSparkConf()) } - private[this] def setupSparkSession(): Unit = - if (SparkSessionProvider.getSparkSession() != null && !isSparkContextStopped()) { - // do nothing - } else { - val builder = SparkSession.builder() - SparkSessionProvider.setSparkSession(builder.getOrCreate()) - } - private[this] def stopSparkContext(): Unit = if (sparkContext != null) { sparkContext.stop() sparkContext = null } - private[this] def isSparkContextStopped(): Boolean = - SparkSessionProvider.getSparkSession().sparkContext.isStopped - private[this] def getSparkConf(): SparkConf = new SparkConf() .setMaster("local[*]") @@ -68,19 +56,3 @@ trait SparkSessionSetup extends BeforeAndAfterAll { self: Suite => this.getClass().getName() + math.floor(math.random() * 1000).toLong.toString() } - -/** - * An object that manages transient Spark session. - */ -object SparkSessionProvider { - @transient var sparkSession: SparkSession = _ - - def getSparkSession(): SparkSession = sparkSession - - def setSparkSession(session: SparkSession): Unit = - sparkSession = session - - def getSQLContext(): SQLContext = - SparkSession.builder().getOrCreate().sqlContext - -} diff --git a/src/test/scala/com/exasol/spark/it/AbstractTableQueryIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/AbstractTableQueryIT.scala similarity index 58% rename from src/test/scala/com/exasol/spark/it/AbstractTableQueryIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/AbstractTableQueryIT.scala index 5ff93f46..c5509bc1 100644 --- a/src/test/scala/com/exasol/spark/it/AbstractTableQueryIT.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/it/AbstractTableQueryIT.scala @@ -3,30 +3,28 @@ package com.exasol.spark import org.apache.spark.sql.DataFrame import org.apache.spark.sql.DataFrameReader -import org.scalatest.BeforeAndAfterEach +import com.exasol.spark.util.ExasolConnectionManager -abstract class AbstractTableQueryIT extends BaseIntegrationTest with SparkSessionSetup with BeforeAndAfterEach { +import org.scalatest.BeforeAndAfterAll + +abstract class AbstractTableQueryIT extends BaseIntegrationTest with BeforeAndAfterAll with SparkSessionSetup { val tableName: String def createTable(): Unit - override def beforeEach(): Unit = + override def beforeAll(): Unit = { + super.beforeAll() + val options = getExasolOptions(getDefaultOptions() ++ Map("table" -> tableName)) + exasolConnectionManager = ExasolConnectionManager(options) createTable() + } - private[spark] def getDataFrameReader(query: String): DataFrameReader = { - val reader = spark.read + private[spark] def getDataFrameReader(query: String): DataFrameReader = + spark.read .format("exasol") - .option("host", jdbcHost) - .option("port", jdbcPort) + .options(getDefaultOptions()) .option("query", query) - if (imageSupportsFingerprint()) { - reader.option("fingerprint", getFingerprint()) - } else { - reader.option("jdbc_options", "validateservercertificate=0") - } - } - private[spark] def getDataFrame(queryOpt: Option[String] = None): DataFrame = { val query = queryOpt.fold(s"SELECT * FROM $tableName")(identity) getDataFrameReader(query).load() diff --git a/src/test/scala/com/exasol/spark/it/BaseIntegrationTest.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/BaseIntegrationTest.scala similarity index 53% rename from src/test/scala/com/exasol/spark/it/BaseIntegrationTest.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/BaseIntegrationTest.scala index ea78ef85..bd217266 100644 --- a/src/test/scala/com/exasol/spark/it/BaseIntegrationTest.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/it/BaseIntegrationTest.scala @@ -1,8 +1,9 @@ package com.exasol.spark import com.exasol.containers.ExasolContainer -import com.exasol.spark.util.ExasolConfiguration +import com.exasol.spark.common.ExasolOptions import com.exasol.spark.util.ExasolConnectionManager +import com.exasol.spark.util.ExasolOptionsProvider import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -12,7 +13,7 @@ import org.scalatest.funsuite.AnyFunSuite */ trait BaseIntegrationTest extends AnyFunSuite with BeforeAndAfterAll { - private[this] val DEFAULT_EXASOL_DOCKER_IMAGE = "7.1.18" + private[this] val DEFAULT_EXASOL_DOCKER_IMAGE = "8.18.1" val network = DockerNamedNetwork("spark-it-network", true) val container = { @@ -21,47 +22,36 @@ trait BaseIntegrationTest extends AnyFunSuite with BeforeAndAfterAll { c } - var jdbcHost: String = _ - var jdbcPort: String = _ var exasolConnectionManager: ExasolConnectionManager = _ - def prepareExasolDatabase(): Unit = { + override def beforeAll(): Unit = container.start() - jdbcHost = container.getDockerNetworkInternalIpAddress() - jdbcPort = s"${container.getDefaultInternalDatabasePort()}" - exasolConnectionManager = ExasolConnectionManager(ExasolConfiguration(getConfiguration())) + + override def afterAll(): Unit = { + container.stop() + network.close() } - def getConfiguration(): Map[String, String] = { - val defaultOptions = Map( - "host" -> jdbcHost, - "port" -> jdbcPort, + def getDefaultOptions(): Map[String, String] = { + val options = Map( + "host" -> container.getDockerNetworkInternalIpAddress(), + "port" -> s"${container.getDefaultInternalDatabasePort()}", "username" -> container.getUsername(), "password" -> container.getPassword(), "max_nodes" -> "200" ) - if (imageSupportsFingerprint()) { - defaultOptions ++ Map("fingerprint" -> getFingerprint()) + if (getFingerprint().isPresent()) { + options ++ Map("fingerprint" -> getFingerprint().get()) } else { - defaultOptions ++ Map("jdbc_options" -> "validateservercertificate=0") + options } } - def getFingerprint(): String = - container.getTlsCertificateFingerprint().get() - - def imageSupportsFingerprint(): Boolean = { - val image = container.getDockerImageReference() - (image.getMajor() >= 7) && (image.getMinor() >= 1) - } + def getExasolOptions(map: Map[String, String]): ExasolOptions = + ExasolOptionsProvider(map) - override def beforeAll(): Unit = - prepareExasolDatabase() - - override def afterAll(): Unit = { - container.stop() - network.close() - } + def getFingerprint(): java.util.Optional[String] = + container.getTlsCertificateFingerprint() private[this] def getExasolDockerImageVersion(): String = { val dockerVersion = System.getenv("EXASOL_DOCKER_VERSION") diff --git a/src/test/scala/com/exasol/spark/it/BaseTableQueryIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/BaseTableQueryIT.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/BaseTableQueryIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/BaseTableQueryIT.scala diff --git a/src/test/scala/com/exasol/spark/it/ColumnPruningIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/ColumnPruningIT.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/ColumnPruningIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/ColumnPruningIT.scala diff --git a/src/test/scala/com/exasol/spark/it/DockerNamedNetwork.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/DockerNamedNetwork.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/DockerNamedNetwork.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/DockerNamedNetwork.scala diff --git a/src/test/scala/com/exasol/spark/it/ExasolDockerContainerWithReuse.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/ExasolDockerContainerWithReuse.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/ExasolDockerContainerWithReuse.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/ExasolDockerContainerWithReuse.scala diff --git a/src/test/scala/com/exasol/spark/it/LoadIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/LoadIT.scala similarity index 80% rename from src/test/scala/com/exasol/spark/it/LoadIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/LoadIT.scala index fe10f194..a942bc6a 100644 --- a/src/test/scala/com/exasol/spark/it/LoadIT.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/it/LoadIT.scala @@ -4,6 +4,8 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.types._ +import com.exasol.spark.common.ExasolValidationException + /** * Tests for loading data from Exasol query as dataframes using short * and long source formats. @@ -45,15 +47,14 @@ class LoadIT extends BaseTableQueryIT { } test("throws if query parameter is not provided") { - val thrown = intercept[UnsupportedOperationException] { + val thrown = intercept[ExasolValidationException] { spark.read .format("com.exasol.spark") - .option("host", jdbcHost) - .option("port", jdbcPort) + .option("host", container.getDockerNetworkInternalIpAddress()) + .option("port", s"${container.getDefaultInternalDatabasePort()}") .load() } - assert(thrown.getMessage().startsWith("E-SEC-1")) - assert(thrown.getMessage().contains("Parameter 'query' is missing.")) + assert(thrown.getMessage().startsWith("E-SCCJ-10")) } test("returns columns from user provided schema") { @@ -81,20 +82,19 @@ class LoadIT extends BaseTableQueryIT { val thrown = intercept[java.sql.SQLException] { df.show(10, false) } - assert(thrown.getMessage().contains("""object "DATE_INFORMATION" not found""")) + assert(thrown.getMessage().contains("""object DATE_INFORMATION not found""")) } test("uses user provided SparkConf") { - var sparkConf = new SparkConf() + val sparkConf = new SparkConf() .setMaster("local[*]") - .set("spark.exasol.host", jdbcHost) - .set("spark.exasol.port", jdbcPort) + .set("spark.exasol.host", container.getDockerNetworkInternalIpAddress()) + .set("spark.exasol.port", s"${container.getDefaultInternalDatabasePort()}") .set("spark.exasol.max_nodes", "20") - if (imageSupportsFingerprint()) { - sparkConf = sparkConf.set("spark.exasol.fingerprint", getFingerprint()) - } else { - sparkConf = sparkConf.set("spark.exasol.jdbc_options", "validateservercertificate=0") + val fingerprintOpt = getFingerprint() + if (fingerprintOpt.isPresent()) { + sparkConf.set("spark.exasol.fingerprint", fingerprintOpt.get()) } val sparkSession = SparkSession diff --git a/src/test/scala/com/exasol/spark/it/PredicatePushdownIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/PredicatePushdownIT.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/PredicatePushdownIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/PredicatePushdownIT.scala diff --git a/src/test/scala/com/exasol/spark/it/QuotedColumnsIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/QuotedColumnsIT.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/QuotedColumnsIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/QuotedColumnsIT.scala diff --git a/src/test/scala/com/exasol/spark/it/ReservedKeywordsIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/ReservedKeywordsIT.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/ReservedKeywordsIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/ReservedKeywordsIT.scala diff --git a/src/test/scala/com/exasol/spark/it/SaveOptionsIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/SaveOptionsIT.scala similarity index 88% rename from src/test/scala/com/exasol/spark/it/SaveOptionsIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/SaveOptionsIT.scala index aeb036d3..47b37d46 100644 --- a/src/test/scala/com/exasol/spark/it/SaveOptionsIT.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/it/SaveOptionsIT.scala @@ -4,18 +4,18 @@ import java.sql.Date import com.exasol.spark.util.Types +import org.scalatest.BeforeAndAfterEach + /** * Integration tests for saving Spark DataFrames into Exasol tables. */ -class SaveOptionsIT extends BaseTableQueryIT { +class SaveOptionsIT extends BaseTableQueryIT with BeforeAndAfterEach { private[this] val saveModes = Seq("append", "errorifexists", "ignore", "overwrite") - private[this] var defaultOptions: Map[String, String] = _ - override def beforeEach(): Unit = { - super.beforeEach() - defaultOptions = getConfiguration() ++ Map("table" -> tableName) - } + // Required for save mode tests, since initial table can be deleted on other tests + override def beforeEach(): Unit = + createTable() private[this] val dataframeTestData: Seq[(String, String, Date, String)] = Seq( ("name1", "city1", Date.valueOf("2019-01-11"), "äpişge"), @@ -96,7 +96,7 @@ class SaveOptionsIT extends BaseTableQueryIT { } test("save with 'create_table' option creates a new table before saving dataframe") { - val newOptions = defaultOptions ++ Map("create_table" -> "true") + val newOptions = getOptions() ++ Map("create_table" -> "true") saveModes.foreach { case mode => exasolConnectionManager.dropTable(tableName) assert(runDataFrameSave(mode, 2, newOptions) === dataframeTestData.size.toLong) @@ -104,7 +104,7 @@ class SaveOptionsIT extends BaseTableQueryIT { } test("save with 'drop_table' option drops and creates a new table before saving dataframe") { - val newOptions = defaultOptions ++ Map("drop_table" -> "true") + val newOptions = getOptions() ++ Map("drop_table" -> "true") saveModes.foreach { case mode => createTable() assert(runDataFrameSave(mode, 3, newOptions) === dataframeTestData.size) @@ -114,7 +114,7 @@ class SaveOptionsIT extends BaseTableQueryIT { private[this] def runDataFrameSave( mode: String, partitionCount: Int, - options: Map[String, String] = defaultOptions + options: Map[String, String] = getOptions() ): Long = { import sqlContext.implicits._ val df = getSparkContext() @@ -130,4 +130,7 @@ class SaveOptionsIT extends BaseTableQueryIT { exasolConnectionManager.withCountQuery(s"SELECT COUNT(*) FROM $tableName") } + // Lazily obtain options after the container is initialized + private[this] def getOptions(): Map[String, String] = getDefaultOptions() ++ Map("table" -> tableName) + } diff --git a/src/test/scala/com/exasol/spark/it/SparkDataImportIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/SparkDataImportIT.scala similarity index 89% rename from src/test/scala/com/exasol/spark/it/SparkDataImportIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/SparkDataImportIT.scala index 6fe9b9e3..7cb0e85a 100644 --- a/src/test/scala/com/exasol/spark/it/SparkDataImportIT.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/it/SparkDataImportIT.scala @@ -8,19 +8,30 @@ import java.sql.Timestamp import org.apache.spark.sql.Encoder +import com.exasol.spark.util.ExasolConnectionManager import com.exasol.matcher.ResultSetStructureMatcher.table import com.exasol.matcher.TypeMatchMode._ import org.hamcrest.Matcher import org.hamcrest.MatcherAssert.assertThat +import org.scalatest.BeforeAndAfterAll -class SparkDataImportIT extends BaseTableQueryIT { +class SparkDataImportIT extends BaseTableQueryIT with BeforeAndAfterAll { private[this] val INT_MIN = -2147483648 private[this] val INT_MAX = 2147483647 private[this] val LONG_MIN = -9223372036854775808L private[this] val LONG_MAX = 9223372036854775807L + private[this] var defaultOptions: Map[String, String] = _ + + override def beforeAll(): Unit = { + super.beforeAll() + defaultOptions = getDefaultOptions() ++ Map("table" -> tableName, "drop_table" -> "true") + exasolConnectionManager = ExasolConnectionManager(getExasolOptions(defaultOptions)) + createTable() + } + import sqlContext.implicits._ test("saves boolean") { @@ -161,7 +172,7 @@ class SparkDataImportIT extends BaseTableQueryIT { .toDF("col_field") .write .mode("overwrite") - .options(getConfiguration() ++ Map("table" -> tableName, "drop_table" -> "true")) + .options(defaultOptions) .format("exasol") .save() diff --git a/src/test/scala/com/exasol/spark/it/TypesIT.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/it/TypesIT.scala similarity index 100% rename from src/test/scala/com/exasol/spark/it/TypesIT.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/it/TypesIT.scala diff --git a/src/test/scala/com/exasol/spark/rdd/ExasolRDDSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/rdd/ExasolRDDSuite.scala similarity index 100% rename from src/test/scala/com/exasol/spark/rdd/ExasolRDDSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/rdd/ExasolRDDSuite.scala diff --git a/src/test/scala/com/exasol/spark/util/ExasolConfigurationSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/util/ExasolConfigurationSuite.scala similarity index 100% rename from src/test/scala/com/exasol/spark/util/ExasolConfigurationSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/util/ExasolConfigurationSuite.scala diff --git a/src/test/scala/com/exasol/spark/util/ExasolConnectionManagerSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/util/ExasolConnectionManagerSuite.scala similarity index 80% rename from src/test/scala/com/exasol/spark/util/ExasolConnectionManagerSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/util/ExasolConnectionManagerSuite.scala index c5405826..cec74c91 100644 --- a/src/test/scala/com/exasol/spark/util/ExasolConnectionManagerSuite.scala +++ b/exasol-jdbc/src/test/scala/com/exasol/spark/util/ExasolConnectionManagerSuite.scala @@ -9,17 +9,15 @@ import org.scalatestplus.mockito.MockitoSugar class ExasolConnectionManagerSuite extends AnyFunSuite with Matchers with MockitoSugar { - def getManager(options: Map[String, String]): ExasolConnectionManager = - ExasolConnectionManager(ExasolConfiguration(options)) - - def getJdbcUrl(options: Map[String, String]): String = - getManager(options).getJdbcConnectionString() + private[this] def getManager(map: Map[String, String]): ExasolConnectionManager = + ExasolConnectionManager(ExasolOptionsProvider(map)) @SuppressWarnings(Array("scala:S1313")) // Hardcoded IP addresses are safe in tests - val requiredOptions: Map[String, String] = Map("host" -> "10.0.0.1", "port" -> "8888") + private[this] val requiredOptions: Map[String, String] = + Map("host" -> "10.0.0.1", "port" -> "8888", "query" -> "SELECT * FROM DUAL") test("check empty jdbc options returns correctly configured jdbc url") { - assert(getJdbcUrl(requiredOptions) === "jdbc:exa:10.0.0.1:8888") + assert(ExasolOptionsProvider(requiredOptions).getJdbcUrl() === "jdbc:exa:10.0.0.1:8888") } test("check extra jdbc options are correctly configured for establishing connection") { @@ -28,14 +26,14 @@ class ExasolConnectionManagerSuite extends AnyFunSuite with Matchers with Mockit "debug=1;encryption=0" -> "jdbc:exa:10.0.0.1:8888;debug=1;encryption=0" ).foreach { case (jdbc_options, expectedJdbcUrl) => val options = requiredOptions ++ Map("jdbc_options" -> jdbc_options) - assert(getJdbcUrl(options) === expectedJdbcUrl) + assert(ExasolOptionsProvider(options).getJdbcUrl() === expectedJdbcUrl) } } test("throws when jdbc options have invalid key-value property format") { val incorrectOpt = requiredOptions ++ Map("jdbc_options" -> "debug==1;encryption=0") val thrown = intercept[IllegalArgumentException] { - getJdbcUrl(incorrectOpt) + ExasolOptionsProvider(incorrectOpt).getJdbcUrl() } val message = thrown.getMessage() assert(message.startsWith("E-SEC-6")) @@ -46,7 +44,7 @@ class ExasolConnectionManagerSuite extends AnyFunSuite with Matchers with Mockit Seq(";debug=1;encryption=0", "encryption=1;").foreach { case options => val incorrectOpt = requiredOptions ++ Map("jdbc_options" -> options) val thrown = intercept[IllegalArgumentException] { - getJdbcUrl(incorrectOpt) + ExasolOptionsProvider(incorrectOpt).getJdbcUrl() } val message = thrown.getMessage() assert(message.startsWith("E-SEC-5")) @@ -69,7 +67,7 @@ class ExasolConnectionManagerSuite extends AnyFunSuite with Matchers with Mockit test("returns jdbc url with fingerprint") { val options = requiredOptions ++ Map("fingerprint" -> "dummy_fingerprint") - assert(getJdbcUrl(options) === "jdbc:exa:10.0.0.1/dummy_fingerprint:8888") + assert(ExasolOptionsProvider(options).getJdbcUrl() === "jdbc:exa:10.0.0.1/dummy_fingerprint:8888") } test("returns jdbc url without fingerprint if validateservercertificate=0") { @@ -77,7 +75,7 @@ class ExasolConnectionManagerSuite extends AnyFunSuite with Matchers with Mockit "jdbc_options" -> "validateservercertificate=0", "fingerprint" -> "dummy_fingerprint" ) - assert(getJdbcUrl(options) === "jdbc:exa:10.0.0.1:8888;validateservercertificate=0") + assert(ExasolOptionsProvider(options).getJdbcUrl() === "jdbc:exa:10.0.0.1:8888;validateservercertificate=0") } test("returns list of worker connections with fingerprint") { diff --git a/src/test/scala/com/exasol/spark/util/FiltersSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/util/FiltersSuite.scala similarity index 100% rename from src/test/scala/com/exasol/spark/util/FiltersSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/util/FiltersSuite.scala diff --git a/src/test/scala/com/exasol/spark/util/TypesSuite.scala b/exasol-jdbc/src/test/scala/com/exasol/spark/util/TypesSuite.scala similarity index 100% rename from src/test/scala/com/exasol/spark/util/TypesSuite.scala rename to exasol-jdbc/src/test/scala/com/exasol/spark/util/TypesSuite.scala diff --git a/src/test/scala/org/apache/spark/SparkContextHelper.scala b/exasol-jdbc/src/test/scala/org/apache/spark/SparkContextHelper.scala similarity index 100% rename from src/test/scala/org/apache/spark/SparkContextHelper.scala rename to exasol-jdbc/src/test/scala/org/apache/spark/SparkContextHelper.scala diff --git a/exasol-jdbc/versionsMavenPluginRules.xml b/exasol-jdbc/versionsMavenPluginRules.xml new file mode 100644 index 00000000..35bd03d2 --- /dev/null +++ b/exasol-jdbc/versionsMavenPluginRules.xml @@ -0,0 +1,18 @@ + + + + + (?i).*Alpha(?:-?[\d.]+)? + (?i).*a(?:-?[\d.]+)? + (?i).*Beta(?:-?[\d.]+)? + (?i).*-B(?:-?[\d.]+)? + (?i).*-b(?:-?[\d.]+)? + (?i).*RC(?:-?[\d.]+)? + (?i).*CR(?:-?[\d.]+)? + (?i).*M(?:-?[\d.]+)? + + + + \ No newline at end of file diff --git a/exasol-s3/.settings/org.eclipse.jdt.core.prefs b/exasol-s3/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..f362cccd --- /dev/null +++ b/exasol-s3/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,502 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullable.secondary= +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.APILeak=warning +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.terminalDeprecation=warning +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=ignore +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentType=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentTypeStrict=disabled +org.eclipse.jdt.core.compiler.problem.unlikelyEqualsArgumentType=info +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unstableAutoModuleName=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false +org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647 +org.eclipse.jdt.core.formatter.align_type_members_on_columns=false +org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns=false +org.eclipse.jdt.core.formatter.align_with_spaces=false +org.eclipse.jdt.core.formatter.alignment_for_additive_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_enum_constant=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter=0 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type=49 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_assertion_message=0 +org.eclipse.jdt.core.formatter.alignment_for_assignment=0 +org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_loops=16 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain=0 +org.eclipse.jdt.core.formatter.alignment_for_enum_constants=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header=0 +org.eclipse.jdt.core.formatter.alignment_for_logical_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 +org.eclipse.jdt.core.formatter.alignment_for_module_statements=16 +org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references=0 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_record_components=16 +org.eclipse.jdt.core.formatter.alignment_for_relational_operator=0 +org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 +org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_shift_operator=0 +org.eclipse.jdt.core.formatter.alignment_for_string_concatenation=16 +org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_record_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_type_annotations=0 +org.eclipse.jdt.core.formatter.alignment_for_type_arguments=0 +org.eclipse.jdt.core.formatter.alignment_for_type_parameters=0 +org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 +org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_after_package=1 +org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 +org.eclipse.jdt.core.formatter.blank_lines_before_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 +org.eclipse.jdt.core.formatter.blank_lines_before_package=0 +org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 +org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch=0 +org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 +org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_lambda_body=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_record_constructor=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_record_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped=true +org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position=false +org.eclipse.jdt.core.formatter.comment.format_block_comments=true +org.eclipse.jdt.core.formatter.comment.format_header=false +org.eclipse.jdt.core.formatter.comment.format_html=true +org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true +org.eclipse.jdt.core.formatter.comment.format_line_comments=true +org.eclipse.jdt.core.formatter.comment.format_source_code=true +org.eclipse.jdt.core.formatter.comment.indent_parameter_description=false +org.eclipse.jdt.core.formatter.comment.indent_root_tags=false +org.eclipse.jdt.core.formatter.comment.indent_tag_description=false +org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags=do not insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=do not insert +org.eclipse.jdt.core.formatter.comment.line_length=120 +org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true +org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true +org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false +org.eclipse.jdt.core.formatter.compact_else_if=true +org.eclipse.jdt.core.formatter.continuation_indentation=2 +org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off +org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on +org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=false +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_record_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true +org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_empty_lines=false +org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false +org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_additive_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default=insert +org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_record_components=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert +org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow=insert +org.eclipse.jdt.core.formatter.insert_space_after_logical_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_not_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_relational_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert +org.eclipse.jdt.core.formatter.insert_space_after_shift_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation=insert +org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_additive_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case=insert +org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default=insert +org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_record_components=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow=insert +org.eclipse.jdt.core.formatter.insert_space_before_logical_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_constructor=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert +org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_relational_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_shift_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation=insert +org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.join_lines_in_comments=true +org.eclipse.jdt.core.formatter.join_wrapped_lines=true +org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_code_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_method_body_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_record_constructor_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_record_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line=false +org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.lineSplit=120 +org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false +org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false +org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block=0 +org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_record_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause=common_lines +org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.jdt.core.formatter.tabulation.char=space +org.eclipse.jdt.core.formatter.tabulation.size=4 +org.eclipse.jdt.core.formatter.text_block_indentation=0 +org.eclipse.jdt.core.formatter.use_on_off_tags=false +org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=true +org.eclipse.jdt.core.formatter.wrap_before_additive_operator=true +org.eclipse.jdt.core.formatter.wrap_before_assertion_message_operator=true +org.eclipse.jdt.core.formatter.wrap_before_assignment_operator=false +org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator=true +org.eclipse.jdt.core.formatter.wrap_before_conditional_operator=true +org.eclipse.jdt.core.formatter.wrap_before_logical_operator=true +org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator=true +org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true +org.eclipse.jdt.core.formatter.wrap_before_relational_operator=true +org.eclipse.jdt.core.formatter.wrap_before_shift_operator=true +org.eclipse.jdt.core.formatter.wrap_before_string_concatenation=true +org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true +org.eclipse.jdt.core.javaFormatter=org.eclipse.jdt.core.defaultJavaFormatter diff --git a/exasol-s3/.settings/org.eclipse.jdt.ui.prefs b/exasol-s3/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 00000000..1add06a7 --- /dev/null +++ b/exasol-s3/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,205 @@ +cleanup.add_default_serial_version_id=true +cleanup.add_generated_serial_version_id=false +cleanup.add_missing_annotations=true +cleanup.add_missing_deprecated_annotations=true +cleanup.add_missing_methods=false +cleanup.add_missing_nls_tags=false +cleanup.add_missing_override_annotations=true +cleanup.add_missing_override_annotations_interface_methods=true +cleanup.add_serial_version_id=false +cleanup.always_use_blocks=true +cleanup.always_use_parentheses_in_expressions=false +cleanup.always_use_this_for_non_static_field_access=true +cleanup.always_use_this_for_non_static_method_access=false +cleanup.convert_functional_interfaces=true +cleanup.convert_to_enhanced_for_loop=true +cleanup.correct_indentation=true +cleanup.format_source_code=true +cleanup.format_source_code_changes_only=false +cleanup.insert_inferred_type_arguments=false +cleanup.make_local_variable_final=true +cleanup.make_parameters_final=true +cleanup.make_private_fields_final=true +cleanup.make_type_abstract_if_missing_method=false +cleanup.make_variable_declarations_final=true +cleanup.never_use_blocks=false +cleanup.never_use_parentheses_in_expressions=true +cleanup.organize_imports=false +cleanup.qualify_static_field_accesses_with_declaring_class=false +cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +cleanup.qualify_static_member_accesses_with_declaring_class=true +cleanup.qualify_static_method_accesses_with_declaring_class=false +cleanup.remove_private_constructors=true +cleanup.remove_redundant_modifiers=false +cleanup.remove_redundant_semicolons=true +cleanup.remove_redundant_type_arguments=true +cleanup.remove_trailing_whitespaces=true +cleanup.remove_trailing_whitespaces_all=true +cleanup.remove_trailing_whitespaces_ignore_empty=false +cleanup.remove_unnecessary_casts=true +cleanup.remove_unnecessary_nls_tags=true +cleanup.remove_unused_imports=true +cleanup.remove_unused_local_variables=false +cleanup.remove_unused_private_fields=true +cleanup.remove_unused_private_members=true +cleanup.remove_unused_private_methods=true +cleanup.remove_unused_private_types=true +cleanup.sort_members=false +cleanup.sort_members_all=false +cleanup.use_anonymous_class_creation=false +cleanup.use_blocks=true +cleanup.use_blocks_only_for_return_and_throw=false +cleanup.use_lambda=true +cleanup.use_parentheses_in_expressions=true +cleanup.use_this_for_non_static_field_access=true +cleanup.use_this_for_non_static_field_access_only_if_necessary=false +cleanup.use_this_for_non_static_method_access=false +cleanup.use_this_for_non_static_method_access_only_if_necessary=true +cleanup_profile=_Exasol +cleanup_settings_version=2 +eclipse.preferences.version=1 +editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true +formatter_profile=_Exasol +formatter_settings_version=21 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=java;javax;org;com; +org.eclipse.jdt.ui.ondemandthreshold=3 +org.eclipse.jdt.ui.staticondemandthreshold=3 +sp_cleanup.add_all=false +sp_cleanup.add_default_serial_version_id=true +sp_cleanup.add_generated_serial_version_id=false +sp_cleanup.add_missing_annotations=true +sp_cleanup.add_missing_deprecated_annotations=true +sp_cleanup.add_missing_methods=false +sp_cleanup.add_missing_nls_tags=false +sp_cleanup.add_missing_override_annotations=true +sp_cleanup.add_missing_override_annotations_interface_methods=true +sp_cleanup.add_serial_version_id=false +sp_cleanup.always_use_blocks=true +sp_cleanup.always_use_parentheses_in_expressions=true +sp_cleanup.always_use_this_for_non_static_field_access=true +sp_cleanup.always_use_this_for_non_static_method_access=false +sp_cleanup.array_with_curly=false +sp_cleanup.arrays_fill=false +sp_cleanup.bitwise_conditional_expression=false +sp_cleanup.boolean_literal=false +sp_cleanup.boolean_value_rather_than_comparison=false +sp_cleanup.break_loop=false +sp_cleanup.collection_cloning=false +sp_cleanup.comparing_on_criteria=false +sp_cleanup.comparison_statement=false +sp_cleanup.controlflow_merge=false +sp_cleanup.convert_functional_interfaces=true +sp_cleanup.convert_to_enhanced_for_loop=true +sp_cleanup.convert_to_enhanced_for_loop_if_loop_var_used=false +sp_cleanup.convert_to_switch_expressions=false +sp_cleanup.correct_indentation=true +sp_cleanup.do_while_rather_than_while=false +sp_cleanup.double_negation=false +sp_cleanup.else_if=false +sp_cleanup.embedded_if=false +sp_cleanup.evaluate_nullable=false +sp_cleanup.extract_increment=false +sp_cleanup.format_source_code=true +sp_cleanup.format_source_code_changes_only=false +sp_cleanup.hash=false +sp_cleanup.if_condition=false +sp_cleanup.insert_inferred_type_arguments=false +sp_cleanup.instanceof=false +sp_cleanup.instanceof_keyword=false +sp_cleanup.invert_equals=false +sp_cleanup.join=false +sp_cleanup.lazy_logical_operator=false +sp_cleanup.make_local_variable_final=true +sp_cleanup.make_parameters_final=true +sp_cleanup.make_private_fields_final=true +sp_cleanup.make_type_abstract_if_missing_method=false +sp_cleanup.make_variable_declarations_final=true +sp_cleanup.map_cloning=false +sp_cleanup.merge_conditional_blocks=false +sp_cleanup.multi_catch=false +sp_cleanup.never_use_blocks=false +sp_cleanup.never_use_parentheses_in_expressions=false +sp_cleanup.no_string_creation=false +sp_cleanup.no_super=false +sp_cleanup.number_suffix=false +sp_cleanup.objects_equals=false +sp_cleanup.on_save_use_additional_actions=true +sp_cleanup.one_if_rather_than_duplicate_blocks_that_fall_through=false +sp_cleanup.operand_factorization=false +sp_cleanup.organize_imports=true +sp_cleanup.overridden_assignment=false +sp_cleanup.plain_replacement=false +sp_cleanup.precompile_regex=false +sp_cleanup.primitive_comparison=false +sp_cleanup.primitive_parsing=false +sp_cleanup.primitive_rather_than_wrapper=false +sp_cleanup.primitive_serialization=false +sp_cleanup.pull_out_if_from_if_else=false +sp_cleanup.pull_up_assignment=false +sp_cleanup.push_down_negation=false +sp_cleanup.qualify_static_field_accesses_with_declaring_class=false +sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_with_declaring_class=true +sp_cleanup.qualify_static_method_accesses_with_declaring_class=false +sp_cleanup.reduce_indentation=false +sp_cleanup.redundant_comparator=false +sp_cleanup.redundant_falling_through_block_end=false +sp_cleanup.remove_private_constructors=true +sp_cleanup.remove_redundant_modifiers=false +sp_cleanup.remove_redundant_semicolons=true +sp_cleanup.remove_redundant_type_arguments=true +sp_cleanup.remove_trailing_whitespaces=true +sp_cleanup.remove_trailing_whitespaces_all=true +sp_cleanup.remove_trailing_whitespaces_ignore_empty=false +sp_cleanup.remove_unnecessary_array_creation=false +sp_cleanup.remove_unnecessary_casts=true +sp_cleanup.remove_unnecessary_nls_tags=true +sp_cleanup.remove_unused_imports=true +sp_cleanup.remove_unused_local_variables=false +sp_cleanup.remove_unused_private_fields=true +sp_cleanup.remove_unused_private_members=false +sp_cleanup.remove_unused_private_methods=true +sp_cleanup.remove_unused_private_types=true +sp_cleanup.return_expression=false +sp_cleanup.simplify_lambda_expression_and_method_ref=false +sp_cleanup.single_used_field=false +sp_cleanup.sort_members=false +sp_cleanup.sort_members_all=false +sp_cleanup.standard_comparison=false +sp_cleanup.static_inner_class=false +sp_cleanup.strictly_equal_or_different=false +sp_cleanup.stringbuffer_to_stringbuilder=false +sp_cleanup.stringbuilder=false +sp_cleanup.stringbuilder_for_local_vars=false +sp_cleanup.substring=false +sp_cleanup.switch=false +sp_cleanup.system_property=false +sp_cleanup.system_property_boolean=false +sp_cleanup.system_property_file_encoding=false +sp_cleanup.system_property_file_separator=false +sp_cleanup.system_property_line_separator=false +sp_cleanup.system_property_path_separator=false +sp_cleanup.ternary_operator=false +sp_cleanup.try_with_resource=false +sp_cleanup.unlooped_while=false +sp_cleanup.unreachable_block=false +sp_cleanup.use_anonymous_class_creation=false +sp_cleanup.use_autoboxing=false +sp_cleanup.use_blocks=true +sp_cleanup.use_blocks_only_for_return_and_throw=false +sp_cleanup.use_directly_map_method=false +sp_cleanup.use_lambda=true +sp_cleanup.use_parentheses_in_expressions=true +sp_cleanup.use_string_is_blank=false +sp_cleanup.use_this_for_non_static_field_access=true +sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=false +sp_cleanup.use_this_for_non_static_method_access=false +sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true +sp_cleanup.use_unboxing=false +sp_cleanup.use_var=false +sp_cleanup.useless_continue=false +sp_cleanup.useless_return=false +sp_cleanup.valueof_rather_than_instantiation=false diff --git a/exasol-s3/error_code_config.yml b/exasol-s3/error_code_config.yml new file mode 100644 index 00000000..f66154a5 --- /dev/null +++ b/exasol-s3/error_code_config.yml @@ -0,0 +1,5 @@ +error-tags: + SEC: + packages: + - com.exasol.spark + highest-index: 27 diff --git a/exasol-s3/pk_generated_parent.pom b/exasol-s3/pk_generated_parent.pom new file mode 100644 index 00000000..e6f582da --- /dev/null +++ b/exasol-s3/pk_generated_parent.pom @@ -0,0 +1,262 @@ + + + 4.0.0 + com.exasol + spark-connector-s3-generated-parent + ${revision} + pom + + com.exasol + spark-connector-parent-pom + ${revision} + ../parent-pom/pom.xml + + + UTF-8 + UTF-8 + 11 + + + + + Apache License + https://github.com/exasol/spark-connector/blob/main/LICENSE + repo + + + + + Exasol + opensource@exasol.com + Exasol AG + https://www.exasol.com/ + + + + scm:git:https://github.com/exasol/spark-connector.git + scm:git:https://github.com/exasol/spark-connector.git + https://github.com/exasol/spark-connector/ + + + + + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.9.1.2184 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + + + enforce-maven + + enforce + + + + + [3.8.7,3.9.0) + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.4.1 + + true + oss + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.sonatype.ossindex.maven + ossindex-maven-plugin + 3.2.0 + + + audit + package + + audit + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + ${test.excludeTags} + + + + org.codehaus.mojo + versions-maven-plugin + 2.15.0 + + + display-updates + package + + display-plugin-updates + display-dependency-updates + + + + + file:///${project.basedir}/versionsMavenPluginRules.xml + + + + org.basepom.maven + duplicate-finder-maven-plugin + 1.5.1 + + + default + verify + + check + + + + + true + true + true + true + true + true + false + true + true + false + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + + ${test.excludeTags} + + + + verify + + integration-test + verify + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.9 + + + prepare-agent + + prepare-agent + + + + merge-results + verify + + merge + + + + + ${project.build.directory}/ + + jacoco*.exec + + + + ${project.build.directory}/aggregate.exec + + + + report + verify + + report + + + ${project.build.directory}/aggregate.exec + + + + + + com.exasol + error-code-crawler-maven-plugin + 1.2.3 + + + verify + + verify + + + + + + io.github.zlika + reproducible-build-maven-plugin + 0.16 + + + strip-jar + package + + strip-jar + + + + + + + diff --git a/exasol-s3/pom.xml b/exasol-s3/pom.xml new file mode 100644 index 00000000..3e0f4fae --- /dev/null +++ b/exasol-s3/pom.xml @@ -0,0 +1,167 @@ + + + 4.0.0 + spark-connector-s3_${scala.compat.version} + ${revision}-spark-${spark.version} + jar + Spark Exasol Connector with S3 + A connector for Apache Spark to access Exasol + https://github.com/exasol/spark-connector/ + + spark-connector-s3-generated-parent + com.exasol + ${revision} + pk_generated_parent.pom + + + + org.scala-lang + scala-library + + + com.exasol + spark-connector-common-java + + + org.apache.spark + spark-core_${scala.compat.version} + + + org.apache.spark + spark-sql_${scala.compat.version} + + + org.apache.hadoop + hadoop-client + + + io.netty + netty-all + + + software.amazon.awssdk + s3 + 2.20.100 + + + org.apache.hadoop + hadoop-aws + ${hadoop.version} + + + org.wildfly.openssl + wildfly-openssl + 2.2.5.Final + + + + org.junit.jupiter + junit-jupiter + test + + + org.junit.jupiter + junit-jupiter-api + test + + + com.exasol + test-db-builder-java + test + + + com.exasol + hamcrest-resultset-matcher + test + + + com.exasol + exasol-testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + org.mockito + mockito-junit-jupiter + test + + + org.testcontainers + localstack + test + + + + com.amazonaws + aws-java-sdk-s3 + 1.12.503 + test + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + false + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${project.artifactId}-${project.version}-assembly + false + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + reference.conf + + + + + + + + org.basepom.maven + duplicate-finder-maven-plugin + + false + false + false + + git.properties + arrow-git.properties + mime.types + .*\.proto$ + + + + + + diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java b/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java new file mode 100644 index 00000000..ab72ae59 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/AbstractImportExportQueryGenerator.java @@ -0,0 +1,61 @@ +package com.exasol.spark.s3; + +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.Option; + +/** + * An common {@code CSV} query generator class. + * + * A generator for Exasol {@code IMPORT} or {@code EXPORT} queries that access {@code CSV} files in intermediate storage + * systems. + * + * @see Exasol Import + * @see Exasol Export + */ +public abstract class AbstractImportExportQueryGenerator { + private static final String DEFAULT_S3_ENDPOINT = "amazonaws.com"; + + /** Spark options for scenarios involving an Exasol database */ + protected final ExasolOptions options; + + /** + * Creates a new instance of {@link AbstractImportExportQueryGenerator}. + * + * @param options user provided options + */ + public AbstractImportExportQueryGenerator(final ExasolOptions options) { + this.options = options; + } + + /** + * Creates an {@code IDENTIFIED BY} part of a query. + * + * @return identifiedBy part of a query + */ + public String getIdentifier() { + final String awsAccessKeyId = this.options.get(Option.AWS_ACCESS_KEY_ID.key()); + final String awsSecretAccessKey = this.options.get(Option.AWS_SECRET_ACCESS_KEY.key()); + return "AT '" + escapeStringLiteral(getBucketURL()) + "'\nUSER '" + escapeStringLiteral(awsAccessKeyId) + + "' IDENTIFIED BY '" + escapeStringLiteral(awsSecretAccessKey) + "'\n"; + } + + private String escapeStringLiteral(final String input) { + return input.replace("'", "''"); + } + + private String getBucketURL() { + return "https://" + this.options.getS3Bucket() + ".s3." + getS3Endpoint(); + } + + private String getS3Endpoint() { + if (!this.options.containsKey(Option.S3_ENDPOINT_OVERRIDE.key())) { + return DEFAULT_S3_ENDPOINT; + } + final String override = this.options.get(Option.S3_ENDPOINT_OVERRIDE.key()); + if (this.options.hasEnabled(Option.REPLACE_LOCALHOST_BY_DEFAULT_S3_ENDPOINT.key())) { + return override.replace("localhost", DEFAULT_S3_ENDPOINT); + } + return override; + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/DelegatingWriteBuilder.java b/exasol-s3/src/main/java/com/exasol/spark/s3/DelegatingWriteBuilder.java new file mode 100644 index 00000000..376177d2 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/DelegatingWriteBuilder.java @@ -0,0 +1,38 @@ +package com.exasol.spark.s3; + +import org.apache.spark.sql.connector.write.BatchWrite; +import org.apache.spark.sql.connector.write.Write; +import org.apache.spark.sql.connector.write.WriteBuilder; + +import com.exasol.spark.common.ExasolOptions; + +/** + * A delegating {@link WriteBuilder} class. + */ +public class DelegatingWriteBuilder implements WriteBuilder { + private final ExasolOptions options; + private final WriteBuilder delegate; + + /** + * Creates a new instance of {@link DelegatingWriteBuilder}. + * + * @param options user provided options + * @param delegate delegate write builder + */ + public DelegatingWriteBuilder(final ExasolOptions options, final WriteBuilder delegate) { + this.options = options; + this.delegate = delegate; + } + + @Override + public Write build() { + return new Write() { + @Override + public BatchWrite toBatch() { + return new ExasolBatchWrite(options, delegate.build()); + } + + }; + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolBatchWrite.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolBatchWrite.java new file mode 100644 index 00000000..c3e69643 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolBatchWrite.java @@ -0,0 +1,158 @@ +package com.exasol.spark.s3; + +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.spark.sql.connector.write.*; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.ExasolValidationException; +import com.exasol.spark.common.Option; + +import software.amazon.awssdk.services.s3.model.S3Object; + +/** + * An Exasol {@link BatchWrite} implementation. + */ +public class ExasolBatchWrite implements BatchWrite { + private static final Logger LOGGER = Logger.getLogger(ExasolBatchWrite.class.getName()); + + private final ExasolOptions options; + private final BatchWrite delegate; + + /** + * Creates a new instance of {@link ExasolBatchWrite}. + * + * @param options user provided options + * @param delegate delegate {@code CSV} batch write + */ + public ExasolBatchWrite(final ExasolOptions options, final Write delegate) { + this.options = options; + this.delegate = delegate.toBatch(); + } + + @Override + public DataWriterFactory createBatchWriterFactory(final PhysicalWriteInfo info) { + return delegate.createBatchWriterFactory(info); + } + + @Override + public boolean useCommitCoordinator() { + return delegate.useCommitCoordinator(); + } + + @Override + public void abort(final WriterCommitMessage[] messages) { + LOGGER.info("Running abort stage of the job."); + cleanup(); + delegate.abort(messages); + } + + private void cleanup() { + final String intermediateLocation = this.options.get(Option.INTERMEDIATE_DATA_PATH.key()); + LOGGER.info(() -> "Running cleanup process for directory '" + intermediateLocation + "'."); + try (final S3FileSystem s3FileSystem = S3FileSystem.fromOptions(this.options)) { + s3FileSystem.deleteKeys(this.options.getS3Bucket(), this.options.get(Option.WRITE_S3_BUCKET_KEY.key())); + } + } + + @Override + public void commit(final WriterCommitMessage[] messages) { + LOGGER.info("Committing the file writing stage of the job."); + delegate.commit(messages); + importIntermediateDataIntoExasol(); + } + + private void importIntermediateDataIntoExasol() { + final long start = System.currentTimeMillis(); + final String table = this.options.getTable(); + final String query = new S3ImportQueryGenerator(options).generateQuery(); + final int rows = executeImportQuery(query); + final long time = System.currentTimeMillis() - start; + LOGGER.info(() -> "Imported '" + rows + "' rows into the table '" + table + "' in '" + time + "' millis."); + } + + private int executeImportQuery(final String query) { + try (final Connection connection = new ExasolConnectionFactory(this.options).getConnection(); + final Statement stmt = connection.createStatement()) { + connection.setAutoCommit(false); + final int rows = stmt.executeUpdate(query); + connection.commit(); + return rows; + } catch (final SQLException exception) { + throw new ExasolConnectionException(ExaError.messageBuilder("E-SEC-24") + .message("Failure running the import {{query}} query.", removeCredentialsFromQuery(query)) + .mitigation("Please check that connection address, username and password are correct.").toString(), + exception); + } finally { + cleanup(); + } + } + + private static String removeCredentialsFromQuery(final String input) { + return Stream.of(input.split("\n")).filter(s -> !s.contains("IDENTIFIED BY")).collect(Collectors.joining("\n")); + } + + /** + * A class that generates {@code SQL} query for importing data from intermediate {@code S3} location into Exasol + * database. + */ + private static class S3ImportQueryGenerator extends AbstractImportExportQueryGenerator { + + public S3ImportQueryGenerator(final ExasolOptions options) { + super(options); + } + + public String generateQuery() { + final String table = this.options.getTable(); + return new StringBuilder() // + .append("IMPORT INTO ") // + .append(table) // + .append(" FROM CSV\n") // + .append(getIdentifier()) // + .append(getFiles()) // + .append(getFooter()) // + .toString(); + } + + private String getFiles() { + final String path = this.options.get(Option.INTERMEDIATE_DATA_PATH.key()); + final URI pathURI = getPathURI(path); + final String bucketName = pathURI.getHost(); + final String bucketKey = pathURI.getPath().substring(1); + try (final S3FileSystem s3FileSystem = S3FileSystem.fromOptions(this.options)) { + final List objects = s3FileSystem.listObjects(bucketName, Optional.of(bucketKey)); + final StringBuilder builder = new StringBuilder(); + for (final S3Object object : objects) { + builder.append("FILE '").append(object.key()).append("'\n"); + } + return builder.toString(); + } + } + + private URI getPathURI(final String path) { + try { + return new URI(path); + } catch (final URISyntaxException exception) { + throw new ExasolValidationException(ExaError.messageBuilder("E-SEC-25") + .message("Provided path {{path}} cannot be converted to URI systax.", path) + .mitigation("Please make sure the path is correct file system (hdfs, s3a, etc) path.") + .toString(), exception); + } + } + + private String getFooter() { + return "SKIP = 1"; + } + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolConnectionException.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolConnectionException.java new file mode 100644 index 00000000..4e21fcf1 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolConnectionException.java @@ -0,0 +1,27 @@ +package com.exasol.spark.s3; + +/** + * An exception for Exasol JDCB connection issues. + */ +public class ExasolConnectionException extends RuntimeException { + private static final long serialVersionUID = 2818034094289319833L; + + /** + * Creates an instance of a {@link ExasolConnectionException}. + * + * @param message error message + * @param cause exception cause + */ + public ExasolConnectionException(final String message, final Throwable cause) { + super(message, cause); + } + + /** + * Creates an instance of a {@link ExasolConnectionException}. + * + * @param message error message + */ + public ExasolConnectionException(final String message) { + super(message); + } +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolConnectionFactory.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolConnectionFactory.java new file mode 100644 index 00000000..ac63c582 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolConnectionFactory.java @@ -0,0 +1,66 @@ +package com.exasol.spark.s3; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.logging.Logger; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.ExasolOptions; + +/** + * A factory that creates JDBC connection to Exasol database. + */ +public final class ExasolConnectionFactory { + private static final Logger LOGGER = Logger.getLogger(ExasolConnectionFactory.class.getName()); + private final ExasolOptions options; + + /** + * Creates an instance of a {@link ExasolConnectionFactory}. + * + * @param options {@link ExasolOptions} options + */ + public ExasolConnectionFactory(final ExasolOptions options) { + this.options = options; + } + + /** + * Creates a JDBC connection to an Exasol database if none exists yet. + * + * @return JDBC connection + * @throws SQLException if the connection cannot be established + */ + public synchronized Connection getConnection() throws SQLException { + verifyExasolJDBCDriverAvailable(); + final String address = options.getJdbcUrl(); + final String username = options.getUsername(); + LOGGER.fine(() -> "Getting connection at '" + address + "' with username '" + username + "' and password."); + try { + final long start = System.currentTimeMillis(); + final Connection connection = DriverManager.getConnection(address, username, options.getPassword()); + final long connectionTime = System.currentTimeMillis() - start; + LOGGER.info(() -> "Obtained connection to '" + address + "' in '" + connectionTime + "' milliseconds."); + return connection; + } catch (final SQLException exception) { + throw new ExasolConnectionException(ExaError.messageBuilder("E-SEC-17") + .message("Could not connect to Exasol address on {{address}} with username {{username}}.") + .parameter("address", address).parameter("username", username) + .mitigation("Please check that connection address, username and password are correct.").toString(), + exception); + } + } + + private void verifyExasolJDBCDriverAvailable() { + final String driverClassName = "com.exasol.jdbc.EXADriver"; + try { + Class.forName(driverClassName); + } catch (final ClassNotFoundException exception) { + throw new ExasolConnectionException( + ExaError.messageBuilder("E-SEC-18") + .message("Failed to find Exasol JDBC Driver class {{class}}.", driverClassName) + .mitigation("Please make sure that Exasol JDBC Driver is installed.").toString(), + exception); + } + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3ScanBuilder.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3ScanBuilder.java new file mode 100644 index 00000000..9bf46bbb --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3ScanBuilder.java @@ -0,0 +1,215 @@ +package com.exasol.spark.s3; + +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.read.*; +import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat; +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable; +import org.apache.spark.sql.sources.Filter; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.ExasolValidationException; + +import scala.collection.JavaConverters; + +/** + * A class that implements {@link ScanBuilder} interface for accessing {@code S3} intermediate storage. + */ +public class ExasolS3ScanBuilder implements ScanBuilder, SupportsPushDownFilters, SupportsPushDownRequiredColumns { + private static final Logger LOGGER = Logger.getLogger(ExasolS3ScanBuilder.class.getName()); + private final ExasolOptions options; + private final CaseInsensitiveStringMap properties; + + private StructType schema; + private Filter[] pushedFilters; + + /** + * Creates a new instance of {@link ExasolS3ScanBuilder}. + * + * @param options user provided options + * @param schema user-provided {@link StructType} schema + * @param properties original key-value properties map that is passed to delegating classes + */ + public ExasolS3ScanBuilder(final ExasolOptions options, final StructType schema, + final CaseInsensitiveStringMap properties) { + this.options = options; + this.schema = schema; + this.properties = properties; + this.pushedFilters = new Filter[0]; + } + + @Override + public Filter[] pushFilters(final Filter[] filters) { + final List unsupportedFilters = getUnsupportedFilters(filters); + final List supportedFilters = new ArrayList<>(Arrays.asList(filters)); + supportedFilters.removeAll(unsupportedFilters); + this.pushedFilters = supportedFilters.toArray(new Filter[] {}); + return unsupportedFilters.toArray(new Filter[] {}); + } + + private List getUnsupportedFilters(final Filter[] filters) { + return Collections.emptyList(); + } + + @Override + public Filter[] pushedFilters() { + return this.pushedFilters; + } + + @Override + public void pruneColumns(final StructType schema) { + this.schema = schema; + } + + @Override + public Scan build() { + final SparkSession sparkSession = SparkSession.active(); + final String bucket = this.options.getS3Bucket(); + final String bucketKey = generateRandomBucketKey(sparkSession); + // Import query data into `s3Bucket/s3BucketKey` location as `CSV` files + LOGGER.info(() -> "Using S3 bucket '" + bucket + "' with folder '" + bucketKey + "' for scan job data."); + addSparkCleanupJobListener(sparkSession, bucketKey); + prepareIntermediateData(bucketKey); + // Uses Spark `CSVTable` to read `CSV` files + return new CSVTable("", // + sparkSession, // + this.properties, // + getCSVFiles(bucket, bucketKey), // + scala.Option.apply(this.schema), // + CSVFileFormat.class // + ).newScanBuilder(getUpdatedMapWithCSVOptions(this.properties)).build(); + } + + private String generateRandomBucketKey(final SparkSession sparkSession) { + return UUID.randomUUID() + "-" + sparkSession.sparkContext().applicationId(); + } + + private void addSparkCleanupJobListener(final SparkSession spark, final String bucketKey) { + spark.sparkContext().addSparkListener(new S3CleanupListener(this.options, bucketKey)); + } + + private scala.collection.immutable.List getCSVFiles(final String bucket, final String bucketKey) { + final String path = "s3a://" + Paths.get(bucket, bucketKey, "*.csv").toString(); + return JavaConverters.collectionAsScalaIterableConverter(Arrays.asList(path)).asScala().toList(); + } + + private CaseInsensitiveStringMap getUpdatedMapWithCSVOptions(final CaseInsensitiveStringMap map) { + final Map updatedMap = new HashMap<>(map.asCaseSensitiveMap()); + updatedMap.put("header", "true"); + updatedMap.put("delimiter", ","); + return new CaseInsensitiveStringMap(updatedMap); + } + + /** + * Returns SQL query that would be run on the Exasol database. + * + * This is enriched query that would add predicates or specific columns on top the user provided query or table. The + * result of this enriched query will be saved into the intermediate storage. + * + * @return Enriched SQL query for the intermediate storage. + */ + protected String getScanQuery() { + return "SELECT * FROM " + getTableOrQuery() + " "; + } + + private String getTableOrQuery() { + if (this.options.hasTable()) { + return this.options.getTable(); + } else { + return "(" + this.options.getQuery() + ")"; + } + } + + private void prepareIntermediateData(final String bucketKey) { + final String exportQuery = new S3ExportQueryGenerator(this.options, bucketKey).generateQuery(getScanQuery()); + new S3DataExporter(this.options, bucketKey).exportData(exportQuery); + } + + /** + * A class that generates {@code SQL} query for exporting data from Exasol database into {@code S3} location. + */ + private static class S3ExportQueryGenerator extends AbstractImportExportQueryGenerator { + private final String bucketKey; + private final int numberOfFiles; + + public S3ExportQueryGenerator(final ExasolOptions options, final String bucketKey) { + super(options); + this.bucketKey = bucketKey; + this.numberOfFiles = options.getNumberOfPartitions(); + } + + public String generateQuery(final String baseQuery) { + return new StringBuilder() // + .append("EXPORT (\n") // + .append(baseQuery) // + .append("\n) INTO CSV\n") // + .append(getIdentifier()) // + .append(getFiles()) // + .append(getFooter()) // + .toString(); + } + + private String getFiles() { + final StringBuilder builder = new StringBuilder(); + final String prefix = "FILE '" + this.bucketKey + "/"; + for (int fileIndex = 1; fileIndex <= this.numberOfFiles; fileIndex++) { + builder.append(prefix).append(String.format("part-%03d", fileIndex)).append(".csv'\n"); + } + return builder.toString(); + } + + private String getFooter() { + return "WITH COLUMN NAMES\nBOOLEAN = 'true/false'"; + } + } + + /** + * A class that exports data from Exasol database into {@code S3} location. + */ + private static class S3DataExporter { + private final ExasolOptions options; + private final String bucket; + private final String bucketKey; + + public S3DataExporter(final ExasolOptions options, final String bucketKey) { + this.options = options; + this.bucket = options.getS3Bucket(); + this.bucketKey = bucketKey; + } + + public int exportData(final String exportQuery) { + final ExasolConnectionFactory connectionFactory = new ExasolConnectionFactory(this.options); + try (final Connection connection = connectionFactory.getConnection(); + final Statement statement = connection.createStatement()) { + final int numberOfExportedRows = statement.executeUpdate(exportQuery); + LOGGER.info(() -> "Exported '" + numberOfExportedRows + "' rows into '" + this.bucket + "/" + + this.bucketKey + "'."); + return numberOfExportedRows; + } catch (final SQLException exception) { + throw new ExasolValidationException(ExaError.messageBuilder("E-SEC-22") + .message("Failed to run export query {{exportQuery}} into S3 location {{s3Path}}.") + .parameter("exportQuery", removeIdentifiedByPart(exportQuery)) + .parameter("s3Path", this.bucket + "/" + this.bucketKey) + .mitigation( + "Please ensure that query and table name are correct and satisfy SQL syntax requirements.") + .toString(), exception); + } + } + } + + private static String removeIdentifiedByPart(final String input) { + return Stream.of(input.split("\n")).filter(s -> !s.contains("IDENTIFIED BY")).collect(Collectors.joining("\n")); + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java new file mode 100644 index 00000000..f64c6b6c --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolS3Table.java @@ -0,0 +1,128 @@ +package com.exasol.spark.s3; + +import java.util.Collections; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.catalog.SupportsRead; +import org.apache.spark.sql.connector.catalog.SupportsWrite; +import org.apache.spark.sql.connector.catalog.TableCapability; +import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.connector.write.WriteBuilder; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.ExasolValidationException; +import com.exasol.spark.common.Option; + +/** + * Represents an instance of {@link ExasolS3Table}. + * + * It uses AWS S3 as an intermediate storage for reading or writing to Exasol database. + */ +public class ExasolS3Table implements SupportsRead, SupportsWrite { + private final StructType schema; + private final Set capabilities; + + /** + * Creates a new instance of {@link ExasolS3Table}. + * + * @param schema user provided schema + */ + public ExasolS3Table(final StructType schema) { + this.schema = schema; + this.capabilities = Collections.unmodifiableSet( + Stream.of(TableCapability.BATCH_READ, TableCapability.BATCH_WRITE).collect(Collectors.toSet())); + } + + @Override + public String name() { + final StringBuilder builder = new StringBuilder(); + builder // + .append("ExasolS3Table[") // + .append("schema='" + this.schema().toString()) // + .append("',") // + .append("capabilities='" + this.capabilities().toString()) // + .append("']"); + return builder.toString(); + } + + @Override + public StructType schema() { + return this.schema; + } + + @Override + public Set capabilities() { + return this.capabilities; + } + + @Override + public ScanBuilder newScanBuilder(final CaseInsensitiveStringMap map) { + final ExasolOptions options = ExasolOptions.from(map); + validateNumberOfPartitions(options); + updateSparkConfigurationForS3(options); + return new ExasolS3ScanBuilder(options, this.schema, map); + } + + @Override + public WriteBuilder newWriteBuilder(final LogicalWriteInfo defaultInfo) { + final ExasolOptions options = ExasolOptions.from(defaultInfo.options()); + validateHasTable(options); + validateNumberOfPartitions(options); + updateSparkConfigurationForS3(options); + final SparkSession sparkSession = SparkSession.active(); + final String applicationId = sparkSession.sparkContext().applicationId(); + final S3BucketKeyPathProvider prov = new UUIDS3BucketKeyPathProvider(applicationId); + return new ExasolWriteBuilderProvider(options, prov).createWriteBuilder(this.schema, defaultInfo); + } + + private void validateNumberOfPartitions(final ExasolOptions options) { + final int numberOfPartitions = options.getNumberOfPartitions(); + final int maxAllowedPartitions = Integer.parseInt(Option.MAX_ALLOWED_NUMBER_OF_PARTITIONS.key()); + if (numberOfPartitions > maxAllowedPartitions) { + throw new ExasolValidationException(ExaError.messageBuilder("E-SEC-23") // + .message("The number of partitions exceeds the supported maximum of {{MAXPARTITIONS}}.", + maxAllowedPartitions) // + .mitigation("Please set parameter {{param}} to a lower value.", Option.NUMBER_OF_PARTITIONS.key()) // + .toString()); + } + } + + private void validateHasTable(final ExasolOptions options) { + if (!options.hasTable()) { + throw new ExasolValidationException(ExaError.messageBuilder("E-SEC-19") + .message("Missing 'table' option when writing into Exasol database.") + .mitigation("Please set 'table' property with fully qualified " + + "(e.g. 'schema_name.table_name') Exasol table name.") + .toString()); + } + } + + private void updateSparkConfigurationForS3(final ExasolOptions options) { + final SparkSession sparkSession = SparkSession.active(); + synchronized (sparkSession.sparkContext().hadoopConfiguration()) { + final Configuration conf = sparkSession.sparkContext().hadoopConfiguration(); + conf.set("fs.s3a.access.key", options.get(Option.AWS_ACCESS_KEY_ID.key())); + conf.set("fs.s3a.secret.key", options.get(Option.AWS_SECRET_ACCESS_KEY.key())); + if (options.containsKey(Option.AWS_CREDENTIALS_PROVIDER.key())) { + conf.set("fs.s3a.aws.credentials.provider", options.get(Option.AWS_CREDENTIALS_PROVIDER.key())); + } else { + conf.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + } + if (options.containsKey(Option.S3_ENDPOINT_OVERRIDE.key())) { + conf.set("fs.s3a.endpoint", "http://" + options.get(Option.S3_ENDPOINT_OVERRIDE.key())); + } + if (options.hasEnabled(Option.S3_PATH_STYLE_ACCESS.key())) { + conf.set("fs.s3a.path.style.access", "true"); + } + } + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolWriteBuilderProvider.java b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolWriteBuilderProvider.java new file mode 100644 index 00000000..83939a19 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/ExasolWriteBuilderProvider.java @@ -0,0 +1,128 @@ +package com.exasol.spark.s3; + +import java.nio.file.Paths; +import java.util.*; +import java.util.logging.Logger; + +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.connector.write.WriteBuilder; +import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat; +import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.ExasolValidationException; +import com.exasol.spark.common.Option; + +import scala.collection.JavaConverters; + +/** + * A class that provides {@link WriteBuilder} instance. + */ +public final class ExasolWriteBuilderProvider { + private static final Logger LOGGER = Logger.getLogger(ExasolWriteBuilderProvider.class.getName()); + + private final ExasolOptions options; + private final S3BucketKeyPathProvider s3BucketKeyPathProvider; + + /** + * Creates a new instance of {@link ExasolWriteBuilderProvider}. + * + * @param options user provided options + * @param s3BucketKeyPathProvider {@code S3} bucket key folder path provider class + */ + public ExasolWriteBuilderProvider(final ExasolOptions options, + final S3BucketKeyPathProvider s3BucketKeyPathProvider) { + this.options = options; + this.s3BucketKeyPathProvider = s3BucketKeyPathProvider; + } + + /** + * Creates a {@link WriteBuilder} for writing into Exasol database. + * + * @param schema user provided {@link StructType} schema + * @param defaultInfo {@link LogicalWriteInfo} information for writing + * @return an instance of {@link WriteBuilder} + */ + public WriteBuilder createWriteBuilder(final StructType schema, final LogicalWriteInfo defaultInfo) { + final SparkSession sparkSession = SparkSession.active(); + final String s3Bucket = this.options.getS3Bucket(); + final String s3BucketKey = this.s3BucketKeyPathProvider.getS3BucketKeyForWriteLocation(defaultInfo.queryId()); + validateWritePathIsEmpty(s3Bucket, s3BucketKey); + return createCSVWriteBuilder(sparkSession, schema, + getUpdatedLogicalWriteInfo(defaultInfo, s3Bucket, s3BucketKey)); + } + + private void validateWritePathIsEmpty(final String s3Bucket, final String s3BucketKey) { + try (final S3FileSystem s3FileSystem = S3FileSystem.fromOptions(this.options)) { + if (!s3FileSystem.isEmpty(s3Bucket, Optional.of(s3BucketKey))) { + throw new ExasolValidationException(ExaError.messageBuilder("E-SEC-27") // + .message("The intermediate write path is not empty.") // + .mitigation("Please ensure that the intermediate write path is empty or cleaned up properly.") // + .toString()); + } + } + } + + private WriteBuilder createCSVWriteBuilder(final SparkSession sparkSession, final StructType schema, + final LogicalWriteInfo info) { + final ExasolOptions updatedOptions = getUpdatedOptions(info.options()); + final String intermediateDataPath = updatedOptions.get(Option.INTERMEDIATE_DATA_PATH.key()); + LOGGER.info(() -> "Writing intermediate data to the '" + intermediateDataPath + "' path for write job."); + final CSVTable csvTable = new CSVTable("", sparkSession, info.options(), getS3WritePath(intermediateDataPath), + scala.Option.apply(schema), CSVFileFormat.class); + return new DelegatingWriteBuilder(updatedOptions, csvTable.newWriteBuilder(info)); + } + + private ExasolOptions getUpdatedOptions(final Map map) { + final ExasolOptions.Builder builder = ExasolOptions.builder() // + .host(this.options.getHost()) // + .port(this.options.getPort()) // + .username(this.options.getUsername()) // + .password(this.options.getPassword()) // + .fingerprint(this.options.getFingerprint()) // + .s3Bucket(this.options.getS3Bucket()); + if (this.options.hasTable()) { + builder.table(this.options.getTable()); + } else { + builder.query(this.options.getQuery()); + } + builder.withOptionsMap(map); + return builder.build(); + } + + private LogicalWriteInfo getUpdatedLogicalWriteInfo(final LogicalWriteInfo defaultInfo, final String s3Bucket, + final String s3BucketKey) { + final Map map = new HashMap<>(defaultInfo.options().asCaseSensitiveMap()); + map.put("header", "true"); + map.put("delimiter", ","); + map.put("mapreduce.fileoutputcommitter.marksuccessfuljobs", "false"); + map.put(Option.INTERMEDIATE_DATA_PATH.key(), "s3a://" + Paths.get(s3Bucket, s3BucketKey).toString()); + map.put(Option.WRITE_S3_BUCKET_KEY.key(), s3BucketKey); + + return new LogicalWriteInfo() { + @Override + public String queryId() { + return defaultInfo.queryId(); + } + + @Override + public StructType schema() { + return defaultInfo.schema(); + } + + @Override + public CaseInsensitiveStringMap options() { + return new CaseInsensitiveStringMap(map); + } + }; + } + + private scala.collection.immutable.List getS3WritePath(final String path) { + return JavaConverters.asScalaIteratorConverter(Arrays.asList(path).iterator()).asScala().toList(); + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/S3BucketKeyPathProvider.java b/exasol-s3/src/main/java/com/exasol/spark/s3/S3BucketKeyPathProvider.java new file mode 100644 index 00000000..23ad112b --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/S3BucketKeyPathProvider.java @@ -0,0 +1,17 @@ +package com.exasol.spark.s3; + +/** + * An interface for creating Spark job write {@code S3} bucket folders for writing intermediate data. + */ +public interface S3BucketKeyPathProvider { + + /** + * Returns an {@code S3} bucket key path for writing intermediate data. + * + * @param queryId Spark query identifier that started the write job + * return {@code S3} bucket key path for intermediate data + */ + public String getS3BucketKeyForWriteLocation(final String queryId); + +} + diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/S3CleanupListener.java b/exasol-s3/src/main/java/com/exasol/spark/s3/S3CleanupListener.java new file mode 100644 index 00000000..39d69981 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/S3CleanupListener.java @@ -0,0 +1,43 @@ +package com.exasol.spark.s3; + +import java.util.logging.Logger; + +import org.apache.spark.scheduler.SparkListener; +import org.apache.spark.scheduler.SparkListenerJobEnd; + +import com.exasol.spark.common.ExasolOptions; + +/** + * A {@link SparkListener} class that cleans up {@code S3} intermediate location at the end of job run. + */ +public final class S3CleanupListener extends SparkListener { + private static final Logger LOGGER = Logger.getLogger(S3CleanupListener.class.getName()); + private final ExasolOptions options; + private final String bucketKey; + + /** + * Creates an instance of {@link S3CleanupListener}. + * + * @param options user provided options + * @param bucketKey bucketKey inside the user provided bucket + */ + public S3CleanupListener(final ExasolOptions options, final String bucketKey) { + this.options = options; + this.bucketKey = bucketKey; + } + + @Override + public void onJobEnd(final SparkListenerJobEnd jobEnd) { + LOGGER.info(() -> "Cleaning up the bucket '" + this.options.getS3Bucket() + "' with key '" + this.bucketKey + + "' in job '" + jobEnd.jobId() + "'."); + deleteObjects(); + super.onJobEnd(jobEnd); + } + + private void deleteObjects() { + try (final S3FileSystem s3FileSystem = S3FileSystem.fromOptions(this.options)) { + s3FileSystem.deleteKeys(this.options.getS3Bucket(), this.bucketKey); + } + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/S3ClientFactory.java b/exasol-s3/src/main/java/com/exasol/spark/s3/S3ClientFactory.java new file mode 100644 index 00000000..030ee027 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/S3ClientFactory.java @@ -0,0 +1,80 @@ +package com.exasol.spark.s3; + +import java.net.URI; + +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.Option; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.*; + +/** + * A factory class that creates S3 clients. + */ +public final class S3ClientFactory { + private final ExasolOptions options; + + /** + * Creates a new instance of {@link S3ClientFactory}. + * + * @param options {@link ExasolOptions} options + */ + public S3ClientFactory(final ExasolOptions options) { + this.options = options; + } + + /** + * Creates a new AWS S3 client. + * + * @return new S3 client + */ + public S3Client getS3Client() { + final S3ClientBuilder builder = S3Client.builder() // + .credentialsProvider(getCredentialsProvider()); + setRegionIfEnabled(builder); + setPathStyleAccessIfEnabled(builder); + setEndpointOverrideIfEnabled(builder); + return builder.build(); + } + + private void setRegionIfEnabled(final S3BaseClientBuilder builder) { + if (this.options.containsKey(Option.AWS_REGION.key())) { + builder.region(Region.of(this.options.get(Option.AWS_REGION.key()))); + } + } + + private AwsCredentialsProvider getCredentialsProvider() { + final String awsAccessKeyId = this.options.get(Option.AWS_ACCESS_KEY_ID.key()); + final String awsSecretAccessKey = this.options.get(Option.AWS_SECRET_ACCESS_KEY.key()); + return StaticCredentialsProvider.create(AwsBasicCredentials.create(awsAccessKeyId, awsSecretAccessKey)); + } + + private void setPathStyleAccessIfEnabled(final S3BaseClientBuilder builder) { + if (this.options.hasEnabled(Option.S3_PATH_STYLE_ACCESS.key())) { + builder.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()); + } + } + + private void setEndpointOverrideIfEnabled(final S3BaseClientBuilder builder) { + if (this.options.containsKey(Option.S3_ENDPOINT_OVERRIDE.key())) { + builder.endpointOverride(URI.create(getEndpointOverride())); + } + } + + private String getEndpointOverride() { + final String protocol = getProtocol(); + return protocol + "://s3." + this.options.get(Option.S3_ENDPOINT_OVERRIDE.key()); + } + + private String getProtocol() { + if (!this.options.containsKey(Option.AWS_USE_SSL.key()) || this.options.hasEnabled(Option.AWS_USE_SSL.key())) { + return "https"; + } else { + return "http"; + } + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/S3FileSystem.java b/exasol-s3/src/main/java/com/exasol/spark/s3/S3FileSystem.java new file mode 100644 index 00000000..4f5e7f38 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/S3FileSystem.java @@ -0,0 +1,155 @@ +package com.exasol.spark.s3; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.ExasolOptions; + +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +/** + * An S3 file system operations implementations. + */ +public final class S3FileSystem implements Closeable { + private static final Logger LOGGER = Logger.getLogger(S3FileSystem.class.getName()); + private final S3Client s3Client; + + /** + * Creates a new instance of {@link S3FileSystem}. + * + * @param s3Client s3 client object + */ + public S3FileSystem(final S3Client s3Client) { + this.s3Client = s3Client; + } + + /** + * Creates a new instance of {@link S3FileSystem} from {@link ExasolOptions} options. + * + * @param options user provided options + * @return new instance of {@link S3FileSystem} + */ + public static S3FileSystem fromOptions(final ExasolOptions options) { + return new S3FileSystem(new S3ClientFactory(options).getS3Client()); + } + + /** + * Checks if a given bucket exists. + * + * @param bucketName name of a bucket + * @return {@code true} if bucket exists, {@code false} otherwise + */ + public boolean doesBucketExist(final String bucketName) { + try { + s3Client.headBucket(HeadBucketRequest.builder().bucket(bucketName).build()); + return true; + } catch (final NoSuchBucketException exception) { + return false; + } + } + + /** + * Deletes a given bucket. + * + * @param bucketName name of a bucket + */ + public void deleteBucket(final String bucketName) { + LOGGER.info(() -> "Deleting S3 bucket '" + bucketName + "'."); + deleteObjects(bucketName, Optional.empty()); + } + + /** + * For a bucket with given name: delete all contents with the specified key. + * + * @param bucketName name of a bucket + * @param bucketKey bucket key value + */ + public void deleteKeys(final String bucketName, final String bucketKey) { + LOGGER.info(() -> "Deleting objects in S3 bucket '" + bucketName + "' with bucket key '" + bucketKey + "'."); + deleteObjects(bucketName, Optional.of(bucketKey)); + } + + private void deleteObjects(final String bucketName, final Optional bucketKey) { + try { + final List objects = listObjects(bucketName, bucketKey); + List objectIdentifiers = objects.stream() // + .map(object -> ObjectIdentifier.builder().key(object.key()).build()) // + .collect(Collectors.toList()); + deleteObjectIdentifiers(bucketName, objectIdentifiers); + } catch (final SdkClientException exception) { + throw new ExasolConnectionException( + ExaError.messageBuilder("E-SEC-20") + .message("Failed to delete objects in {{BUCKET}} with key {{KEY}}.", bucketName, + bucketKey.orElse("emptyBucketKey")) + .mitigation("Please check that credentials and bucket name are correct.").toString(), + exception); + } catch (final S3Exception exception) { + throw new ExasolConnectionException(ExaError.messageBuilder("E-SEC-21") + .message("Failed to delete objects in {{BUCKET}} with key {{KEY}} because of unexpected S3 exception.") + .parameter("BUCKET", bucketName).parameter("KEY", bucketKey.orElse("emptyBucketKey")) + .ticketMitigation().toString(), exception); + } + } + + /** + * Lists objects in a given bucket with optional bucket key. + * + * @param bucketName name of a bucket + * @param bucketKey optional bucket key + */ + public List listObjects(final String bucketName, final Optional bucketKey) { + final List result = new ArrayList<>(); + final ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder().bucket(bucketName); + if (bucketKey.isPresent()) { + builder.prefix(bucketKey.get()); + } + for (final ListObjectsV2Response page: s3Client.listObjectsV2Paginator(builder.build())) { + for (final S3Object s3Object: page.contents()) { + result.add(s3Object); + } + } + return result; + } + + /** + * Checks if a given bucket with optional bucket key is empty. + * + * @param bucketName name of a bucket + * @param bucketKey optional bucket key + */ + public boolean isEmpty(final String bucketName, final Optional bucketKey) { + final ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder().bucket(bucketName); + if (bucketKey.isPresent()) { + builder.prefix(bucketKey.get()); + } + for (final ListObjectsV2Response page: s3Client.listObjectsV2Paginator(builder.build())) { + if (!page.contents().isEmpty()) { + return false; + } + } + return true; + } + + private void deleteObjectIdentifiers(final String bucketName, final List objectIdentifiers) { + if (!objectIdentifiers.isEmpty()) { + DeleteObjectsRequest deleteObjectsRequest = DeleteObjectsRequest.builder() // + .bucket(bucketName) // + .delete(Delete.builder().objects(objectIdentifiers).build()) // + .build(); + s3Client.deleteObjects(deleteObjectsRequest); + } + } + + @Override + public void close() { + this.s3Client.close(); + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/S3Source.java b/exasol-s3/src/main/java/com/exasol/spark/s3/S3Source.java new file mode 100644 index 00000000..80153e78 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/S3Source.java @@ -0,0 +1,139 @@ +package com.exasol.spark.s3; + +import java.sql.*; +import java.util.*; +import java.util.logging.Logger; + +import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableProvider; +import org.apache.spark.sql.connector.expressions.Transform; +import org.apache.spark.sql.sources.DataSourceRegister; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +import com.exasol.errorreporting.ExaError; +import com.exasol.spark.common.*; +import com.exasol.sql.StatementFactory; +import com.exasol.sql.dql.select.Select; +import com.exasol.sql.dql.select.rendering.SelectRenderer; +import com.exasol.sql.rendering.StringRendererConfig; + +/** + * An S3 Spark Connector Source. + */ +public class S3Source implements TableProvider, DataSourceRegister { + private static final Logger LOGGER = Logger.getLogger(S3Source.class.getName()); + private static final List REQUIRED_OPTIONS = Arrays.asList(Option.HOST.key(), Option.PORT.key(), + Option.USERNAME.key(), Option.PASSWORD.key()); + + @Override + public String shortName() { + return "exasol-s3"; + } + + @Override + public boolean supportsExternalMetadata() { + return true; + } + + @Override + public StructType inferSchema(final CaseInsensitiveStringMap map) { + LOGGER.fine(() -> "Running schema inference for the S3 source."); + validateOptions(map); + return getSchema(ExasolOptions.from(map)); + } + + @Override + public Table getTable(final StructType schema, final Transform[] partitioning, + final Map properties) { + return new ExasolS3Table(schema); + } + + private void validateOptions(final CaseInsensitiveStringMap options) { + LOGGER.finest(() -> "Validating options of the s3 source."); + if (!options.containsKey(Option.TABLE.key()) && !options.containsKey(Option.QUERY.key())) { + throw new IllegalArgumentException( + ExaError.messageBuilder("E-SEC-12").message("Missing 'query' or 'table' option.") + .mitigation("Please provide either one of 'query' or 'table' options.").toString()); + } + if (options.containsKey(Option.TABLE.key()) && options.containsKey(Option.QUERY.key())) { + throw new IllegalArgumentException( + ExaError.messageBuilder("E-SEC-13").message("Both 'query' and 'table' options are provided.") + .mitigation("Please use only either one of the options.").toString()); + } + validateRequiredOptions(options); + } + + private void validateRequiredOptions(CaseInsensitiveStringMap options) { + for (final String key : REQUIRED_OPTIONS) { + if (!options.containsKey(key)) { + throw new IllegalArgumentException(ExaError.messageBuilder("E-SEC-14") + .message("Required option {{KEY}} is not found.") + .mitigation("Please provide a value for the {{KEY}} option.").parameter("KEY", key).toString()); + } + } + } + + private StructType getSchema(final ExasolOptions options) { + final String limitQuery = generateInferSchemaQuery(options); + LOGGER.info(() -> "Running schema inference using limited query '" + limitQuery + "' for the s3 source."); + try (final Connection connection = new ExasolConnectionFactory(options).getConnection(); + final Statement statement = connection.createStatement()) { + final StructType schema = getSparkSchema(statement.executeQuery(limitQuery)); + LOGGER.info(() -> "Inferred schema as '" + schema.toString() + "' for the s3 source."); + return schema; + } catch (final SQLException exception) { + throw new ExasolConnectionException(ExaError.messageBuilder("E-SEC-15") + .message("Could not run the limit query {{limitQuery}} to infer the schema.", limitQuery) + .mitigation("Please check that connection properties and original query are correct.").toString(), + exception); + } + } + + private String generateInferSchemaQuery(final ExasolOptions options) { + final Select select = StatementFactory.getInstance().select(); + select.all().from().table(""); + if (options.hasQuery()) { + select.limit(1); + } + final StringRendererConfig rendererConfig = StringRendererConfig.builder().quoteIdentifiers(true).build(); + final SelectRenderer renderer = new SelectRenderer(rendererConfig); + select.accept(renderer); + return renderer.render().replace("\"\"", getTableOrQuery(options)); + } + + private String getTableOrQuery(final ExasolOptions options) { + if (options.hasTable()) { + return options.getTable(); + } else { + return "(" + options.getQuery() + ")"; + } + } + + private StructType getSparkSchema(final ResultSet resultSet) { + try { + final ResultSetMetaData metadata = resultSet.getMetaData(); + final int numberOfColumns = metadata.getColumnCount(); + final List columns = new ArrayList<>(numberOfColumns); + for (int i = 1; i <= numberOfColumns; i++) { + columns.add(ColumnDescription.builder() // + .name(metadata.getColumnLabel(i)) // + .type(metadata.getColumnType(i)) // + .precision(metadata.getPrecision(i)) // + .scale(metadata.getScale(i)) // + .isSigned(metadata.isSigned(i)) // + .isNullable(metadata.isNullable(i) != ResultSetMetaData.columnNoNulls) // + .build()); + + } + return new SchemaConverter().convert(columns); + } catch (final SQLException exception) { + throw new ExasolConnectionException( + ExaError.messageBuilder("E-SEC-16") + .message("Could not create Spark schema from provided Exasol SQL query or table name.") + .mitigation("Please make sure that Exasol SQL query or table have columns.").toString(), + exception); + } + } + +} diff --git a/exasol-s3/src/main/java/com/exasol/spark/s3/UUIDS3BucketKeyPathProvider.java b/exasol-s3/src/main/java/com/exasol/spark/s3/UUIDS3BucketKeyPathProvider.java new file mode 100644 index 00000000..a46b2c04 --- /dev/null +++ b/exasol-s3/src/main/java/com/exasol/spark/s3/UUIDS3BucketKeyPathProvider.java @@ -0,0 +1,24 @@ +package com.exasol.spark.s3; + +import java.util.UUID; + +/** + * An implementation of {@link S3BucketKeyPathProvider} that uses {@code UUID} prefixes for intermediate write path. + * + * It creates {@code S3} write path as following {@code -/}. + */ +public class UUIDS3BucketKeyPathProvider implements S3BucketKeyPathProvider { + private final String applicationId; + + public UUIDS3BucketKeyPathProvider(final String applicationId) { + this.applicationId = applicationId; + } + + @Override + public String getS3BucketKeyForWriteLocation(final String queryId) { + final StringBuilder builder = new StringBuilder(); + builder.append(UUID.randomUUID()).append("-").append(this.applicationId).append("/").append(queryId); + return builder.toString(); + } + +} diff --git a/exasol-s3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/exasol-s3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..0a25d7e3 --- /dev/null +++ b/exasol-s3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1 @@ +com.exasol.spark.s3.S3Source diff --git a/exasol-s3/src/test/java/com/exasol/spark/BaseIntegrationSetup.java b/exasol-s3/src/test/java/com/exasol/spark/BaseIntegrationSetup.java new file mode 100644 index 00000000..9bc8db40 --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/BaseIntegrationSetup.java @@ -0,0 +1,99 @@ +package com.exasol.spark; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.junit.jupiter.Container; + +import com.exasol.containers.ExasolContainer; +import com.exasol.dbbuilder.dialects.exasol.ExasolObjectFactory; +import com.exasol.dbbuilder.dialects.exasol.ExasolSchema; + +/** + * A base integration test class with Exasol docker container setup. + */ +public class BaseIntegrationSetup { + private static final Logger LOGGER = Logger.getLogger(BaseIntegrationSetup.class.getName()); + private static final String DEFAULT_DOCKER_IMAGE = "8.18.1"; + + @Container + protected static final ExasolContainer> EXASOL = new ExasolContainer<>( + getExasolDockerImage()).withReuse(true); + + protected static Connection connection; + protected static ExasolObjectFactory factory; + protected static ExasolSchema exasolSchema; + protected static SparkSession spark; + + @BeforeAll + public static void beforeAll() throws SQLException { + EXASOL.purgeDatabase(); + connection = EXASOL.createConnection(); + factory = new ExasolObjectFactory(connection); + spark = SparkSessionProvider.getSparkSession(createSparkConfiguration()); + createExasolSchema("DEFAULT_SCHEMA"); + } + + @AfterAll + public static void afterAll() throws SQLException { + dropExasolSchema(); + connection.close(); + spark.close(); + } + + private static void createExasolSchema(final String exasolSchemaName) { + LOGGER.fine(() -> "Creating a new Exasol schema '" + exasolSchemaName + '"'); + dropExasolSchema(); + exasolSchema = factory.createSchema(exasolSchemaName); + } + + private static void dropExasolSchema() { + if (exasolSchema != null) { + LOGGER.fine(() -> "Dropping Exasol schema '" + exasolSchema.getName() + '"'); + exasolSchema.drop(); + exasolSchema = null; + } + } + + public Map getOptionsMap() { + final Map map = Stream.of(new String[][] { // + { "host", EXASOL.getDockerNetworkInternalIpAddress() }, // + { "port", EXASOL.getMappedPort(8563) + "" }, // + { "username", EXASOL.getUsername() }, // + { "password", EXASOL.getPassword() }, // + { "fingerprint", getFingerprint() }, // + }).collect(Collectors.toMap(e -> e[0], e -> e[1])); + LOGGER.fine(() -> "Prepared options '" + map.toString() + "'."); + return map; + } + + private String getFingerprint() { + return EXASOL.getTlsCertificateFingerprint().get(); + } + + private static SparkConf createSparkConfiguration() { + return new SparkConf() // + .setMaster("local[*]") // + .setAppName("Tests") // + .set("spark.ui.enabled", "false") // + .set("spark.app.id", getRandomAppId()) // + .set("spark.driver.host", "localhost"); + } + + private static String getRandomAppId() { + return "SparkAppID" + (int) (Math.random() * 1000 + 1); + } + + private static String getExasolDockerImage() { + return System.getProperty("com.exasol.dockerdb.image", DEFAULT_DOCKER_IMAGE); + } + +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/SparkSessionProvider.java b/exasol-s3/src/test/java/com/exasol/spark/SparkSessionProvider.java new file mode 100644 index 00000000..0c66632b --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/SparkSessionProvider.java @@ -0,0 +1,41 @@ +package com.exasol.spark; + +import org.apache.spark.SparkConf; +import org.apache.spark.sql.SparkSession; + +/** + * A class that provides Spark session for tests. + */ +public final class SparkSessionProvider { + private static volatile SparkSession sparkSession; + + private SparkSessionProvider() { + // Intentionally private + } + + public static SparkSession getSparkSession(final SparkConf sparkConf) { + final SparkSession result = sparkSession; + if (result != null && !isSparkContextStopped()) { + return result; + } + synchronized (SparkSessionProvider.class) { + if (sparkSession == null || isSparkContextStopped()) { + sparkSession = createSparkSession(sparkConf); + } + return sparkSession; + } + } + + private static boolean isSparkContextStopped() { + return sparkSession.sparkContext().isStopped(); + } + + private static SparkSession createSparkSession(final SparkConf sparkConf) { + SparkSession.Builder sparkSessionBuilder = SparkSession.builder(); + if (sparkConf != null) { + sparkSessionBuilder = sparkSessionBuilder.config(sparkConf); + } + return sparkSessionBuilder.getOrCreate(); + } + +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/ExasolWriteBuilderProviderIT.java b/exasol-s3/src/test/java/com/exasol/spark/s3/ExasolWriteBuilderProviderIT.java new file mode 100644 index 00000000..682ee4cc --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/ExasolWriteBuilderProviderIT.java @@ -0,0 +1,109 @@ +package com.exasol.spark.s3; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.spark.sql.connector.write.LogicalWriteInfo; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.exasol.spark.common.ExasolOptions; +import com.exasol.spark.common.ExasolValidationException; + +import io.netty.util.internal.ThreadLocalRandom; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +@ExtendWith(MockitoExtension.class) +class ExasolWriteBuilderProviderIT extends S3IntegrationTestSetup { + static final String s3BucketKey = "testS3BucketKey"; + final String applicationId = "spark-test-app-id"; + + final S3BucketKeyPathProvider s3BucketKeyPathProvider = new TestS3BucketKeyPathProvider(applicationId); + final ExasolOptions options = ExasolOptions.from(new CaseInsensitiveStringMap(getMapWithTable())); + + @Mock + private StructType schema; + + @AfterEach + void afterEach() { + try (final S3FileSystem s3FileSystem = S3FileSystem.fromOptions(options)) { + s3FileSystem.deleteKeys(DEFAULT_BUCKET_NAME, s3BucketKey + "-" + applicationId); + } + } + + @Test + void testValidateWritePathWhenEmpty() { + final ExasolWriteBuilderProvider writeBuilderProvider = new ExasolWriteBuilderProvider(options, + s3BucketKeyPathProvider); + assertDoesNotThrow(() -> writeBuilderProvider.createWriteBuilder(schema, getLogicalWriteInfo("queryId1"))); + } + + @Test + void testThrowsValidateWritePathWhenNotEmpty() throws IOException { + final String bucketKeyPrefix = s3BucketKey + "-" + applicationId + "/queryIdTest"; + PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(DEFAULT_BUCKET_NAME) + .key(bucketKeyPrefix + "/hello.txt").build(); + s3Client.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10))); + final ExasolWriteBuilderProvider writeBuilderProvider = new ExasolWriteBuilderProvider(options, + s3BucketKeyPathProvider); + final LogicalWriteInfo info = getLogicalWriteInfo("queryIdTest"); + assertThrows(ExasolValidationException.class, () -> writeBuilderProvider.createWriteBuilder(schema, info)); + } + + private Map getMapWithTable() { + final Map map = new HashMap<>(getSparkOptions()); + map.put("table", "T1"); + return map; + } + + private static ByteBuffer getRandomByteBuffer(int size) throws IOException { + byte[] b = new byte[size]; + ThreadLocalRandom.current().nextBytes(b); + return ByteBuffer.wrap(b); + } + + private LogicalWriteInfo getLogicalWriteInfo(final String queryId) { + return new LogicalWriteInfo() { + @Override + public String queryId() { + return queryId; + } + + @Override + public StructType schema() { + return schema; + } + + @Override + public CaseInsensitiveStringMap options() { + return new CaseInsensitiveStringMap(Collections.emptyMap()); + } + }; + } + + private static class TestS3BucketKeyPathProvider implements S3BucketKeyPathProvider { + private final String appId; + + public TestS3BucketKeyPathProvider(final String appId) { + this.appId = appId; + } + + @Override + public String getS3BucketKeyForWriteLocation(final String queryId) { + return s3BucketKey + "-" + this.appId + "/" + queryId; + } + } + +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/LocalstackS3WithReuse.java b/exasol-s3/src/test/java/com/exasol/spark/s3/LocalstackS3WithReuse.java new file mode 100644 index 00000000..464bd8ef --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/LocalstackS3WithReuse.java @@ -0,0 +1,32 @@ +package com.exasol.spark.s3; + +import java.util.logging.Logger; + +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.TestcontainersConfiguration; + +/** + * Reusable version of {@code localstack} container. + * + * Using {@code .withReuse(true)} on main container does not work. + */ +public final class LocalstackS3WithReuse extends LocalStackContainer { + private static final Logger LOGGER = Logger.getLogger(LocalstackS3WithReuse.class.getName()); + + public LocalstackS3WithReuse(final DockerImageName dockerImageName) { + super(dockerImageName); + withServices(Service.S3); + withReuse(true); + } + + @Override + public void stop() { + if (this.isShouldBeReused() && TestcontainersConfiguration.getInstance().environmentSupportsReuse()) { + LOGGER.warning("Leaving container running since reuse is enabled. Don't forget to stop and remove " + + "the container manually using docker rm -f CONTAINER_ID."); + } else { + super.stop(); + } + } +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/S3CleanupIT.java b/exasol-s3/src/test/java/com/exasol/spark/s3/S3CleanupIT.java new file mode 100644 index 00000000..0e8d4281 --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/S3CleanupIT.java @@ -0,0 +1,248 @@ +package com.exasol.spark.s3; + +import static com.exasol.matcher.ResultSetStructureMatcher.table; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.*; + +import org.apache.spark.SparkConf; +import org.apache.spark.SparkException; +import org.apache.spark.api.java.function.*; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.*; +import org.testcontainers.junit.jupiter.Testcontainers; + +import com.exasol.dbbuilder.dialects.Table; +import com.exasol.spark.SparkSessionProvider; + +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +/** + * This suite tests for cleanup processes at the end of Spark jobs. + * + * For that, we start Spark session with local mode {@code local[*]} with multiple threads for each test unit to force + * the job end cleanup calls. + * + * We use helper {@link TaskFailureStateCounter} class which is a singleton because we will count the number of Spark + * task failures concurrently from multiple threads. With this, singleton class variables are shared among {@code JVM} + * threads. On the other hand, using here instance object would not work because it will be serialized to Spark tasks + * and a copy of an instance would be created on each thread and they will be different from each other. + */ +@Tag("integration") +@Testcontainers +class S3CleanupIT extends S3IntegrationTestSetup { + private final int MAX_ALLOWED_SPARK_TASK_FAILURES = 3; + + private static Table table; + + private final SparkConf conf = new SparkConf() // + .setMaster("local[*," + this.MAX_ALLOWED_SPARK_TASK_FAILURES + "]") // + .setAppName("S3CleanupTests") // + .set("spark.ui.enabled", "false") // + .set("spark.driver.host", "localhost"); + + @BeforeAll + static void startAll() { + table = exasolSchema.createTableBuilder("table_cleanup") // + .column("c1", "CHAR") // + .build() // + .bulkInsert(Stream.of("1", "2", "3").map(n -> Arrays.asList(n))); + spark.stop(); + } + + @BeforeEach + void beforeEach() { + spark = SparkSessionProvider.getSparkSession(this.conf); + } + + @AfterEach + void afterEach() { + TaskFailureStateCounter.getInstance().clear(); + spark = SparkSessionProvider.getSparkSession(this.conf); + } + + private boolean isBucketEmpty(final String bucketName) { + final List objects = s3Client.listObjects(ListObjectsRequest.builder().bucket(bucketName).build()) + .contents(); + return objects.isEmpty(); + } + + private void assertThatBucketIsEmpty() { + spark.stop(); + assertThat(isBucketEmpty(DEFAULT_BUCKET_NAME), equalTo(true)); + } + + private Dataset getSparkDataset() { + return spark.read() // + .format("exasol-s3") // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load(); + } + + @Test + void testSourceSuccessJobEndCleanup() { + final MapFunction firstString = row -> row.getString(0); + final Dataset df = getSparkDataset().map(firstString, Encoders.STRING()); + assertThat(df.collectAsList(), contains("1", "2", "3")); + assertThatBucketIsEmpty(); + } + + @Test + void testSourceSingleMapTaskFailureJobEndCleanup() { + final MapFunction failOnValue1 = row -> { + final int value = Integer.valueOf(row.getString(0)); + final String message = "The filter task value '" + value + "'."; + final TaskFailureStateCounter counter = TaskFailureStateCounter.getInstance(); + if (value == 1 && counter.getCount() == 0) { + counter.increment(); + throw new RuntimeException("Intentional failure, please ignore it. " + message); + } + return value; + }; + final Dataset df = getSparkDataset().map(failOnValue1, Encoders.INT()); + assertThat(df.collectAsList(), contains(1, 2, 3)); + assertThatBucketIsEmpty(); + } + + @Test + void testSourceMultiStageMapWithCacheFailureJobEndCleanup() { + final FilterFunction failOn1And3 = row -> { + final int value = Integer.valueOf(row.getString(0)); + final String message = "The filter task value '" + value + "'."; + final TaskFailureStateCounter counter = TaskFailureStateCounter.getInstance(); + if ((value == 1 || value == 3) && counter.getCount() < 2) { + counter.increment(); + throw new RuntimeException("Intentional failure, please ignore it. " + message); + } + return value == 3; + }; + final Dataset cachedDF = getSparkDataset().filter(failOn1And3).cache(); + long size = cachedDF.count(); + assertThat(size, equalTo(1L)); + // Should stay the same size = cachedDF.count(); + size = cachedDF.count(); + assertThat(size, equalTo(1L)); + assertThatBucketIsEmpty(); + } + + @Test + void testSourceMapReduceFailureJobEndCleanup() { + final MapGroupsFunction failOnKeyEven = (key, values) -> { + final String message = "The reduce task with 'even' key."; + final TaskFailureStateCounter counter = TaskFailureStateCounter.getInstance(); + if (key.equals("even") && counter.getCount() == 0) { + counter.increment(); + throw new RuntimeException("Intentional failure, please ignore it. " + message); + } + final List longs = StreamSupport + .stream(Spliterators.spliteratorUnknownSize(values, Spliterator.ORDERED), false) + .collect(Collectors.toList()); + return key + ": " + longs.toString(); + }; + final MapFunction convertToLong = row -> Integer.valueOf(row.getString(0)) * 1L; + final Dataset df = getSparkDataset() // + .map(convertToLong, Encoders.LONG()) // + .groupByKey((MapFunction) v -> (v % 2) == 0 ? "even" : "odd", Encoders.STRING()) // + .mapGroups(failOnKeyEven, Encoders.STRING()); + assertThat(df.collectAsList(), containsInAnyOrder("even: [2]", "odd: [1, 3]")); + assertThatBucketIsEmpty(); + } + + @Test + void testSourceJobAlwaysFailsJobEndCleanup() { + final MapFunction failAlways = row -> { + throw new RuntimeException("Intentional failure for all tasks. Please ignore it."); + }; + final Dataset df = getSparkDataset().map(failAlways, Encoders.INT()); + final SparkException exception = assertThrows(SparkException.class, () -> df.collectAsList()); + assertThat(exception.getMessage(), containsString("Intentional failure for all tasks.")); + assertThatBucketIsEmpty(); + } + + private Dataset getSampleSparkDataset() { + final StructType schema = new StructType() // + .add("c_str", DataTypes.StringType, false) // + .add("c_int", DataTypes.IntegerType, false) // + .add("c_double", DataTypes.DoubleType, false) // + .add("c_bool", DataTypes.BooleanType, false); + return spark.createDataFrame(Arrays.asList( // + RowFactory.create("str", 10, 3.14, true), // + RowFactory.create("abc", 20, 2.72, false) // + ), schema); + } + + @Test + void testSinkSuccessJobEndCleanup() throws SQLException { + final Table table = exasolSchema.createTableBuilder("table_cleanup_save") // + .column("c_str", "VARCHAR(3)") // + .column("c_int", "DECIMAL(9,0)") // + .column("c_double", "DOUBLE") // + .column("c_bool", "BOOLEAN") // + .build(); + getSampleSparkDataset() // + .write() // + .mode("append") // + .format("exasol-s3") // + .options(getSparkOptions()) // + .option("table", table.getFullyQualifiedName()) // + .save(); + final String query = "SELECT * FROM " + table.getFullyQualifiedName() + " ORDER BY \"c_int\" ASC"; + try (final ResultSet result = connection.createStatement().executeQuery(query)) { + assertThat(result, table().row("str", 10, 3.14, true).row("abc", 20, 2.72, false).matches()); + } + assertThatBucketIsEmpty(); + } + + @Test + void testSinkJobAlwaysFailsJobEndCleanup() { + final DataFrameWriter df = getSampleSparkDataset() // + .write() // + .mode("append") // + .format("exasol-s3") // + .options(getSparkOptions()) // + .option("table", "non_existent_table"); + final Exception exception = assertThrows(ExasolConnectionException.class, () -> df.save()); + assertThat(exception.getMessage(), containsString("Failure running the import")); + assertThatBucketIsEmpty(); + } + + /** + * This class keeps count of failures among multiple Spark runner threads. + */ + private static class TaskFailureStateCounter { + private static volatile TaskFailureStateCounter instance; + private int totalTaskFailures = 0; + + public static TaskFailureStateCounter getInstance() { + if (instance == null) { + synchronized (TaskFailureStateCounter.class) { + if (instance == null) { + instance = new TaskFailureStateCounter(); + } + } + } + return instance; + } + + public synchronized void increment() { + this.totalTaskFailures += 1; + } + + public synchronized int getCount() { + return this.totalTaskFailures; + } + + public synchronized void clear() { + this.totalTaskFailures = 0; + } + } +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/S3DataReadingIT.java b/exasol-s3/src/test/java/com/exasol/spark/s3/S3DataReadingIT.java new file mode 100644 index 00000000..0f3fa296 --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/S3DataReadingIT.java @@ -0,0 +1,135 @@ +package com.exasol.spark.s3; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +import org.apache.spark.api.java.function.*; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.types.StructType; +import org.junit.Ignore; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Testcontainers; + +import com.exasol.dbbuilder.dialects.Table; +import com.exasol.spark.common.ExasolValidationException; + +@Tag("integration") +@Testcontainers +class S3DataReadingIT extends S3IntegrationTestSetup { + + private static Table table; + private final String format = "exasol-s3"; + + @BeforeAll + static void dataReaderSetup() { + table = exasolSchema.createTableBuilder("table_transformation") // + .column("c1", "SMALLINT") // + .build() // + .bulkInsert(Stream.of(1, 2, 3, 4, 5, 6).map(n -> Arrays.asList(n))); + } + + @Ignore // there is flaky error with show + void testDataFrameShow() { + final Dataset df = spark.read() // + .format(this.format) // + .option("query", "SELECT * FROM " + table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load(); + df.show(); + assertThat(df.count(), equalTo(6L)); + } + + @Test + void testProvidedSchema() { + final StructType expectedSchema = StructType.fromDDL("col_str STRING"); + final StructType schema = spark.read() // + .schema(expectedSchema) // + .format(this.format) // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load() // + .schema(); + assertThat(schema, equalTo(expectedSchema)); + } + + @Test + void testMapTransformation() { + final MapFunction doubleValue = row -> row.getInt(0) * 2; + final Dataset df = spark.read() // + .format(this.format) // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load() // + .map(doubleValue, Encoders.INT()); + assertThat(df.collectAsList(), contains(2, 4, 6, 8, 10, 12)); + } + + @Test + void testMapPartitionsTransformation() { + final MapPartitionsFunction convertToString = it -> { + final List result = new ArrayList<>(); + while (it.hasNext()) { + result.add(String.valueOf(it.next().getInt(0))); + } + return result.iterator(); + }; + final Dataset df = spark.read() // + .format(this.format) // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load() // + .mapPartitions(convertToString, Encoders.STRING()); + assertThat(df.collectAsList(), contains("1", "2", "3", "4", "5", "6")); + } + + @Test + void testFlatMapTransformation() { + final FlatMapFunction repeatItem = row -> Arrays.asList(row.getInt(0), row.getInt(0)).iterator(); + final Dataset df = spark.read() // + .format(this.format) // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load() // + .flatMap(repeatItem, Encoders.INT()); + assertThat(df.collectAsList(), contains(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6)); + } + + @Test + void testFilterTransformation() { + final FilterFunction keepEvenNumbers = row -> (row.getInt(0) % 2) == 0 ? true : false; + final Dataset df = spark.read() // + .format(this.format) // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .load() // + .filter(keepEvenNumbers) // + .map((MapFunction) row -> row.getInt(0), Encoders.INT()); + assertThat(df.collectAsList(), contains(2, 4, 6)); + } + + @Test + void testThrowsIfNumberOfPartitionsExceedsMaximumAllowed() { + final Dataset df = spark.read() // + .format(this.format) // + .option("table", table.getFullyQualifiedName()) // + .options(getSparkOptions()) // + .option("numPartitions", "1001") // + .load(); + final ExasolValidationException exception = assertThrows(ExasolValidationException.class, + () -> df.collectAsList()); + assertThat(exception.getMessage(), containsString("exceeds the supported maximum of 1000.")); + } + +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/S3DataWritingIT.java b/exasol-s3/src/test/java/com/exasol/spark/s3/S3DataWritingIT.java new file mode 100644 index 00000000..2e0b4f19 --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/S3DataWritingIT.java @@ -0,0 +1,150 @@ +package com.exasol.spark.s3; + +import static com.exasol.matcher.ResultSetStructureMatcher.table; +import static com.exasol.matcher.TypeMatchMode.NO_JAVA_TYPE_CHECK; +import static org.apache.spark.sql.functions.col; +import static org.apache.spark.sql.functions.date_format; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.math.BigDecimal; +import java.sql.*; +import java.util.Arrays; +import java.util.List; + +import org.apache.spark.sql.*; +import org.hamcrest.Matcher; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Testcontainers; + +import com.exasol.dbbuilder.dialects.Table; +import com.exasol.spark.common.ExasolValidationException; + +@Tag("integration") +@Testcontainers +class S3DataWritingIT extends S3IntegrationTestSetup { + + private final String format = "exasol-s3"; + + private Dataset getDataset(final List values, final Encoder encoder) { + return spark.createDataset(values, encoder); + } + + private void verify(final List values, final Encoder encoder, final Table table, + final Matcher matcher) throws SQLException { + getDataset(values, encoder) // + .write() // + .mode("append") // + .format(this.format) // + .options(getSparkOptions()) // + .option("table", table.getFullyQualifiedName()) // + .save(); + verifyResultSet(table.getFullyQualifiedName(), matcher); + } + + private void verifyResultSet(final String tableName, final Matcher matcher) throws SQLException { + final String query = "SELECT * FROM " + tableName + " ORDER BY \"c1\" ASC"; + try (final ResultSet result = connection.createStatement().executeQuery(query)) { + assertThat(result, matcher); + } + } + + @Test + void testThrowsWhenSavingIfNumberOfPartitionsExceedsMaximumAllowed() { + final Table table = exasolSchema.createTable("table_write_integer_throws", "c1", "INTEGER"); + final DataFrameWriter df = getDataset(Arrays.asList(1, 2), Encoders.INT()) // + .write() // + .mode("append") // + .format(this.format) // + .options(getSparkOptions()) // + .option("numPartitions", "1351") // + .option("table", table.getFullyQualifiedName()); + final ExasolValidationException exception = assertThrows(ExasolValidationException.class, () -> df.save()); + assertThat(exception.getMessage(), startsWith("E-SEC-23")); + } + + @Test + void testThrowsIfTableParameterIsNotSet() { + final DataFrameWriter df = getDataset(Arrays.asList(1, 2), Encoders.INT()) // + .write() // + .mode("append") // + .format(this.format) // + .options(getSparkOptions()); + final ExasolValidationException exception = assertThrows(ExasolValidationException.class, () -> df.save()); + assertThat(exception.getMessage(), startsWith("E-SCCJ-10")); + } + + @Test + void testWriteBoolean() throws SQLException { + final Table table = exasolSchema.createTable("table_write_bool", "c1", "BOOLEAN"); + verify(Arrays.asList(true, false), Encoders.BOOLEAN(), table, table().row(false).row(true).matches()); + } + + @Test + void testWriteInteger() throws SQLException { + final Table table = exasolSchema.createTable("table_write_integer", "c1", "INTEGER"); + verify(Arrays.asList(-1, 0, 10, Integer.MIN_VALUE, Integer.MAX_VALUE), Encoders.INT(), table, // + table().row(Integer.MIN_VALUE).row(-1).row(0).row(10).row(Integer.MAX_VALUE) + .matches(NO_JAVA_TYPE_CHECK)); + } + + @Test + void testWriteLong() throws SQLException { + final Table table = exasolSchema.createTable("table_write_long", "c1", "DECIMAL(36,0)"); + verify(Arrays.asList(-1L, 0L, 1L, Long.MIN_VALUE, Long.MAX_VALUE), Encoders.LONG(), table, // + table().row(Long.MIN_VALUE).row(-1L).row(0L).row(1L).row(Long.MAX_VALUE).matches(NO_JAVA_TYPE_CHECK)); + } + + @Test + void testWriteDouble() throws SQLException { + final Table table = exasolSchema.createTable("table_write_double", "c1", "DOUBLE"); + verify(Arrays.asList(2.72, 3.14), Encoders.DOUBLE(), table, table().row(2.72).row(3.14).matches()); + } + + @Test + void testWriteFloat() throws SQLException { + final Table table = exasolSchema.createTable("table_write_float", "c1", "FLOAT"); + verify(Arrays.asList(0.72F, 1.11F), Encoders.FLOAT(), table, table().row(0.72).row(1.11).matches()); + } + + @Test + void testWriteBigDecimal() throws SQLException { + final Table table = exasolSchema.createTable("table_write_bigdecimal", "c1", "DECIMAL(10,3)"); + verify(Arrays.asList(new BigDecimal("12.172"), new BigDecimal("113.014")), Encoders.DECIMAL(), table, + table().row(12.172).row(113.014).matches(NO_JAVA_TYPE_CHECK)); + } + + @Test + void testWriteString() throws SQLException { + final Table table = exasolSchema.createTable("table_write_string", "c1", "VARCHAR(5)"); + verify(Arrays.asList("xyz", "abc"), Encoders.STRING(), table, table().row("abc").row("xyz").matches()); + } + + @Test + void testWriteDate() throws SQLException { + final Date date1 = Date.valueOf("2022-05-10"); + final Date date2 = Date.valueOf("2022-05-20"); + final Table table = exasolSchema.createTable("table_write_date", "c1", "DATE"); + verify(Arrays.asList(date1, date2), Encoders.DATE(), table, table().row(date1).row(date2).matches()); + } + + @Test + void testWriteTimestamp() throws SQLException { + spark.conf().set("spark.sql.session.timeZone", "UTC"); + final Timestamp ts1 = Timestamp.from(java.time.Instant.EPOCH); + final Timestamp ts2 = new Timestamp(System.currentTimeMillis()); + final Table table = exasolSchema.createTable("table_write_timestamp", "c1", "TIMESTAMP"); + getDataset(Arrays.asList(ts1, ts2), Encoders.TIMESTAMP()) // + .withColumn("value", date_format(col("value"), "yyyy-MM-dd HH:mm:ss.SSS")) // + .write() // + .mode("append") // + .format(this.format) // + .options(getSparkOptions()) // + .option("table", table.getFullyQualifiedName()) // + .save(); + verifyResultSet(table.getFullyQualifiedName(), table().withUtcCalendar().row(ts1).row(ts2).matches()); + } + +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/S3IntegrationTestSetup.java b/exasol-s3/src/test/java/com/exasol/spark/s3/S3IntegrationTestSetup.java new file mode 100644 index 00000000..3f9f55e1 --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/S3IntegrationTestSetup.java @@ -0,0 +1,95 @@ +package com.exasol.spark.s3; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container.ExecResult; +import org.testcontainers.containers.localstack.LocalStackContainer.Service; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.utility.DockerImageName; + +import com.exasol.containers.ExasolContainer; +import com.exasol.spark.BaseIntegrationSetup; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * An integration test class with {@link LocalStackContainer} S3 setup. + */ +public abstract class S3IntegrationTestSetup extends BaseIntegrationSetup { + private static final Logger LOGGER = Logger.getLogger(S3IntegrationTestSetup.class.getName()); + private static final String HOSTS_FILE = "/etc/hosts"; + protected static final String DEFAULT_BUCKET_NAME = "csvtest"; + + @Container + protected static final LocalstackS3WithReuse S3 = new LocalstackS3WithReuse( + DockerImageName.parse("localstack/localstack:2.0")); + + protected static S3Client s3Client; + + @BeforeAll + public static void setup() throws SQLException { + LOGGER.info(() -> "Created localstack S3 client with region '" + S3.getRegion() + "'."); + s3Client = S3Client.builder() // + .endpointOverride(S3.getEndpointOverride(Service.S3)) // + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create(S3.getAccessKey(), S3.getSecretKey()))) // + .region(Region.of(S3.getRegion())) // + .build(); + redirectIpAddress(EXASOL, "csvtest.s3.amazonaws.com", getS3ContainerInternalIp()); + createBucket(DEFAULT_BUCKET_NAME); + } + + public static void createBucket(final String bucketName) { + LOGGER.info(() -> "Creating S3 bucket '" + bucketName + "'."); + s3Client.createBucket(b -> b.bucket(bucketName)); + } + + public Map getSparkOptions() { + final String endpointOverride = DockerClientFactory.instance().dockerHostIpAddress() + ":" + + S3.getMappedPort(4566); + final Map options = getOptionsMap(); + options.put("awsAccessKeyId", S3.getAccessKey()); + options.put("awsSecretAccessKey", S3.getSecretKey()); + options.put("awsRegion", S3.getRegion()); + options.put("s3Bucket", DEFAULT_BUCKET_NAME); + options.put("s3PathStyleAccess", "true"); + options.put("awsEndpointOverride", endpointOverride); + options.put("useSsl", "false"); + options.put("numPartitions", "3"); + options.put("replaceLocalhostByDefaultS3Endpoint", "true"); + return options; + } + + private static void redirectIpAddress(final ExasolContainer exasolContainer, final String original, + final String redirect) { + final List commands = Arrays.asList( // + "sed -i '/amazonaws/d' " + HOSTS_FILE, // + "echo '" + redirect + " " + original + "' >> " + HOSTS_FILE); + commands.forEach(command -> { + try { + final ExecResult exitCode = exasolContainer.execInContainer("/bin/sh", "-c", command); + if (exitCode.getExitCode() != 0) { + throw new RuntimeException( + "Command to update Exasol container '" + HOSTS_FILE + "' file returned non-zero result."); + } + } catch (final InterruptedException | IOException exception) { + throw new RuntimeException("Failed to update Exasol container '" + HOSTS_FILE + "'.", exception); + } + }); + } + + private static String getS3ContainerInternalIp() { + return S3.getContainerInfo().getNetworkSettings().getNetworks().values().iterator().next().getGateway(); + } + +} diff --git a/exasol-s3/src/test/java/com/exasol/spark/s3/SchemaInferenceIT.java b/exasol-s3/src/test/java/com/exasol/spark/s3/SchemaInferenceIT.java new file mode 100644 index 00000000..48c219b8 --- /dev/null +++ b/exasol-s3/src/test/java/com/exasol/spark/s3/SchemaInferenceIT.java @@ -0,0 +1,57 @@ +package com.exasol.spark.s3; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import java.util.Arrays; + +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Testcontainers; + +import com.exasol.dbbuilder.dialects.Table; + +@Tag("integration") +@Testcontainers +class SchemaInferenceIT extends S3IntegrationTestSetup { + + @Test + void testSparkWorks() { + final StructType schema = new StructType() // + .add("col_str", DataTypes.StringType, false) // + .add("col_int", DataTypes.IntegerType, false); + final Dataset df = spark.createDataFrame(Arrays.asList(RowFactory.create("value", 10)), schema); + df.show(); + assertThat(df.count(), equalTo(1L)); + } + + @Test + void testInferSchemaFromExasolTable() { + final Table table = exasolSchema.createTable("multi_column_table", // + Arrays.asList("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8"), // + Arrays.asList("VARCHAR(10)", "INTEGER", "DATE", "DOUBLE", "BOOLEAN", "TIMESTAMP", "DECIMAL(18,2)", "CHAR(255)")); + final StructType expectedSchema = new StructType() // + .add("c1", DataTypes.StringType, true) // + .add("c2", DataTypes.LongType, true) // + .add("c3", DataTypes.DateType, true) // + .add("c4", DataTypes.DoubleType, true) // + .add("c5", DataTypes.BooleanType, true) // + .add("c6", DataTypes.TimestampType, true) // + .add("c7", DataTypes.createDecimalType(18, 2), true) // + .add("c8", DataTypes.StringType, true); + final StructType schema = spark.read() // + .format("exasol-s3") // + .option("query", "SELECT * FROM " + table.getFullyQualifiedName()) // + .options(getOptionsMap()) // + .load() // + .schema(); + assertThat(schema, equalTo(expectedSchema)); + } + + +} diff --git a/exasol-s3/src/test/resources/logging.properties b/exasol-s3/src/test/resources/logging.properties new file mode 100644 index 00000000..8c97abe9 --- /dev/null +++ b/exasol-s3/src/test/resources/logging.properties @@ -0,0 +1,6 @@ +handlers=java.util.logging.ConsoleHandler +.level=INFO +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%1$tF %1$tT.%1$tL [%4$-7s] %5$s %n +com.exasol.level=ALL diff --git a/exasol-s3/versionsMavenPluginRules.xml b/exasol-s3/versionsMavenPluginRules.xml new file mode 100644 index 00000000..35bd03d2 --- /dev/null +++ b/exasol-s3/versionsMavenPluginRules.xml @@ -0,0 +1,18 @@ + + + + + (?i).*Alpha(?:-?[\d.]+)? + (?i).*a(?:-?[\d.]+)? + (?i).*Beta(?:-?[\d.]+)? + (?i).*-B(?:-?[\d.]+)? + (?i).*-b(?:-?[\d.]+)? + (?i).*RC(?:-?[\d.]+)? + (?i).*CR(?:-?[\d.]+)? + (?i).*M(?:-?[\d.]+)? + + + + \ No newline at end of file diff --git a/parent-pom/.settings/org.eclipse.jdt.core.prefs b/parent-pom/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..8b5a9aaa --- /dev/null +++ b/parent-pom/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,502 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullable.secondary= +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.APILeak=warning +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.terminalDeprecation=warning +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=ignore +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentType=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentTypeStrict=disabled +org.eclipse.jdt.core.compiler.problem.unlikelyEqualsArgumentType=info +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unstableAutoModuleName=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false +org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647 +org.eclipse.jdt.core.formatter.align_type_members_on_columns=false +org.eclipse.jdt.core.formatter.align_variable_declarations_on_columns=false +org.eclipse.jdt.core.formatter.align_with_spaces=false +org.eclipse.jdt.core.formatter.alignment_for_additive_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_enum_constant=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package=49 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter=0 +org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type=49 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_assertion_message=0 +org.eclipse.jdt.core.formatter.alignment_for_assignment=0 +org.eclipse.jdt.core.formatter.alignment_for_bitwise_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_loops=16 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression_chain=0 +org.eclipse.jdt.core.formatter.alignment_for_enum_constants=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_for_loop_header=0 +org.eclipse.jdt.core.formatter.alignment_for_logical_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 +org.eclipse.jdt.core.formatter.alignment_for_module_statements=16 +org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_multiplicative_operator=16 +org.eclipse.jdt.core.formatter.alignment_for_parameterized_type_references=0 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_record_components=16 +org.eclipse.jdt.core.formatter.alignment_for_relational_operator=0 +org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 +org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_shift_operator=0 +org.eclipse.jdt.core.formatter.alignment_for_string_concatenation=16 +org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_record_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_type_annotations=0 +org.eclipse.jdt.core.formatter.alignment_for_type_arguments=0 +org.eclipse.jdt.core.formatter.alignment_for_type_parameters=0 +org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 +org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_after_last_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_after_package=1 +org.eclipse.jdt.core.formatter.blank_lines_before_abstract_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 +org.eclipse.jdt.core.formatter.blank_lines_before_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 +org.eclipse.jdt.core.formatter.blank_lines_before_package=0 +org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 +org.eclipse.jdt.core.formatter.blank_lines_between_statement_group_in_switch=0 +org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 +org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_lambda_body=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_record_constructor=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_record_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.comment.align_tags_descriptions_grouped=true +org.eclipse.jdt.core.formatter.comment.align_tags_names_descriptions=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.count_line_length_from_starting_position=false +org.eclipse.jdt.core.formatter.comment.format_block_comments=true +org.eclipse.jdt.core.formatter.comment.format_header=false +org.eclipse.jdt.core.formatter.comment.format_html=true +org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true +org.eclipse.jdt.core.formatter.comment.format_line_comments=true +org.eclipse.jdt.core.formatter.comment.format_source_code=true +org.eclipse.jdt.core.formatter.comment.indent_parameter_description=false +org.eclipse.jdt.core.formatter.comment.indent_root_tags=false +org.eclipse.jdt.core.formatter.comment.indent_tag_description=false +org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_between_different_tags=do not insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=do not insert +org.eclipse.jdt.core.formatter.comment.line_length=120 +org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true +org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true +org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false +org.eclipse.jdt.core.formatter.compact_else_if=true +org.eclipse.jdt.core.formatter.continuation_indentation=2 +org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off +org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on +org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=false +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_record_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true +org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_empty_lines=false +org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false +org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_type_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_additive_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_arrow_in_switch_default=insert +org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_bitwise_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_record_components=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_switch_case_expressions=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert +org.eclipse.jdt.core.formatter.insert_space_after_lambda_arrow=insert +org.eclipse.jdt.core.formatter.insert_space_after_logical_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_multiplicative_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_not_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_relational_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert +org.eclipse.jdt.core.formatter.insert_space_after_shift_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_string_concatenation=insert +org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_additive_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_case=insert +org.eclipse.jdt.core.formatter.insert_space_before_arrow_in_switch_default=insert +org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_bitwise_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_record_components=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_switch_case_expressions=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_lambda_arrow=insert +org.eclipse.jdt.core.formatter.insert_space_before_logical_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_multiplicative_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_constructor=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_record_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_record_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert +org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_relational_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_shift_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_string_concatenation=insert +org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.join_lines_in_comments=true +org.eclipse.jdt.core.formatter.join_wrapped_lines=true +org.eclipse.jdt.core.formatter.keep_annotation_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_anonymous_type_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_code_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.jdt.core.formatter.keep_enum_constant_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_enum_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_if_then_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.jdt.core.formatter.keep_lambda_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_loop_body_block_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_method_body_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_record_constructor_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_record_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.keep_simple_do_while_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_simple_for_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_simple_getter_setter_on_one_line=false +org.eclipse.jdt.core.formatter.keep_simple_while_body_on_same_line=false +org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_type_declaration_on_one_line=one_line_never +org.eclipse.jdt.core.formatter.lineSplit=120 +org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false +org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false +org.eclipse.jdt.core.formatter.number_of_blank_lines_after_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_code_block=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_end_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_blank_lines_before_code_block=0 +org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.parentheses_positions_in_annotation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_catch_clause=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_enum_constant_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_for_statment=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_if_while_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_lambda_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_delcaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_method_invocation=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_record_declaration=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_switch_statement=common_lines +org.eclipse.jdt.core.formatter.parentheses_positions_in_try_clause=common_lines +org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.jdt.core.formatter.tabulation.char=space +org.eclipse.jdt.core.formatter.tabulation.size=4 +org.eclipse.jdt.core.formatter.text_block_indentation=0 +org.eclipse.jdt.core.formatter.use_on_off_tags=false +org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=true +org.eclipse.jdt.core.formatter.wrap_before_additive_operator=true +org.eclipse.jdt.core.formatter.wrap_before_assertion_message_operator=true +org.eclipse.jdt.core.formatter.wrap_before_assignment_operator=false +org.eclipse.jdt.core.formatter.wrap_before_bitwise_operator=true +org.eclipse.jdt.core.formatter.wrap_before_conditional_operator=true +org.eclipse.jdt.core.formatter.wrap_before_logical_operator=true +org.eclipse.jdt.core.formatter.wrap_before_multiplicative_operator=true +org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true +org.eclipse.jdt.core.formatter.wrap_before_relational_operator=true +org.eclipse.jdt.core.formatter.wrap_before_shift_operator=true +org.eclipse.jdt.core.formatter.wrap_before_string_concatenation=true +org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true +org.eclipse.jdt.core.javaFormatter=org.eclipse.jdt.core.defaultJavaFormatter diff --git a/parent-pom/.settings/org.eclipse.jdt.ui.prefs b/parent-pom/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 00000000..1add06a7 --- /dev/null +++ b/parent-pom/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,205 @@ +cleanup.add_default_serial_version_id=true +cleanup.add_generated_serial_version_id=false +cleanup.add_missing_annotations=true +cleanup.add_missing_deprecated_annotations=true +cleanup.add_missing_methods=false +cleanup.add_missing_nls_tags=false +cleanup.add_missing_override_annotations=true +cleanup.add_missing_override_annotations_interface_methods=true +cleanup.add_serial_version_id=false +cleanup.always_use_blocks=true +cleanup.always_use_parentheses_in_expressions=false +cleanup.always_use_this_for_non_static_field_access=true +cleanup.always_use_this_for_non_static_method_access=false +cleanup.convert_functional_interfaces=true +cleanup.convert_to_enhanced_for_loop=true +cleanup.correct_indentation=true +cleanup.format_source_code=true +cleanup.format_source_code_changes_only=false +cleanup.insert_inferred_type_arguments=false +cleanup.make_local_variable_final=true +cleanup.make_parameters_final=true +cleanup.make_private_fields_final=true +cleanup.make_type_abstract_if_missing_method=false +cleanup.make_variable_declarations_final=true +cleanup.never_use_blocks=false +cleanup.never_use_parentheses_in_expressions=true +cleanup.organize_imports=false +cleanup.qualify_static_field_accesses_with_declaring_class=false +cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +cleanup.qualify_static_member_accesses_with_declaring_class=true +cleanup.qualify_static_method_accesses_with_declaring_class=false +cleanup.remove_private_constructors=true +cleanup.remove_redundant_modifiers=false +cleanup.remove_redundant_semicolons=true +cleanup.remove_redundant_type_arguments=true +cleanup.remove_trailing_whitespaces=true +cleanup.remove_trailing_whitespaces_all=true +cleanup.remove_trailing_whitespaces_ignore_empty=false +cleanup.remove_unnecessary_casts=true +cleanup.remove_unnecessary_nls_tags=true +cleanup.remove_unused_imports=true +cleanup.remove_unused_local_variables=false +cleanup.remove_unused_private_fields=true +cleanup.remove_unused_private_members=true +cleanup.remove_unused_private_methods=true +cleanup.remove_unused_private_types=true +cleanup.sort_members=false +cleanup.sort_members_all=false +cleanup.use_anonymous_class_creation=false +cleanup.use_blocks=true +cleanup.use_blocks_only_for_return_and_throw=false +cleanup.use_lambda=true +cleanup.use_parentheses_in_expressions=true +cleanup.use_this_for_non_static_field_access=true +cleanup.use_this_for_non_static_field_access_only_if_necessary=false +cleanup.use_this_for_non_static_method_access=false +cleanup.use_this_for_non_static_method_access_only_if_necessary=true +cleanup_profile=_Exasol +cleanup_settings_version=2 +eclipse.preferences.version=1 +editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true +formatter_profile=_Exasol +formatter_settings_version=21 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=java;javax;org;com; +org.eclipse.jdt.ui.ondemandthreshold=3 +org.eclipse.jdt.ui.staticondemandthreshold=3 +sp_cleanup.add_all=false +sp_cleanup.add_default_serial_version_id=true +sp_cleanup.add_generated_serial_version_id=false +sp_cleanup.add_missing_annotations=true +sp_cleanup.add_missing_deprecated_annotations=true +sp_cleanup.add_missing_methods=false +sp_cleanup.add_missing_nls_tags=false +sp_cleanup.add_missing_override_annotations=true +sp_cleanup.add_missing_override_annotations_interface_methods=true +sp_cleanup.add_serial_version_id=false +sp_cleanup.always_use_blocks=true +sp_cleanup.always_use_parentheses_in_expressions=true +sp_cleanup.always_use_this_for_non_static_field_access=true +sp_cleanup.always_use_this_for_non_static_method_access=false +sp_cleanup.array_with_curly=false +sp_cleanup.arrays_fill=false +sp_cleanup.bitwise_conditional_expression=false +sp_cleanup.boolean_literal=false +sp_cleanup.boolean_value_rather_than_comparison=false +sp_cleanup.break_loop=false +sp_cleanup.collection_cloning=false +sp_cleanup.comparing_on_criteria=false +sp_cleanup.comparison_statement=false +sp_cleanup.controlflow_merge=false +sp_cleanup.convert_functional_interfaces=true +sp_cleanup.convert_to_enhanced_for_loop=true +sp_cleanup.convert_to_enhanced_for_loop_if_loop_var_used=false +sp_cleanup.convert_to_switch_expressions=false +sp_cleanup.correct_indentation=true +sp_cleanup.do_while_rather_than_while=false +sp_cleanup.double_negation=false +sp_cleanup.else_if=false +sp_cleanup.embedded_if=false +sp_cleanup.evaluate_nullable=false +sp_cleanup.extract_increment=false +sp_cleanup.format_source_code=true +sp_cleanup.format_source_code_changes_only=false +sp_cleanup.hash=false +sp_cleanup.if_condition=false +sp_cleanup.insert_inferred_type_arguments=false +sp_cleanup.instanceof=false +sp_cleanup.instanceof_keyword=false +sp_cleanup.invert_equals=false +sp_cleanup.join=false +sp_cleanup.lazy_logical_operator=false +sp_cleanup.make_local_variable_final=true +sp_cleanup.make_parameters_final=true +sp_cleanup.make_private_fields_final=true +sp_cleanup.make_type_abstract_if_missing_method=false +sp_cleanup.make_variable_declarations_final=true +sp_cleanup.map_cloning=false +sp_cleanup.merge_conditional_blocks=false +sp_cleanup.multi_catch=false +sp_cleanup.never_use_blocks=false +sp_cleanup.never_use_parentheses_in_expressions=false +sp_cleanup.no_string_creation=false +sp_cleanup.no_super=false +sp_cleanup.number_suffix=false +sp_cleanup.objects_equals=false +sp_cleanup.on_save_use_additional_actions=true +sp_cleanup.one_if_rather_than_duplicate_blocks_that_fall_through=false +sp_cleanup.operand_factorization=false +sp_cleanup.organize_imports=true +sp_cleanup.overridden_assignment=false +sp_cleanup.plain_replacement=false +sp_cleanup.precompile_regex=false +sp_cleanup.primitive_comparison=false +sp_cleanup.primitive_parsing=false +sp_cleanup.primitive_rather_than_wrapper=false +sp_cleanup.primitive_serialization=false +sp_cleanup.pull_out_if_from_if_else=false +sp_cleanup.pull_up_assignment=false +sp_cleanup.push_down_negation=false +sp_cleanup.qualify_static_field_accesses_with_declaring_class=false +sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_with_declaring_class=true +sp_cleanup.qualify_static_method_accesses_with_declaring_class=false +sp_cleanup.reduce_indentation=false +sp_cleanup.redundant_comparator=false +sp_cleanup.redundant_falling_through_block_end=false +sp_cleanup.remove_private_constructors=true +sp_cleanup.remove_redundant_modifiers=false +sp_cleanup.remove_redundant_semicolons=true +sp_cleanup.remove_redundant_type_arguments=true +sp_cleanup.remove_trailing_whitespaces=true +sp_cleanup.remove_trailing_whitespaces_all=true +sp_cleanup.remove_trailing_whitespaces_ignore_empty=false +sp_cleanup.remove_unnecessary_array_creation=false +sp_cleanup.remove_unnecessary_casts=true +sp_cleanup.remove_unnecessary_nls_tags=true +sp_cleanup.remove_unused_imports=true +sp_cleanup.remove_unused_local_variables=false +sp_cleanup.remove_unused_private_fields=true +sp_cleanup.remove_unused_private_members=false +sp_cleanup.remove_unused_private_methods=true +sp_cleanup.remove_unused_private_types=true +sp_cleanup.return_expression=false +sp_cleanup.simplify_lambda_expression_and_method_ref=false +sp_cleanup.single_used_field=false +sp_cleanup.sort_members=false +sp_cleanup.sort_members_all=false +sp_cleanup.standard_comparison=false +sp_cleanup.static_inner_class=false +sp_cleanup.strictly_equal_or_different=false +sp_cleanup.stringbuffer_to_stringbuilder=false +sp_cleanup.stringbuilder=false +sp_cleanup.stringbuilder_for_local_vars=false +sp_cleanup.substring=false +sp_cleanup.switch=false +sp_cleanup.system_property=false +sp_cleanup.system_property_boolean=false +sp_cleanup.system_property_file_encoding=false +sp_cleanup.system_property_file_separator=false +sp_cleanup.system_property_line_separator=false +sp_cleanup.system_property_path_separator=false +sp_cleanup.ternary_operator=false +sp_cleanup.try_with_resource=false +sp_cleanup.unlooped_while=false +sp_cleanup.unreachable_block=false +sp_cleanup.use_anonymous_class_creation=false +sp_cleanup.use_autoboxing=false +sp_cleanup.use_blocks=true +sp_cleanup.use_blocks_only_for_return_and_throw=false +sp_cleanup.use_directly_map_method=false +sp_cleanup.use_lambda=true +sp_cleanup.use_parentheses_in_expressions=true +sp_cleanup.use_string_is_blank=false +sp_cleanup.use_this_for_non_static_field_access=true +sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=false +sp_cleanup.use_this_for_non_static_method_access=false +sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true +sp_cleanup.use_unboxing=false +sp_cleanup.use_var=false +sp_cleanup.useless_continue=false +sp_cleanup.useless_return=false +sp_cleanup.valueof_rather_than_instantiation=false diff --git a/parent-pom/pk_generated_parent.pom b/parent-pom/pk_generated_parent.pom new file mode 100644 index 00000000..499591bc --- /dev/null +++ b/parent-pom/pk_generated_parent.pom @@ -0,0 +1,235 @@ + + + 4.0.0 + com.exasol + spark-connector-parent-pom-generated-parent + ${revision} + pom + + UTF-8 + UTF-8 + 11 + + + + + Apache License + https://github.com/exasol/spark-connector/blob/main/LICENSE + repo + + + + + Exasol + opensource@exasol.com + Exasol AG + https://www.exasol.com/ + + + + scm:git:https://github.com/exasol/spark-connector.git + scm:git:https://github.com/exasol/spark-connector.git + https://github.com/exasol/spark-connector/ + + + + + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.9.1.2184 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + ${java.version} + ${java.version} + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.3.0 + + + enforce-maven + + enforce + + + + + [3.8.7,3.9.0) + + + + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.5.0 + + true + oss + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.sonatype.ossindex.maven + ossindex-maven-plugin + 3.2.0 + + + audit + package + + audit + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.1.2 + + + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} + ${test.excludeTags} + + + + org.codehaus.mojo + versions-maven-plugin + 2.16.0 + + + display-updates + package + + display-plugin-updates + display-dependency-updates + + + + + file:///${project.basedir}/versionsMavenPluginRules.xml + + + + org.basepom.maven + duplicate-finder-maven-plugin + 2.0.1 + + + default + verify + + check + + + + + true + true + true + true + true + true + false + true + true + false + + + + org.jacoco + jacoco-maven-plugin + 0.8.10 + + + prepare-agent + + prepare-agent + + + + merge-results + verify + + merge + + + + + ${project.build.directory}/ + + jacoco*.exec + + + + ${project.build.directory}/aggregate.exec + + + + report + verify + + report + + + ${project.build.directory}/aggregate.exec + + + + + + com.exasol + error-code-crawler-maven-plugin + 1.3.0 + + + verify + + verify + + + + + + io.github.zlika + reproducible-build-maven-plugin + 0.16 + + + strip-jar + package + + strip-jar + + + + + + + diff --git a/parent-pom/pom.xml b/parent-pom/pom.xml new file mode 100644 index 00000000..b0f3efde --- /dev/null +++ b/parent-pom/pom.xml @@ -0,0 +1,547 @@ + + + 4.0.0 + com.exasol + spark-connector-parent-pom + ${revision} + Spark Exasol Connector Parent POM + A connector for Apache Spark to access Exasol + https://github.com/exasol/spark-connector/ + pom + + spark-connector-parent-pom-generated-parent + com.exasol + ${revision} + pk_generated_parent.pom + + + 2.0.0 + 8 + 2.20.0 + 5.9.3 + 5.4.0 + + + + + org.scala-lang + scala-library + ${scala.version} + + + com.exasol + exasol-jdbc + 7.1.20 + + + com.exasol + sql-statement-builder-java8 + 4.5.4 + + + com.exasol + error-reporting-java8 + 1.0.1 + + + com.exasol + spark-connector-common-java + 1.1.1 + + + org.apache.spark + spark-core_${scala.compat.version} + ${spark.version} + provided + + + org.apache.thrift + libthrift + + + org.apache.thrift + libfb303 + + + org.apache.avro + avro + + + com.google.guava + guava + + + org.spark-project.spark + unused + + + log4j + log4j + + + io.netty + netty + + + net.minidev + json-smart + + + org.apache.hadoop + hadoop-client + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + + org.apache.zookeeper + zookeeper + + + org.codehaus.jackson + jackson-mapper-asl + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + com.squareup.okhttp + okhttp + + + + org.glassfish.jersey.core + jersey-common + + + + org.glassfish.jersey.media + jersey-media-jaxb + + + + org.glassfish.jersey.core + jersey-client + + + + org.glassfish.jersey.core + jersey-server + + + + org.apache.ivy + ivy + + + + commons-net + commons-net + + + javax.activation + activation + + + org.slf4j + jcl-over-slf4j + + + + org.xerial.snappy + snappy-java + + + + + org.apache.spark + spark-sql_${scala.compat.version} + ${spark.version} + provided + + + org.spark-project.spark + unused + + + org.codehaus.jackson + jackson-mapper-asl + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + org.xerial.snappy + snappy-java + + + + + com.google.guava + guava + 32.1.1-jre + provided + + + io.netty + netty-all + 4.1.94.Final + provided + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + provided + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + provided + + + net.minidev + json-smart + + + log4j + log4j + + + org.apache.commons + commons-compress + + + io.netty + netty + + + org.codehaus.jackson + jackson-mapper-asl + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-http + + + + org.eclipse.jetty + jetty-client + + + + org.eclipse.jetty + jetty-util + + + + com.squareup.okhttp + okhttp + + + + commons-net + commons-net + + + javax.servlet + * + + + javax.xml.bind + * + + + javax.ws.rs + * + + + org.slf4j + slf4j-reload4j + + + + org.xerial.snappy + snappy-java + + + + + + com.google.protobuf + protobuf-java + 3.23.4 + + + org.apache.commons + commons-text + 1.10.0 + + + com.fasterxml.woodstox + woodstox-core + 6.5.1 + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + test + + + com.exasol + test-db-builder-java + 3.4.2 + test + + + com.exasol + hamcrest-resultset-matcher + 1.6.0 + test + + + com.exasol + exasol-testcontainers + 6.6.1 + test + + + org.testcontainers + junit-jupiter + 1.18.3 + test + + + org.testcontainers + localstack + 1.18.3 + test + + + nl.jqno.equalsverifier + equalsverifier + 3.14.3 + test + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.4.1 + + + package + + shade + + + ${project.artifactId}-${project.version}-assembly + false + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + reference.conf + + + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.1 + + true + + + + org.sonatype.ossindex.maven + ossindex-maven-plugin + + + + + CVE-2023-33546 + + + CVE-2023-22946 + + + + CVE-2022-25168 + + + + CVE-2022-26612 + + + + + org.itsallcode + openfasttrace-maven-plugin + 1.6.2 + + + trace-requirements + + trace + + + + + html + ALL + true + + + + + + + spark3.4 + + true + + + 3.4.1 + 2.13.11 + 2.13 + 3.3.4 + 2.14.2 + 3.1.2 + + + exasol-jdbc + exasol-s3 + exasol-dist + + + + spark3.4-scala2.12 + + 3.4.1 + 2.12.17 + 2.12 + 3.3.4 + 2.14.2 + 3.1.2 + + + exasol-jdbc + exasol-s3 + exasol-dist + + + + spark3.3 + + 3.3.2 + 2.13.10 + 2.13 + 3.3.2 + + 2.13.4.2 + 3.1.2 + + + exasol-jdbc + exasol-dist + + + + spark3.3-scala2.12 + + 3.3.2 + 2.12.17 + 2.12 + 3.3.2 + 2.13.4.2 + 3.1.2 + + + exasol-jdbc + exasol-dist + + + + diff --git a/parent-pom/src/test/resources/logging.properties b/parent-pom/src/test/resources/logging.properties new file mode 100644 index 00000000..8c97abe9 --- /dev/null +++ b/parent-pom/src/test/resources/logging.properties @@ -0,0 +1,6 @@ +handlers=java.util.logging.ConsoleHandler +.level=INFO +java.util.logging.ConsoleHandler.level=ALL +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%1$tF %1$tT.%1$tL [%4$-7s] %5$s %n +com.exasol.level=ALL diff --git a/parent-pom/versionsMavenPluginRules.xml b/parent-pom/versionsMavenPluginRules.xml new file mode 100644 index 00000000..35bd03d2 --- /dev/null +++ b/parent-pom/versionsMavenPluginRules.xml @@ -0,0 +1,18 @@ + + + + + (?i).*Alpha(?:-?[\d.]+)? + (?i).*a(?:-?[\d.]+)? + (?i).*Beta(?:-?[\d.]+)? + (?i).*-B(?:-?[\d.]+)? + (?i).*-b(?:-?[\d.]+)? + (?i).*RC(?:-?[\d.]+)? + (?i).*CR(?:-?[\d.]+)? + (?i).*M(?:-?[\d.]+)? + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9e4b76a6..a2fe9a7a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,722 +1,50 @@ 4.0.0 - spark-connector - 1.4.1 - The Spark Exasol Connector + com.exasol + spark-connector-root + 0.0.0 + Spark Exasol Connector Root A connector for Apache Spark to access Exasol https://github.com/exasol/spark-connector/ - - spark-connector-generated-parent - com.exasol - 1.4.1 - pk_generated_parent.pom - - - 1.8 - 2.20.0 - - . - src/main/** - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - org.scala-lang - scala-library - ${scala.version} - - - com.exasol - exasol-jdbc - 7.1.19 - - - com.exasol - sql-statement-builder-java8 - 4.5.4 - - - com.exasol - error-reporting-java8 - 1.0.1 - - - org.apache.spark - spark-core_${scala.compat.version} - ${spark.version} - provided - - - org.apache.thrift - libthrift - - - org.apache.thrift - libfb303 - - - org.apache.avro - avro - - - com.google.guava - guava - - - org.spark-project.spark - unused - - - log4j - log4j - - - io.netty - netty-all - - - io.netty - netty - - - net.minidev - json-smart - - - org.apache.hadoop - hadoop-client - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - org.apache.hadoop - hadoop-client-api - - - org.apache.hadoop - hadoop-client-runtime - - - org.apache.zookeeper - zookeeper - - - org.codehaus.jackson - jackson-mapper-asl - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - com.squareup.okhttp - okhttp - - - - org.glassfish.jersey.core - jersey-common - - - - org.glassfish.jersey.media - jersey-media-jaxb - - - - org.glassfish.jersey.core - jersey-client - - - - org.glassfish.jersey.core - jersey-server - - - - org.apache.ivy - ivy - - - - commons-net - commons-net - - - javax.activation - activation - - - org.slf4j - jcl-over-slf4j - - - - - org.apache.spark - spark-sql_${scala.compat.version} - ${spark.version} - provided - - - org.spark-project.spark - unused - - - org.codehaus.jackson - jackson-mapper-asl - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - - com.google.guava - guava - 31.1-jre - provided - - - io.netty - netty-all - 4.1.92.Final - provided - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - provided - - - - org.glassfish.jersey.core - jersey-common - ${jersey.version} - provided - - - - org.glassfish.jersey.media - jersey-media-jaxb - ${jersey.version} - provided - - - - org.glassfish.jersey.core - jersey-server - ${jersey.version} - provided - - - - org.glassfish.jersey.core - jersey-client - ${jersey.version} - provided - - - - org.apache.avro - avro-mapred - 1.11.1 - provided - - - javax.servlet - * - - - org.eclipse.jetty - jetty-server - - - - org.eclipse.jetty - jetty-util - - - - - org.apache.hadoop - hadoop-client - 3.3.5 - provided - - - net.minidev - json-smart - - - log4j - log4j - - - org.apache.commons - commons-compress - - - io.netty - netty - - - org.codehaus.jackson - jackson-mapper-asl - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-http - - - - org.eclipse.jetty - jetty-client - - - - org.eclipse.jetty - jetty-util - - - - com.squareup.okhttp - okhttp - - - - commons-net - commons-net - - - javax.servlet - * - - - javax.xml.bind - * - - - org.slf4j - slf4j-reload4j - - - - - - com.google.protobuf - protobuf-java - 3.22.4 - - - org.apache.commons - commons-text - 1.10.0 - - - com.fasterxml.woodstox - woodstox-core - 6.5.1 - - - - org.scalatest - scalatest_${scala.compat.version} - 3.2.9 - test - - - org.scalatestplus - scalatestplus-mockito_${scala.compat.version} - 1.0.0-M2 - test - - - org.mockito - mockito-core - 5.3.1 - test - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - test - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - test - - - com.exasol - test-db-builder-java - 3.4.2 - test - - - com.exasol - hamcrest-resultset-matcher - 1.6.0 - test - - - com.exasol - exasol-testcontainers - 6.5.2 - test - - + pom - - net.alchim31.maven - scala-maven-plugin - 4.8.1 - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-scaladocs - verify - - doc - doc-jar - - - - - ${scala.version} - ${scala.compat.version} - true - true - incremental - - -unchecked - -deprecation - -feature - -explaintypes - -Xcheckinit - -Xfatal-warnings - -Xlint:_ - -Ywarn-dead-code - -Ywarn-numeric-widen - -Ywarn-value-discard - -Ywarn-extra-implicit - -Ywarn-unused:_ - - - -source - ${java.version} - -target - ${java.version} - -deprecation - -parameters - -Xlint:all - - - -Xmx2048m - -Xss64m - - - - org.scalameta - semanticdb-scalac_${scala.version} - 4.5.11 - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.11.0 - - ${java.version} - ${java.version} - - - - compile - - compile - - - - - - org.scalatest - scalatest-maven-plugin - 2.2.0 - - . - TestSuite.txt - -Djava.util.logging.config.file=src/test/resources/logging.properties - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/sun.util.calendar=ALL-UNNAMED - ${argLine} - ${project.build.directory}/surefire-reports - - - - test - - test - - - (?<!IT) - - - - integration-test - integration-test - - test - - - (?<=IT) - - - - org.apache.maven.plugins - maven-jar-plugin + maven-enforcer-plugin 3.3.0 - default-jar - package - - jar - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.4.1 - - - package + enforce-maven - shade + enforce - ${project.artifactId}-${project.version}-assembly - false - false - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - reference.conf - - + + + [3.8.7,3.9.0) + + - - org.apache.maven.plugins - maven-failsafe-plugin - - false - - - - org.sonatype.ossindex.maven - ossindex-maven-plugin - - - - - sonatype-2022-5732 - - sonatype-2022-6438 - - CVE-2020-8908 - - - CVE-2023-22946 - - - - - org.basepom.maven - duplicate-finder-maven-plugin - - false - false - false - - git.properties - arrow-git.properties - .*\.proto$ - - - - - com.exasol - project-keeper-maven-plugin - 2.9.7 - - - - verify - - - - - - com.exasol - artifact-reference-checker-maven-plugin - 0.4.2 - - - - verify - - - - - - org.itsallcode - openfasttrace-maven-plugin - 1.6.2 - - - trace-requirements - - trace - - - - - html - ALL - true - - - - com.diffplug.spotless - spotless-maven-plugin - 2.36.0 - - - - ${project.basedir}/.scalafmt.conf - - - - - - - check - - - - - - io.github.evis - scalafix-maven-plugin_${scala.compat.version} - 0.1.4_0.9.31 - - - com.geirsson - metaconfig-pprint_${scala.compat.version} - 0.11.1 - - - com.github.liancheng - organize-imports_${scala.compat.version} - 0.6.0 - - - com.github.vovapolu - scaluzzi_${scala.compat.version} - 0.1.23 - - - - CHECK - - - spark3.4 - - true - + spark3.3-scala2.12 - 3.4.0 - 2.13.10 - 2.13 - 2.14.2 + 3.3.2 + 2.12.17 + 2.12 + 2.13.4.2 3.1.1 + + exasol-jdbc + spark3.3 @@ -731,16 +59,26 @@ 2.13.4.2 3.1.1 + + exasol-jdbc + - spark3.3-scala2.12 + spark3.4 + + true + - 3.3.2 - 2.12.17 - 2.12 - 2.13.4.2 - 3.1.1 + 3.4.0 + 2.13.10 + 2.13 + 2.14.2 + 3.1.2 + + exasol-jdbc + exasol-s3 + diff --git a/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index 342ce441..6dda1cab 100644 --- a/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1 +1,2 @@ com.exasol.spark.DefaultSource +com.exasol.spark.s3.S3Source diff --git a/tools/createReleasePom.sh b/tools/createReleasePom.sh deleted file mode 100755 index 0b57820a..00000000 --- a/tools/createReleasePom.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -if ! command -v xmlstarlet &>/dev/null; then - echo "xmlstarlet tool is not available, please install it to continue." - exit 1 -fi - -if [[ $# -eq 0 ]]; then - echo "Please provide the pom build profile (e.g, '-Pspark3.3')" - exit 1 -fi - -PROFILE="$1" - -echo "Copying original POM file" -cp pom.xml release.xml - -PROJECT_VERSION=$(mvn --file release.xml -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec "$PROFILE") -SPARK_VERSION=$(mvn --file release.xml -q -Dexec.executable="echo" -Dexec.args='${spark.version}' --non-recursive exec:exec "$PROFILE") -SCALA_COMPAT_VERSION=$(mvn --file release.xml -q -Dexec.executable="echo" -Dexec.args='${scala.compat.version}' --non-recursive exec:exec "$PROFILE") - -echo "Updating artifactId value for release POM" -xmlstarlet edit -L --ps -N pom="http://maven.apache.org/POM/4.0.0" \ - --update "/pom:project/pom:artifactId" \ - --value "spark-connector_$SCALA_COMPAT_VERSION" release.xml - -echo "Updating version value for release POM" -xmlstarlet edit -L --ps -N pom="http://maven.apache.org/POM/4.0.0" \ - --update "/pom:project/pom:version" \ - --value "$PROJECT_VERSION-spark-$SPARK_VERSION" release.xml