diff --git a/.gitattributes b/.gitattributes index dc173f376418..1cea00f34a7c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -28,8 +28,6 @@ /examples export-ignore /examples/** export-ignore jitpack.yml export-ignore -/python export-ignore -/python/** export-ignore /docs export-ignore /docs/** export-ignore diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 58501be062b3..cb84491c703e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,12 +19,6 @@ version: 2 updates: - - package-ecosystem: "pip" - directory: "/python/" - schedule: - interval: "weekly" - day: "sunday" - open-pull-requests-limit: 5 - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/labeler.yml b/.github/labeler.yml index 8dbbaba83755..2739cd39a2f2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -43,8 +43,6 @@ API: - api/**/* CORE: - core/**/* -PYTHON: - - python/**/* PARQUET: - parquet/**/* ARROW: diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index 6e241f48aadf..2d085ec521bf 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -28,7 +28,6 @@ on: pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - - '.github/workflows/python-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.gitignore' @@ -40,7 +39,6 @@ on: - 'hive-runtime/**' - 'flink/**' - 'pig/**' - - 'python/**' - 'docs/**' - 'open-api/**' - 'format/**' diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index 9ce922b29b16..a8db7fc3352b 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -28,7 +28,6 @@ on: pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - - '.github/workflows/python-ci.yml' - '.github/workflows/spark-ci.yml' - '.github/workflows/hive-ci.yml' - '.gitignore' @@ -40,7 +39,6 @@ on: - 'hive-runtime/**' - 'spark/**' - 'pig/**' - - 'python/**' - 'docs/**' - 'open-api/**' - 'format/**' diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index 192461429b9e..5a3bea4117fc 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -28,7 +28,6 @@ on: pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - - '.github/workflows/python-ci.yml' - '.github/workflows/spark-ci.yml' - '.github/workflows/flink-ci.yml' - '.gitignore' @@ -38,7 +37,6 @@ on: - 'spark/**' - 'flink/**' - 'pig/**' - - 'python/**' - 'docs/**' - 'open-api/**' - 'format/**' diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index f44733885f8b..5c8d31ec5156 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -28,14 +28,12 @@ on: pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - - '.github/workflows/python-ci.yml' - '.github/workflows/spark-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.gitignore' - '.asf.yml' - 'dev/**' - - 'python/**' - 'docs/**' - 'open-api/**' - 'format/**' diff --git a/.github/workflows/python-ci-docs.yml b/.github/workflows/python-ci-docs.yml deleted file mode 100644 index c21c14d2abce..000000000000 --- a/.github/workflows/python-ci-docs.yml +++ /dev/null @@ -1,56 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Python Docs" -on: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - docs: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Install - working-directory: ./python/mkdocs - run: pip install -r requirements.txt - - name: Build - working-directory: ./python/mkdocs - run: mkdocs build --strict - - name: Copy - working-directory: ./python/mkdocs - run: mv ./site /tmp/site - - name: Push changes to gh-pages branch - run: | - git checkout --orphan gh-pages-tmp - git rm --quiet -rf . - cp -r /tmp/site/* . - git config --global user.name 'GitHub Actions' - git config --global user.email 'actions@github.com' - echo "py.iceberg.apache.org" > CNAME - git add --all - git commit -m 'Publish Python docs' - git push -f origin gh-pages-tmp:gh-pages || true diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml deleted file mode 100644 index d5936d47bc66..000000000000 --- a/.github/workflows/python-ci.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Python CI" -on: - push: - branches: - - 'master' - - '0.**' - tags: - - 'apache-iceberg-**' - pull_request: - paths: - - '.github/workflows/python-ci.yml' - - 'python/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - lint-and-test: - runs-on: ubuntu-22.04 - strategy: - matrix: - python: ['3.8', '3.9', '3.10', '3.11'] - - steps: - - uses: actions/checkout@v4 - - name: Install poetry - working-directory: ./python - run: make install-poetry - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - cache: poetry - cache-dependency-path: | - ./python/poetry.lock - - name: Install - working-directory: ./python - run: make install-dependencies - - name: Linters - working-directory: ./python - run: make lint - - name: Tests - working-directory: ./python - run: make test-coverage diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml deleted file mode 100644 index be6ae925322d..000000000000 --- a/.github/workflows/python-integration.yml +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Python Integration" -on: - push: - branches: - - 'master' - - '0.**' - tags: - - 'apache-iceberg-**' - pull_request: - paths: - - '.github/workflows/python-integration.yml' - - 'python/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - integration-test: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: Install - run: make install - working-directory: ./python - - name: Run integration tests - run: make test-integration - working-directory: ./python - - name: Show debug logs - if: ${{ failure() }} - run: docker-compose -f python/dev/docker-compose.yml logs diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml deleted file mode 100644 index 406b9c2e0d32..000000000000 --- a/.github/workflows/python-release.yml +++ /dev/null @@ -1,89 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Python Release" - -on: - workflow_dispatch: - inputs: - version: - description: 'Version' - type: string - default: 'master' - - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-22.04, windows-2022, macos-11 ] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install poetry - run: pip install poetry - - - name: Set version - run: python -m poetry version "${{ inputs.version }}" - working-directory: ./python - if: "${{ github.event.inputs.version != 'master' }}" - - # Publish the source distribution with the version that's in - # the repository, otherwise the tests will fail - - name: Compile source distribution - run: python3 -m poetry build --format=sdist - if: startsWith(matrix.os, 'ubuntu') - working-directory: ./python - - - name: Build wheels - uses: pypa/cibuildwheel@v2.16.0 - with: - package-dir: python/ - output-dir: wheelhouse - config-file: "python/pyproject.toml" - env: - # Ignore 32 bit architectures - CIBW_ARCHS: "auto64" - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8,<3.12" - CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==4.2.2" - CIBW_TEST_EXTRAS: "s3fs,glue" - CIBW_TEST_COMMAND: "pytest -Werror {project}/python/tests/avro/test_decoder.py" - # There is an upstream issue with installing on MacOSX - # https://github.com/pypa/cibuildwheel/issues/1603 - # Ignore tests for pypy since not all dependencies are compiled for it - # and would require a local rust build chain - CIBW_TEST_SKIP: "pp* *macosx*" - - - name: Add source distribution - if: startsWith(matrix.os, 'ubuntu') - run: ls -lah python/dist/* && cp python/dist/* wheelhouse/ - - - uses: actions/upload-artifact@v3 - with: - name: "release-${{ github.event.inputs.version }}" - path: ./wheelhouse/* diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index e34b73101d92..5114bcd90383 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -28,7 +28,6 @@ on: pull_request: paths-ignore: - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - - '.github/workflows/python-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.gitignore' @@ -40,7 +39,6 @@ on: - 'hive-runtime/**' - 'flink/**' - 'pig/**' - - 'python/**' - 'docs/**' - 'open-api/**' - 'format/**' diff --git a/.gitignore b/.gitignore index 59977da38ef5..dd0ba6ea1762 100644 --- a/.gitignore +++ b/.gitignore @@ -64,13 +64,3 @@ metastore_db/ # Spark/metastore files spark-warehouse/ derby.log - -# Python stuff -python/.mypy_cache/ -python/htmlcov -python/coverage.xml - -python/pyiceberg/avro/decoder_fast.c -python/pyiceberg/avro/*.html -python/pyiceberg/avro/*.so - diff --git a/README.md b/README.md index a0b7b00208c0..1e37c2204665 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ ![Iceberg](https://iceberg.apache.org/docs/latest/img/Iceberg-logo.png) [![](https://github.com/apache/iceberg/actions/workflows/java-ci.yml/badge.svg)](https://github.com/apache/iceberg/actions/workflows/java-ci.yml) -[![](https://github.com/apache/iceberg/actions/workflows/python-ci.yml/badge.svg)](https://github.com/apache/iceberg/actions/workflows/python-ci.yml) [![Slack](https://img.shields.io/badge/chat-on%20Slack-brightgreen.svg)](https://apache-iceberg.slack.com/) Iceberg is a high-performance format for huge analytic tables. Iceberg brings the reliability and simplicity of SQL tables to big data, while making it possible for engines like Spark, Trino, Flink, Presto, Hive and Impala to safely work with the same tables, at the same time.