diff --git a/setup_utils/__init__.py b/pypaimon/__init__.py similarity index 91% rename from setup_utils/__init__.py rename to pypaimon/__init__.py index 27dc0ac..6eff775 100644 --- a/setup_utils/__init__.py +++ b/pypaimon/__init__.py @@ -14,6 +14,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -################################################################################ +################################################################################# + +from .api import Schema + +__all__ = [ + 'Schema', -"""This module only contains utils for setup and won't be packaged.""" +] diff --git a/paimon_python_api/__init__.py b/pypaimon/api/__init__.py similarity index 100% rename from paimon_python_api/__init__.py rename to pypaimon/api/__init__.py diff --git a/paimon_python_api/catalog.py b/pypaimon/api/catalog.py similarity index 97% rename from paimon_python_api/catalog.py rename to pypaimon/api/catalog.py index a4a863f..3132159 100644 --- a/paimon_python_api/catalog.py +++ b/pypaimon/api/catalog.py @@ -18,7 +18,7 @@ from abc import ABC, abstractmethod from typing import Optional -from paimon_python_api import Table, Schema +from pypaimon.api import Table, Schema class Catalog(ABC): diff --git a/paimon_python_api/commit_message.py b/pypaimon/api/commit_message.py similarity index 100% rename from paimon_python_api/commit_message.py rename to pypaimon/api/commit_message.py diff --git a/paimon_python_api/predicate.py b/pypaimon/api/predicate.py similarity index 100% rename from paimon_python_api/predicate.py rename to pypaimon/api/predicate.py diff --git a/paimon_python_api/read_builder.py b/pypaimon/api/read_builder.py similarity index 95% rename from paimon_python_api/read_builder.py rename to pypaimon/api/read_builder.py index a031a05..68b7d46 100644 --- a/paimon_python_api/read_builder.py +++ b/pypaimon/api/read_builder.py @@ -17,7 +17,7 @@ ################################################################################# from abc import ABC, abstractmethod -from paimon_python_api import TableRead, TableScan, Predicate, PredicateBuilder +from pypaimon.api import TableRead, TableScan, Predicate, PredicateBuilder from typing import List diff --git a/paimon_python_api/split.py b/pypaimon/api/split.py similarity index 100% rename from paimon_python_api/split.py rename to pypaimon/api/split.py diff --git a/paimon_python_api/table.py b/pypaimon/api/table.py similarity index 96% rename from paimon_python_api/table.py rename to pypaimon/api/table.py index 0170cb1..7eef7b4 100644 --- a/paimon_python_api/table.py +++ b/pypaimon/api/table.py @@ -19,7 +19,7 @@ import pyarrow as pa from abc import ABC, abstractmethod -from paimon_python_api import ReadBuilder, BatchWriteBuilder +from pypaimon.api import ReadBuilder, BatchWriteBuilder from typing import Optional, List diff --git a/paimon_python_api/table_commit.py b/pypaimon/api/table_commit.py similarity index 97% rename from paimon_python_api/table_commit.py rename to pypaimon/api/table_commit.py index d76ed6a..eed8a82 100644 --- a/paimon_python_api/table_commit.py +++ b/pypaimon/api/table_commit.py @@ -17,7 +17,7 @@ ################################################################################# from abc import ABC, abstractmethod -from paimon_python_api import CommitMessage +from pypaimon.api import CommitMessage from typing import List diff --git a/paimon_python_api/table_read.py b/pypaimon/api/table_read.py similarity index 98% rename from paimon_python_api/table_read.py rename to pypaimon/api/table_read.py index f0a7b59..9fcb78c 100644 --- a/paimon_python_api/table_read.py +++ b/pypaimon/api/table_read.py @@ -22,7 +22,7 @@ from abc import ABC, abstractmethod from duckdb.duckdb import DuckDBPyConnection -from paimon_python_api import Split +from pypaimon.api import Split from typing import List, Optional diff --git a/paimon_python_api/table_scan.py b/pypaimon/api/table_scan.py similarity index 97% rename from paimon_python_api/table_scan.py rename to pypaimon/api/table_scan.py index d998462..8830676 100644 --- a/paimon_python_api/table_scan.py +++ b/pypaimon/api/table_scan.py @@ -18,7 +18,7 @@ from abc import ABC, abstractmethod from typing import List -from paimon_python_api import Split +from pypaimon.api import Split class TableScan(ABC): diff --git a/paimon_python_api/table_write.py b/pypaimon/api/table_write.py similarity index 97% rename from paimon_python_api/table_write.py rename to pypaimon/api/table_write.py index d1d39a7..8839620 100644 --- a/paimon_python_api/table_write.py +++ b/pypaimon/api/table_write.py @@ -20,7 +20,7 @@ import pyarrow as pa from abc import ABC, abstractmethod -from paimon_python_api import CommitMessage +from pypaimon.api import CommitMessage from typing import List diff --git a/paimon_python_api/write_builder.py b/pypaimon/api/write_builder.py similarity index 96% rename from paimon_python_api/write_builder.py rename to pypaimon/api/write_builder.py index 7835179..ce9cba3 100644 --- a/paimon_python_api/write_builder.py +++ b/pypaimon/api/write_builder.py @@ -17,7 +17,7 @@ ################################################################################# from abc import ABC, abstractmethod -from paimon_python_api import BatchTableCommit, BatchTableWrite +from pypaimon.api import BatchTableCommit, BatchTableWrite from typing import Optional diff --git a/paimon_python_java/__init__.py b/pypaimon/py4j/__init__.py similarity index 82% rename from paimon_python_java/__init__.py rename to pypaimon/py4j/__init__.py index 9b0d002..3da8b12 100644 --- a/paimon_python_java/__init__.py +++ b/pypaimon/py4j/__init__.py @@ -17,9 +17,9 @@ ################################################################################ from .util import constants -from .pypaimon import (Catalog, Table, ReadBuilder, TableScan, Plan, Split, TableRead, - BatchWriteBuilder, BatchTableWrite, CommitMessage, BatchTableCommit, - Predicate, PredicateBuilder) +from .java_implementation import (Catalog, Table, ReadBuilder, TableScan, Plan, Split, TableRead, + BatchWriteBuilder, BatchTableWrite, CommitMessage, BatchTableCommit, + Predicate, PredicateBuilder) __all__ = [ 'constants', diff --git a/paimon_python_java/gateway_server.py b/pypaimon/py4j/gateway_server.py similarity index 98% rename from paimon_python_java/gateway_server.py rename to pypaimon/py4j/gateway_server.py index 2061d59..5bdd1ac 100644 --- a/paimon_python_java/gateway_server.py +++ b/pypaimon/py4j/gateway_server.py @@ -22,7 +22,7 @@ import signal from subprocess import Popen, PIPE -from paimon_python_java import constants +from pypaimon.py4j import constants def on_windows(): diff --git a/paimon_python_java/java_gateway.py b/pypaimon/py4j/java_gateway.py similarity index 96% rename from paimon_python_java/java_gateway.py rename to pypaimon/py4j/java_gateway.py index 3dabcfd..41d7a0d 100644 --- a/paimon_python_java/java_gateway.py +++ b/pypaimon/py4j/java_gateway.py @@ -26,9 +26,9 @@ CallbackServerParameters) from threading import RLock -from paimon_python_java.gateway_server import launch_gateway_server_process -from paimon_python_java import constants -from paimon_python_java.util.exceptions import install_py4j_hooks +from pypaimon.py4j.gateway_server import launch_gateway_server_process +from pypaimon.py4j import constants +from pypaimon.py4j.util.exceptions import install_py4j_hooks _gateway = None _lock = RLock() diff --git a/paimon_python_java/pypaimon.py b/pypaimon/py4j/java_implementation.py similarity index 97% rename from paimon_python_java/pypaimon.py rename to pypaimon/py4j/java_implementation.py index 803540c..ebe5b8e 100644 --- a/paimon_python_java/pypaimon.py +++ b/pypaimon/py4j/java_implementation.py @@ -16,17 +16,18 @@ # limitations under the License. ################################################################################ +# pypaimon.api implementation based on Java code & py4j lib + import duckdb import pandas as pd import pyarrow as pa import ray from duckdb.duckdb import DuckDBPyConnection -from paimon_python_java.java_gateway import get_gateway -from paimon_python_java.util import java_utils, constants -from paimon_python_api import (catalog, table, read_builder, table_scan, split, table_read, - write_builder, table_write, commit_message, table_commit, Schema, - predicate) +from pypaimon.py4j.java_gateway import get_gateway +from pypaimon.py4j.util import java_utils, constants +from pypaimon.api import (catalog, table, read_builder, table_scan, split, table_read, write_builder, + table_write, commit_message, table_commit, Schema, predicate) from typing import List, Iterator, Optional, Any diff --git a/paimon_python_java/paimon-python-java-bridge/copyright.txt b/pypaimon/py4j/paimon-python-java-bridge/copyright.txt similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/copyright.txt rename to pypaimon/py4j/paimon-python-java-bridge/copyright.txt diff --git a/paimon_python_java/paimon-python-java-bridge/pom.xml b/pypaimon/py4j/paimon-python-java-bridge/pom.xml similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/pom.xml rename to pypaimon/py4j/paimon-python-java-bridge/pom.xml diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PredicationUtil.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java b/pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java rename to pypaimon/py4j/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/SchemaUtil.java diff --git a/paimon_python_java/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE b/pypaimon/py4j/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE rename to pypaimon/py4j/paimon-python-java-bridge/src/main/resources/META-INF/NOTICE diff --git a/paimon_python_java/paimon-python-java-bridge/tools/ci/log4j.properties b/pypaimon/py4j/paimon-python-java-bridge/tools/ci/log4j.properties similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/tools/ci/log4j.properties rename to pypaimon/py4j/paimon-python-java-bridge/tools/ci/log4j.properties diff --git a/paimon_python_java/paimon-python-java-bridge/tools/maven/checkstyle.xml b/pypaimon/py4j/paimon-python-java-bridge/tools/maven/checkstyle.xml similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/tools/maven/checkstyle.xml rename to pypaimon/py4j/paimon-python-java-bridge/tools/maven/checkstyle.xml diff --git a/paimon_python_java/paimon-python-java-bridge/tools/maven/suppressions.xml b/pypaimon/py4j/paimon-python-java-bridge/tools/maven/suppressions.xml similarity index 100% rename from paimon_python_java/paimon-python-java-bridge/tools/maven/suppressions.xml rename to pypaimon/py4j/paimon-python-java-bridge/tools/maven/suppressions.xml diff --git a/paimon_python_java/tests/__init__.py b/pypaimon/py4j/tests/__init__.py similarity index 100% rename from paimon_python_java/tests/__init__.py rename to pypaimon/py4j/tests/__init__.py diff --git a/paimon_python_java/tests/test_data_types.py b/pypaimon/py4j/tests/test_data_types.py similarity index 95% rename from paimon_python_java/tests/test_data_types.py rename to pypaimon/py4j/tests/test_data_types.py index 3920cd2..5fb809e 100644 --- a/paimon_python_java/tests/test_data_types.py +++ b/pypaimon/py4j/tests/test_data_types.py @@ -24,10 +24,10 @@ import pyarrow as pa import unittest -from paimon_python_api import Schema -from paimon_python_java import Catalog -from paimon_python_java.tests import utils -from paimon_python_java.util import java_utils +from pypaimon import Schema +from pypaimon.py4j import Catalog +from pypaimon.py4j.tests import utils +from pypaimon.py4j.util import java_utils from setup_utils import java_setuputils diff --git a/paimon_python_java/tests/test_preicates.py b/pypaimon/py4j/tests/test_preicates.py similarity index 99% rename from paimon_python_java/tests/test_preicates.py rename to pypaimon/py4j/tests/test_preicates.py index 7ee1a91..5b63759 100644 --- a/paimon_python_java/tests/test_preicates.py +++ b/pypaimon/py4j/tests/test_preicates.py @@ -24,9 +24,9 @@ import pandas as pd import pyarrow as pa -from paimon_python_api import Schema -from paimon_python_java import Catalog -from paimon_python_java.tests import utils +from pypaimon import Schema +from pypaimon.py4j import Catalog +from pypaimon.py4j.tests import utils from setup_utils import java_setuputils diff --git a/paimon_python_java/tests/test_write_and_read.py b/pypaimon/py4j/tests/test_write_and_read.py similarity index 98% rename from paimon_python_java/tests/test_write_and_read.py rename to pypaimon/py4j/tests/test_write_and_read.py index e2c631d..27528d1 100644 --- a/paimon_python_java/tests/test_write_and_read.py +++ b/pypaimon/py4j/tests/test_write_and_read.py @@ -24,11 +24,11 @@ import pyarrow as pa from py4j.protocol import Py4JJavaError -from paimon_python_api import Schema -from paimon_python_java import Catalog -from paimon_python_java.java_gateway import get_gateway -from paimon_python_java.tests import utils -from paimon_python_java.util import java_utils +from pypaimon import Schema +from pypaimon.py4j import Catalog +from pypaimon.py4j.java_gateway import get_gateway +from pypaimon.py4j.tests import utils +from pypaimon.py4j.util import java_utils from setup_utils import java_setuputils diff --git a/paimon_python_java/tests/utils.py b/pypaimon/py4j/tests/utils.py similarity index 97% rename from paimon_python_java/tests/utils.py rename to pypaimon/py4j/tests/utils.py index 350f80e..18d924e 100644 --- a/paimon_python_java/tests/utils.py +++ b/pypaimon/py4j/tests/utils.py @@ -19,7 +19,7 @@ import os import urllib.request -from paimon_python_java.util import constants +from pypaimon.py4j import constants def setup_hadoop_bundle_jar(hadoop_dir): diff --git a/paimon_python_java/util/__init__.py b/pypaimon/py4j/util/__init__.py similarity index 100% rename from paimon_python_java/util/__init__.py rename to pypaimon/py4j/util/__init__.py diff --git a/paimon_python_java/util/constants.py b/pypaimon/py4j/util/constants.py similarity index 100% rename from paimon_python_java/util/constants.py rename to pypaimon/py4j/util/constants.py diff --git a/paimon_python_java/util/exceptions.py b/pypaimon/py4j/util/exceptions.py similarity index 100% rename from paimon_python_java/util/exceptions.py rename to pypaimon/py4j/util/exceptions.py diff --git a/setup_utils/java_setuputils.py b/pypaimon/py4j/util/java_setuputils.py similarity index 94% rename from setup_utils/java_setuputils.py rename to pypaimon/py4j/util/java_setuputils.py index 49e2ea9..bd611fb 100755 --- a/setup_utils/java_setuputils.py +++ b/pypaimon/py4j/util/java_setuputils.py @@ -22,7 +22,6 @@ from xml.etree import ElementTree -_JAVA_IMPL_MODULE = 'paimon_python_java' _JAVA_DEPS = 'java_dependencies' _JAVA_BRIDGE = 'paimon-python-java-bridge' @@ -81,5 +80,4 @@ def _extract_bridge_version(): def _find_java_impl_dir(): this_dir = os.path.abspath(os.path.dirname(__file__)) - paimon_python_dir = os.path.dirname(this_dir) - return os.path.join(paimon_python_dir, _JAVA_IMPL_MODULE) + return os.path.dirname(this_dir) diff --git a/paimon_python_java/util/java_utils.py b/pypaimon/py4j/util/java_utils.py similarity index 97% rename from paimon_python_java/util/java_utils.py rename to pypaimon/py4j/util/java_utils.py index ce0404a..0beb527 100644 --- a/paimon_python_java/util/java_utils.py +++ b/pypaimon/py4j/util/java_utils.py @@ -18,8 +18,8 @@ import pyarrow as pa -from paimon_python_api import Schema -from paimon_python_java.java_gateway import get_gateway +from pypaimon import Schema +from pypaimon.py4j.java_gateway import get_gateway def to_j_catalog_context(catalog_options: dict): diff --git a/setup_utils/version.py b/pypaimon/version.py similarity index 100% rename from setup_utils/version.py rename to pypaimon/version.py diff --git a/setup.py b/setup.py index 999e3c5..f3090a4 100644 --- a/setup.py +++ b/setup.py @@ -19,9 +19,9 @@ import fnmatch import os import shutil -import setup_utils.java_setuputils as java_setuputils -import setup_utils.version +from pypaimon.version import __version__ as version +from pypaimon.py4j.util import java_setuputils from setuptools import Command, setup @@ -50,13 +50,14 @@ def run(self): try: PACKAGES = [ - 'paimon_python_api', - 'paimon_python_java', - 'paimon_python_java.util' + 'pypaimon', + 'pypaimon.api', + 'pypaimon.py4j', + 'pypaimon.py4j.util' ] PACKAGE_DATA = { - 'paimon_python_java': java_setuputils.get_package_data() + 'pypaimon.py4j': java_setuputils.get_package_data() } install_requires = [ @@ -72,8 +73,8 @@ def run(self): [Doc](https://paimon.apache.org/docs/master/program-api/python-api/) for usage.' setup( - name='paimon_python', - version=setup_utils.version.__version__, + name='pypaimon', + version=version, packages=PACKAGES, include_package_data=True, package_data=PACKAGE_DATA, diff --git a/tools/releasing/create_source_release.sh b/tools/releasing/create_source_release.sh index 4d07db5..7d6a8a9 100755 --- a/tools/releasing/create_source_release.sh +++ b/tools/releasing/create_source_release.sh @@ -66,7 +66,7 @@ echo "Creating source package" git clone ${PROJECT_ROOT} ${CLONE_DIR} cd ${CLONE_DIR} -JAVA_ROOT="paimon_python_java/paimon-python-java-bridge" +JAVA_ROOT="pypaimon/py4j/paimon-python-java-bridge" rsync -a \ --exclude ".DS_Store" --exclude ".asf.yaml" --exclude ".git" \ --exclude ".github" --exclude ".gitignore" --exclude ".idea" \ diff --git a/tools/releasing/update_branch_version.sh b/tools/releasing/update_branch_version.sh index 5c8fc49..9c20171 100755 --- a/tools/releasing/update_branch_version.sh +++ b/tools/releasing/update_branch_version.sh @@ -34,7 +34,7 @@ set -o nounset CURR_DIR=`pwd` BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" PROJECT_ROOT="${BASE_DIR}/../../" -SETUP_UTILS_DIR="${PROJECT_ROOT}/setup_utils" +PYPAIMON_DIR="${PROJECT_ROOT}/pypaimon" # Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root if [ ! -f ${PROJECT_ROOT}/LICENSE ]; then @@ -44,7 +44,7 @@ fi ########################### -cd ${SETUP_UTILS_DIR}/ +cd ${PYPAIMON_DIR}/ # change version perl -pi -e "s#^__version__ = \".*\"#__version__ = \"${NEW_VERSION}\"#" version.py