Merge branch 'main' into dev/birschick-bq/tracing-base-library

birschick-bq · Feb 4, 2025 · 723deed · 723deed
2 parents 17f3d99 + 8231c41
commit 723deed
Show file tree

Hide file tree

Showing 139 changed files with 4,774 additions and 2,113 deletions.
diff --git a/.env b/.env
@@ -16,7 +16,7 @@
 # under the License.
 
 # All of the following environment variables are required to set default values
-# for the parameters in docker-compose.yml.
+# for the parameters in compose.yaml.
 
 # Default repository to pull and push images from
 REPO=apache/arrow-dev
@@ -37,14 +37,14 @@ GO=1.22.9
 ARROW_MAJOR_VERSION=14
 DOTNET=8.0
 
-# Used through docker-compose.yml and serves as the default version for the
+# Used through compose.yaml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
 VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01"
 
 # These are used to tell tests where to find services for integration testing.
-# They are valid if the services are started with the docker-compose config.
+# They are valid if the services are started with the compose config.
 ADBC_DREMIO_FLIGHTSQL_PASS=dremio123
 ADBC_DREMIO_FLIGHTSQL_URI=grpc+tcp://localhost:32010
 ADBC_DREMIO_FLIGHTSQL_USER=dremio

diff --git a/.github/workflows/dev_adbc.yml b/.github/workflows/dev_adbc.yml
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Dev ADBC
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "dev/**"
+      - ".github/workflows/dev_adbc.yml"
+  push:
+    paths:
+      - "dev/**"
+      - ".github/workflows/dev_adbc.yml"
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash -l -eo pipefail {0}
+
+jobs:
+  pre-commit:
+    name: "pre-commit"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Cache Conda
+        uses: actions/cache@v4
+        with:
+          path: ~/conda_pkgs_dir
+          key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          use-only-tar-bz2: false
+          use-mamba: true
+
+      - name: Install Dependencies
+        run: |
+          mamba install -c conda-forge \
+            --file ci/conda_env_dev.txt \
+            pytest
+
+      - name: Test
+        run: |
+          pytest -vv dev/adbc_dev/
diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml
@@ -63,7 +63,7 @@ jobs:
         env:
           PR_TITLE: ${{ github.event.pull_request.title }}
         run: |
-          python .github/workflows/dev_pr/title_check.py $(pwd)/pr_checkout "$PR_TITLE"
+          python dev/adbc_dev/title_check.py $(pwd)/pr_checkout "$PR_TITLE"
 
       # Pings make it into the commit message where they annoy the user every
       # time the commit gets pushed somewhere

diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml
@@ -23,6 +23,8 @@ on:
       - main
     paths:
       - ".github/workflows/nightly-verify.yml"
+      - "ci/docker/cpp-clang-latest.dockerfile"
+      - "ci/docker/cpp-gcc-latest.dockerfile"
       - "dev/release/verify-release-candidate.sh"
       - "dev/release/verify-release-candidate.ps1"
   schedule:

diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
@@ -300,6 +300,7 @@ jobs:
           - almalinux-9
           - debian-bookworm
           - ubuntu-jammy
+          - ubuntu-noble
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -552,23 +553,21 @@ jobs:
           ./adbc/ci/scripts/python_conda_test.sh $(pwd)/adbc $(pwd)/adbc/build
 
   python-manylinux:
-    name: "Python ${{ matrix.arch }} manylinux${{ matrix.manylinux_version }}"
-    runs-on: ubuntu-latest
+    name: "Python ${{ matrix.arch }} manylinux"
+    runs-on: ${{ matrix.runs_on }}
     needs:
       - source
     strategy:
       fail-fast: false
       matrix:
         arch: ["amd64", "arm64v8"]
-        is_pr:
-          - ${{ startsWith(github.ref, 'refs/pull/') }}
-        exclude:
-          # Don't run arm64v8 build on PRs since the build is excessively slow
-          - arch: arm64v8
-            is_pr: true
         include:
-          - {arch: amd64, platform: linux/amd64}
-          - {arch: arm64v8, platform: linux/arm64/v8}
+          - arch: amd64
+            platform: "linux/amd64"
+            runs_on: "ubuntu-latest"
+          - arch: arm64v8
+            platform: "linux/arm64/v8"
+            runs_on: "ubuntu-24.04-arm"
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -593,9 +592,6 @@ jobs:
           echo "schedule: ${{ github.event.schedule }}" >> $GITHUB_STEP_SUMMARY
           echo "ref: ${{ github.ref }}" >> $GITHUB_STEP_SUMMARY
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Build wheel
         env:
           ARCH: ${{ matrix.arch }}
@@ -656,6 +652,14 @@ jobs:
           pushd adbc
           env PYTHON=3.12 docker compose run python-wheel-manylinux-test
 
+      - name: Test wheel 3.13
+        env:
+          ARCH: ${{ matrix.arch }}
+          PLATFORM: ${{ matrix.platform }}
+        run: |
+          pushd adbc
+          env PYTHON=3.13 docker compose run python-wheel-manylinux-test
+
   python-macos:
     name: "Python ${{ matrix.arch }} macOS"
     runs-on: ${{ matrix.os }}
@@ -737,6 +741,7 @@ jobs:
           sudo ci/scripts/install_python.sh macos 3.10
           sudo ci/scripts/install_python.sh macos 3.11
           sudo ci/scripts/install_python.sh macos 3.12
+          sudo ci/scripts/install_python.sh macos 3.13
           popd
 
       - name: Build wheel
@@ -804,6 +809,16 @@ jobs:
           ./ci/scripts/python_wheel_unix_test.sh $(pwd)
           deactivate
 
+      - name: Test wheel 3.13
+        run: |
+          pushd adbc
+
+          /Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13 -m venv test-env-313
+          source test-env-313/bin/activate
+          export PYTHON_VERSION=3.13
+          ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+          deactivate
+
   python-windows:
     name: "Python ${{ matrix.python_version }} Windows"
     runs-on: windows-latest

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -759,3 +759,46 @@
 - **c/driver/postgresql**: Use Status instead of AdbcStatusCode/AdbcError in result helper (#2178)
 - **c/driver**: Use non-objects framework components in Postgres driver (#2166)
 - **c/driver/postgresql**: Use copy writer in BindStream for parameter binding (#2157)
+
+## ADBC Libraries 16 (2025-01-17)
+
+### Versions
+
+- C/C++/GLib/Go/Python/Ruby: 1.4.0
+- C#: 0.16.0
+- Java: 0.16.0
+- R: 0.16.0
+- Rust: 0.16.0
+
+### Breaking Changes
+
+- ⚠️ **rust/driver/snowflake**: return a `Result` from `Builder::from_env` when parsing fails (#2334)
+
+### New Features
+
+- **csharp/src/Client**: parse custom properties from connection string (#2352)
+- **csharp/src/Drivers**: introduce Interop.FlightSql driver (#2214)
+- **csharp/src/Drivers/Apache**: add connect and query timeout options (#2312)
+- **csharp/src/Drivers/Apache**: make Apache driver tests inheritable (#2341)
+- **rust/driver/snowflake**: add `adbc_snowflake` crate with Go driver wrapper (#2207)
+- ⚠️ **rust/driver/snowflake**: return a `Result` from `Builder::from_env` when parsing fails (#2334)
+
+### Bugfixes
+
+- **c/driver/postgresql**: don't unnecessarily COMMIT (#2412)
+- **c/driver/postgresql**: return unknown OIDs as opaque (#2450)
+- **ci**: ensure wheels are built with older manylinux (#2351)
+- **csharp/src/Apache.Arrow.Adbc/C**: export statement_execute_schema correctly (#2409)
+- **csharp/src/Drivers/Apache**: detect sever error when polling for response (#2355)
+- **csharp/src/Drivers/BigQuery**: Use job reference instead of job id to get job to avoid interference between different locations (#2433)
+- **csharp/src/Drivers/BigQuery**: ensure BigQuery DATE type is Date32 Arrow type (#2446)
+- **csharp/src/Drivers/BigQuery**: remove details to have type names match ODBC (#2431)
+- **go/adbc/driver/bigquery**: set default project and dataset for new statements (#2342)
+- **go/adbc/driver/snowflake**: update default values for fetch params (#2325)
+- **java/driver-manager**: typo (#2336)
+
+### Documentation Improvements
+
+- add related work (#2333)
+- change Flight SQL driver usage to executable example (#2395)
+- remove crosslinking to Arrow Javadocs (#2455)
diff --git a/c/cmake_modules/AdbcVersion.cmake b/c/cmake_modules/AdbcVersion.cmake
@@ -21,7 +21,7 @@
 # ------------------------------------------------------------
 # Version definitions
 
-set(ADBC_VERSION "1.4.0-SNAPSHOT")
+set(ADBC_VERSION "1.5.0-SNAPSHOT")
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ADBC_BASE_VERSION "${ADBC_VERSION}")
 string(REPLACE "." ";" _adbc_version_list "${ADBC_BASE_VERSION}")
 list(GET _adbc_version_list 0 ADBC_VERSION_MAJOR)

diff --git a/c/driver/postgresql/postgresql_test.cc b/c/driver/postgresql/postgresql_test.cc
@@ -1696,6 +1696,29 @@ TEST_F(PostgresStatementTest, SetUseCopyFalse) {
   ASSERT_EQ(reader.array->release, nullptr);
 }
 
+TEST_F(PostgresStatementTest, UnknownOid) {
+  // Regression test for https://github.com/apache/arrow-adbc/issues/2448
+  ASSERT_THAT(AdbcStatementNew(&connection, &statement, &error), IsOkStatus(&error));
+  ASSERT_THAT(AdbcStatementSetSqlQuery(
+                  &statement, "SELECT typacl FROM pg_type WHERE oid <= 6157", &error),
+              IsOkStatus(&error));
+  adbc_validation::StreamReader reader;
+  ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value,
+                                        &reader.rows_affected, &error),
+              IsOkStatus(&error));
+  ASSERT_NO_FATAL_FAILURE(reader.GetSchema());
+  ASSERT_EQ(1, reader.fields.size());
+  ASSERT_EQ(NANOARROW_TYPE_BINARY, reader.fields[0].type);
+  struct ArrowStringView extension_name = reader.fields[0].extension_name;
+  ASSERT_EQ("arrow.opaque",
+            std::string_view(extension_name.data,
+                             static_cast<size_t>(extension_name.size_bytes)));
+  struct ArrowStringView extension_metadata = reader.fields[0].extension_metadata;
+  ASSERT_EQ(R"({"type_name": "unnamed<oid:1034>", "vendor_name": "PostgreSQL"})",
+            std::string_view(extension_metadata.data,
+                             static_cast<size_t>(extension_metadata.size_bytes)));
+}
+
 struct TypeTestCase {
   std::string name;
   std::string sql_type;

diff --git a/c/driver/postgresql/result_helper.cc b/c/driver/postgresql/result_helper.cc
@@ -167,11 +167,10 @@ Status PqResultHelper::ResolveOutputTypes(PostgresTypeResolver& type_resolver,
     const Oid pg_oid = PQftype(result_, i);
     PostgresType pg_type;
     if (type_resolver.Find(pg_oid, &pg_type, &na_error) != NANOARROW_OK) {
-      Status status =
-          Status::NotImplemented("[libpq] Column #", i + 1, " (\"", PQfname(result_, i),
-                                 "\") has unknown type code ", pg_oid);
-      ClearResult();
-      return status;
+      // We couldn't look up the OID.
+      // TODO(apache/arrow-adbc#1243): issue a warning (maybe reloading the
+      // connection will load the OIDs if it was a newly created type)
+      pg_type = PostgresType::Unnamed(pg_oid);
     }
 
     root_type.AppendChild(PQfname(result_, i), pg_type);

diff --git a/c/meson.build b/c/meson.build
@@ -18,7 +18,7 @@
 project(
     'arrow-adbc',
     'c', 'cpp',
-    version: '1.4.0-SNAPSHOT',
+    version: '1.5.0-SNAPSHOT',
     license: 'Apache-2.0',
     meson_version: '>=1.3.0',
     default_options: [

diff --git a/ci/conda/meta.yaml b/ci/conda/meta.yaml
@@ -17,7 +17,7 @@
 
 package:
   name: arrow-adbc-split
-  version: 1.4.0
+  version: 1.5.0
 
 source:
   path: ../../

diff --git a/ci/conda_env_dev.txt b/ci/conda_env_dev.txt
@@ -15,8 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-commitizen
 gh>=2.32.0
 jq
 pre-commit
+pygit2
+python
+python-dotenv
 twine
diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
@@ -20,6 +20,7 @@ importlib-resources
 # nodejs is required by pyright
 nodejs >=13.0.0
 pandas
+pip
 pyarrow-all
 pyright
 pytest

diff --git a/ci/docker/cpp-clang-latest.dockerfile b/ci/docker/cpp-clang-latest.dockerfile
@@ -20,14 +20,24 @@ ARG GO
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update -y && \
-    apt-get install -y curl gnupg && \
-    echo "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm main" \
+    apt-get install -y curl gnupg jq && \
+    # llvmorg-19.1.7 ->
+    # 19.1.7 ->
+    # 19
+    latest_llvm_major_version=$( \
+      curl https://api.github.com/repos/llvm/llvm-project/releases/latest | \
+        jq -r .tag_name | \
+        cut -d- -f2 | \
+        cut -d. -f1) && \
+    echo "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${latest_llvm_major_version} main" \
          > /etc/apt/sources.list.d/llvm.list && \
-    echo "deb-src http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm main" \
+    echo "deb-src http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${latest_llvm_major_version} main" \
          >> /etc/apt/sources.list.d/llvm.list && \
     curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     apt-get update -y && \
-    apt-get install -y clang libc++abi-dev libc++-dev libomp-dev && \
+    apt-get install -y clang-${latest_llvm_major_version} libc++abi-${latest_llvm_major_version}-dev libc++-${latest_llvm_major_version}-dev libomp-${latest_llvm_major_version}-dev && \
+    ln -s /usr/bin/clang-${latest_llvm_major_version} /usr/bin/clang && \
+    ln -s /usr/bin/clang++-${latest_llvm_major_version} /usr/bin/clang++ && \
     apt-get clean
 
 RUN export DEBIAN_FRONTEND=noninteractive && \

diff --git a/ci/linux-packages/Rakefile b/ci/linux-packages/Rakefile
@@ -133,6 +133,8 @@ class ADBCPackageTask < PackageTask
       # "debian-bookworm-arm64",
       "ubuntu-jammy",
       # "ubuntu-jammy-arm64",
+      "ubuntu-noble",
+      # "ubuntu-noble-arm64",
     ]
   end
-Original file line number
+Diff line change
@@ Expand Up / @@ -20,6 +20,7 @@ importlib-resources @@
     # nodejs is required by pyright
     nodejs >=13.0.0
     pandas
+    pip
     pyarrow-all
     pyright
     pytest
@@ Expand Down @@