Skip to content

Commit

Permalink
Update all windows docker image to use new base
Browse files Browse the repository at this point in the history
  • Loading branch information
amoeba committed Feb 8, 2025
1 parent 4be495b commit ed31241
Show file tree
Hide file tree
Showing 10 changed files with 108 additions and 171 deletions.
4 changes: 2 additions & 2 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release
# ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-02-03
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-01-27
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=ltsc2022-KB5049983
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=ltsc2022-KB5049983

# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
# See https://github.com/conan-io/conan-docker-tools#readme and
Expand Down
51 changes: 0 additions & 51 deletions ci/docker/python-wheel-windows-test-vs2019-base.dockerfile

This file was deleted.

67 changes: 67 additions & 0 deletions ci/docker/python-wheel-windows-test-vs2022-base.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# escape=`

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
# when you update this file.

FROM mcr.microsoft.com/windows/servercore:ltsc2022

# Ensure we in a command shell and not Powershell
SHELL ["cmd", "/S", "/C"]

# Install MSVC BuildTools
#
# The set of components below (lines starting with --add) is the most minimal
# set we could find that would still compile Arrow C++.
RUN `
curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe `
&& (start /w vs_buildtools.exe --quiet --wait --norestart --nocache `
--installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" `
--add Microsoft.VisualStudio.Component.VC.CoreBuildTools `
--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
--add Microsoft.VisualStudio.Component.Windows10SDK.20348 `
--add Microsoft.VisualStudio.Component.VC.CMake.Project `
|| IF "%ERRORLEVEL%"=="3010" EXIT 0) `
&& del /q vs_buildtools.exe

# Install choco CLI
#
# We switch into Powershell just for this command and switch back to cmd
# See https://chocolatey.org/install#completely-offline-install
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
RUN `
Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
SHELL ["cmd", "/S", "/C"]

# Install git, wget, minio
RUN choco install --no-progress -r -y git wget
RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z `
--output "C:\Windows\Minio.exe"

# Install archiver to extract xz archives (for timezone database).
# Install the GCS testbench using a well-known Python version.
# NOTE: cannot use pipx's `--fetch-missing-python` because of
# https://github.com/pypa/pipx/issues/1521, therefore download Python ourselves.
RUN choco install --no-progress -r -y archiver && `
choco install -r -y --pre --no-progress python --version=3.11.9
ENV PIPX_BIN_DIR=C:\\Windows\\
ENV PIPX_PYTHON="C:\Python311\python.exe"
COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && `
storage-testbench -h
79 changes: 0 additions & 79 deletions ci/docker/python-wheel-windows-vs2019-base.dockerfile

This file was deleted.

10 changes: 5 additions & 5 deletions dev/tasks/python-wheels/github.windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@
jobs:
build:
name: "Build wheel for Windows"
runs-on: windows-2019
runs-on: windows-2022
env:
# archery uses this environment variable
PYTHON: "{{ python_version }}"
PYTHON_ABI_TAG: "{{ python_abi_tag }}"
# this is a private repository at the moment (mostly because of licensing
# consideration of windows images with visual studio), but anyone can
# recreate the image by manually building it via:
# `archery build python-wheel-windows-vs2019`
# `archery build python-wheel-windows-vs2022`
# note that we don't run docker build since there wouldn't be a cache hit
# and rebuilding the dependencies takes a fair amount of time
REPO: ghcr.io/ursacomputing/arrow
Expand Down Expand Up @@ -62,15 +62,15 @@ jobs:
run: |
cd arrow
@rem We want to use only
@rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2019
@rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2022
@rem but it doesn't use pulled caches.
@rem It always build an image from scratch.
@rem We can remove this workaround once we find a way to use
@rem pulled caches when build an image.
echo on
archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2019
archery docker pull --no-ignore-pull-failures %TEST_IMAGE_PREFIX%-wheel-windows-vs2022
if errorlevel 1 (
archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2019 || exit /B 1
archery docker build --no-pull %TEST_IMAGE_PREFIX%-wheel-windows-vs2022 || exit /B 1
)
archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2019
Expand Down
68 changes: 34 additions & 34 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -175,9 +175,9 @@ x-hierarchy:
- python-free-threaded-wheel-manylinux-test-imports
- python-wheel-manylinux-test-unittests
- python-free-threaded-wheel-manylinux-test-unittests
- python-wheel-windows-vs2019-base:
- python-wheel-windows-vs2019
- python-free-threaded-wheel-windows-vs2019
- python-wheel-windows-vs2022-base:
- python-wheel-windows-vs2022
- python-free-threaded-wheel-windows-vs2022
- python-wheel-windows-test-base:
- python-wheel-windows-test
- python-free-threaded-wheel-windows-test
Expand Down Expand Up @@ -1266,81 +1266,81 @@ services:
CHECK_UNITTESTS: "ON"
command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

python-wheel-windows-vs2019-base:
image: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python-wheel-windows-vs2022-base:
image: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
vcpkg: ${VCPKG}
context: .
dockerfile: ci/docker/python-wheel-windows-vs2019-base.dockerfile
dockerfile: ci/docker/python-wheel-windows-vs2022-base.dockerfile
# This should make the pushed images reusable, but the image gets rebuilt.
# Uncomment if no local cache is available.
# cache_from:
# - abrarov/msvc-2019:2.11.0
# - ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes: &python-wheel-windows-vs2019-volumes
# - mcr.microsoft.com/windows/servercore:ltsc2022
# - ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes: &python-wheel-windows-vs2022-volumes
- type: bind
source: .
target: "C:/arrow"

python-wheel-windows-vs2019:
image: ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python-wheel-windows-vs2022:
image: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
base: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile
dockerfile: ci/docker/python-wheel-windows-vs2022.dockerfile
# This should make the pushed images reusable, but the image gets rebuilt.
# Uncomment if no local cache is available.
# cache_from:
# - abrarov/msvc-2019:2.11.0
# - ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes: *python-wheel-windows-vs2019-volumes
# - mcr.microsoft.com/windows/servercore:ltsc2022
# - ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes: *python-wheel-windows-vs2022-volumes
command: arrow\\ci\\scripts\\python_wheel_windows_build.bat

python-free-threaded-wheel-windows-vs2019:
image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python-free-threaded-wheel-windows-vs2022:
image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
base: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile
dockerfile: ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile
# This should make the pushed images reusable, but the image gets rebuilt.
# Uncomment if no local cache is available.
# cache_from:
# - abrarov/msvc-2019:2.11.0
# - ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes: *python-wheel-windows-vs2019-volumes
# - mcr.microsoft.com/windows/servercore:ltsc2022
# - ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
volumes: *python-wheel-windows-vs2022-volumes
command: arrow\\ci\\scripts\\python_wheel_windows_build.bat

python-wheel-windows-test-base:
image: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
image: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
build:
context: .
dockerfile: ci/docker/python-wheel-windows-test-vs2019-base.dockerfile
volumes: *python-wheel-windows-vs2019-volumes
dockerfile: ci/docker/python-wheel-windows-test-vs2022-base.dockerfile
volumes: *python-wheel-windows-vs2022-volumes

python-wheel-windows-test:
image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
build:
args:
base: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-windows-test-vs2019.dockerfile
volumes: *python-wheel-windows-vs2019-volumes
dockerfile: ci/docker/python-wheel-windows-test-vs2022.dockerfile
volumes: *python-wheel-windows-vs2022-volumes
command: arrow\\ci\\scripts\\python_wheel_windows_test.bat

python-free-threaded-wheel-windows-test:
image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
build:
args:
base: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
context: .
dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile
volumes: *python-wheel-windows-vs2019-volumes
dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile
volumes: *python-wheel-windows-vs2022-volumes
command: arrow\\ci\\scripts\\python_wheel_windows_test.bat

############################## Integration #################################
Expand Down

0 comments on commit ed31241

Please sign in to comment.