From 30ac2670840a9d52ee5149ad524eb7a96c5026d0 Mon Sep 17 00:00:00 2001 From: Xiang Fu Date: Tue, 14 Mar 2023 15:37:05 -0700 Subject: [PATCH] Adding Pinot base build/runtime image support for Amazon Corretto and MS OpenJDK --- .../workflows/build-pinot-docker-image.yml | 6 +- .../workflows/build-presto-docker-image.yml | 6 +- .../workflows/build-superset-docker-image.yml | 6 +- docker/images/pinot-base/README.md | 34 ++++------ .../amazoncorretto.dockerfile | 64 +++++++++++++++++++ .../pinot-base-build/ms-openjdk.dockerfile | 48 ++++++++++++++ .../{Dockerfile => openjdk.dockerfile} | 4 +- .../amazoncorretto.dockerfile | 40 ++++++++++++ .../pinot-base-runtime/ms-openjdk.dockerfile | 40 ++++++++++++ .../{Dockerfile => openjdk.dockerfile} | 4 +- 10 files changed, 217 insertions(+), 35 deletions(-) create mode 100644 docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile create mode 100644 docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile rename docker/images/pinot-base/pinot-base-build/{Dockerfile => openjdk.dockerfile} (95%) create mode 100644 docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile create mode 100644 docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile rename docker/images/pinot-base/pinot-base-runtime/{Dockerfile => openjdk.dockerfile} (95%) diff --git a/.github/workflows/build-pinot-docker-image.yml b/.github/workflows/build-pinot-docker-image.yml index 345440750e97..93ab5850a704 100644 --- a/.github/workflows/build-pinot-docker-image.yml +++ b/.github/workflows/build-pinot-docker-image.yml @@ -47,13 +47,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: docker/setup-qemu-action@v1 + - uses: docker/setup-qemu-action@v2 name: Set up QEMU - - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-buildx-action@v2 name: Set up Docker Buildx - uses: actions/checkout@v3 - name: Build and push the Docker image diff --git a/.github/workflows/build-presto-docker-image.yml b/.github/workflows/build-presto-docker-image.yml index 7ae35253615d..43b09ea5deff 100644 --- a/.github/workflows/build-presto-docker-image.yml +++ b/.github/workflows/build-presto-docker-image.yml @@ -47,13 +47,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: docker/setup-qemu-action@v1 + - uses: docker/setup-qemu-action@v2 name: Set up QEMU - - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-buildx-action@v2 name: Set up Docker Buildx - uses: actions/checkout@v3 - name: Build and push the Docker image diff --git a/.github/workflows/build-superset-docker-image.yml b/.github/workflows/build-superset-docker-image.yml index c0e871991691..55d28786252e 100644 --- a/.github/workflows/build-superset-docker-image.yml +++ b/.github/workflows/build-superset-docker-image.yml @@ -43,13 +43,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: docker/setup-qemu-action@v1 + - uses: docker/setup-qemu-action@v2 name: Set up QEMU - - uses: docker/setup-buildx-action@v1 + - uses: docker/setup-buildx-action@v2 name: Set up Docker Buildx - uses: actions/checkout@v3 - name: Build and push the Docker image diff --git a/docker/images/pinot-base/README.md b/docker/images/pinot-base/README.md index 1b4667529f0f..92430de06c68 100644 --- a/docker/images/pinot-base/README.md +++ b/docker/images/pinot-base/README.md @@ -23,42 +23,32 @@ This is the base docker image to build [Apache Pinot](https://github.com/apache/pinot). -## How to build a docker image +## Build and publish the docker image -Arguments: +Here is +the [Github Action task](https://github.com/apachepinot/pinot-fork/actions/workflows/build-pinot-docker-base-image.yml) +to build and publish pinot base docker images. -`JAVA_VERSION`: The Java Build and Runtime image version. Default is `11` +This task can be triggered manually to build the cross platform(amd64 and arm64v8) base image. -`OPENJDK_IMAGE`: Base image to use for Pinot build and runtime, e.g. `arm64v8/openjdk`. Default is `openjdk`. +The build shell is: -Usage: +For Amazon Corretto 11: ```SHELL -docker build -t apachepinot/pinot-base-build:openjdk11 --no-cache --network=host --build-arg JAVA_VERSION=11 -f pinot-base-build/Dockerfile . +docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-build/amazoncorretto.dockerfile --tag apachepinot/pinot-base-build:11-amazoncorretto --push . ``` ```SHELL -docker build -t apachepinot/pinot-base-runtime:openjdk11 --no-cache --network=host --build-arg JAVA_VERSION=11 -f pinot-base-runtime/Dockerfile . +docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-runtime/amazoncorretto.dockerfile --tag apachepinot/pinot-base-runtime:11-amazoncorretto --push . ``` -Note that if you are not on arm64 machine, you can still build the image by turning on the experimental feature of docker, and add `--platform linux/arm64` into the `docker build ...` script, e.g. - -```SHELL -docker build -t apachepinot/pinot-base-build:openjdk11-arm64v8 --platform linux/arm64 --no-cache --network=host --build-arg JAVA_VERSION=11 --build-arg OPENJDK_IMAGE=arm64v8/openjdk -f pinot-base-build/Dockerfile . -``` +For MS OpenJDK, the build shell is: ```SHELL -docker build -t apachepinot/pinot-base-runtime:openjdk11-arm64v8 --platform linux/arm64 --no-cache --network=host --build-arg JAVA_VERSION=11 --build-arg OPENJDK_IMAGE=arm64v8/openjdk -f pinot-base-runtime/Dockerfile . +docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-build/amazoncorretto.dockerfile --tag apachepinot/pinot-base-build:11-ms-openjdk --push . ``` -## Publish the docker image - -Here is the [Github Action task](https://github.com/apachepinot/pinot-fork/actions/workflows/build-pinot-docker-base-image.yml) to build and publish pinot base docker images. - -This task can be triggered manually to build the cross platform(amd64 and arm64v8) base image. - -The build shell is: - ```SHELL -docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file Dockerfile --tag apachepinot/pinot-base-build:openjdk11 --push . +docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-runtime/amazoncorretto.dockerfile --tag apachepinot/pinot-base-runtime:11-ms-openjdk --push . ``` diff --git a/docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile b/docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile new file mode 100644 index 000000000000..1f5d80f5b2b4 --- /dev/null +++ b/docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile @@ -0,0 +1,64 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM debian:buster-slim + +ARG version=11.0.18.10-1 +# In addition to installing the Amazon corretto, we also install +# fontconfig. The folks who manage the docker hub's +# official image library have found that font management +# is a common usecase, and painpoint, and have +# recommended that Java images include font support. +# +# See: +# https://github.com/docker-library/official-images/blob/master/test/tests/java-uimanager-font/container.java + +LABEL MAINTAINER=dev@pinot.apache.org + +RUN set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates gnupg software-properties-common fontconfig java-common vim wget git automake bison flex g++ libboost-all-dev libevent-dev libssl-dev libtool make pkg-config\ + && curl -fL https://apt.corretto.aws/corretto.key | apt-key add - \ + && add-apt-repository 'deb https://apt.corretto.aws stable main' \ + && mkdir -p /usr/share/man/man1 || true \ + && apt-get update \ + && apt-get install -y java-11-amazon-corretto-jdk=1:$version \ + && rm -rf /var/lib/apt/lists/* + +ENV LANG C.UTF-8 +ENV JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto + +# install maven +RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ + && wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -P /tmp \ + && tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven-*.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn +ENV MAVEN_HOME /usr/share/maven +ENV MAVEN_CONFIG /opt/.m2 + +# install thrift +RUN wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \ + tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \ + base_dir=`pwd` && \ + cd /tmp/thrift-0.12.0 && \ + ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \ + make install + +CMD ["-help"] diff --git a/docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile b/docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile new file mode 100644 index 000000000000..5f2a1086b3da --- /dev/null +++ b/docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +ARG JAVA_VERSION=11 +ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk +FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu AS pinot_build_env + +LABEL MAINTAINER=dev@pinot.apache.org + +# extra dependency for running launcher +RUN apt-get update && \ + apt-get install -y --no-install-recommends vim wget curl git automake bison flex g++ libboost-all-dev libevent-dev \ + libssl-dev libtool make pkg-config && \ + rm -rf /var/lib/apt/lists/* + +# install maven +RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ + && wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -P /tmp \ + && tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven-*.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn +ENV MAVEN_HOME /usr/share/maven +ENV MAVEN_CONFIG /opt/.m2 + +# install thrift +RUN wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \ + tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \ + base_dir=`pwd` && \ + cd /tmp/thrift-0.12.0 && \ + ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \ + make install + +CMD ["-help"] diff --git a/docker/images/pinot-base/pinot-base-build/Dockerfile b/docker/images/pinot-base/pinot-base-build/openjdk.dockerfile similarity index 95% rename from docker/images/pinot-base/pinot-base-build/Dockerfile rename to docker/images/pinot-base/pinot-base-build/openjdk.dockerfile index 5b93684c5637..9add85d193c0 100644 --- a/docker/images/pinot-base/pinot-base-build/Dockerfile +++ b/docker/images/pinot-base/pinot-base-build/openjdk.dockerfile @@ -17,8 +17,8 @@ # under the License. # ARG JAVA_VERSION=11 -ARG OPENJDK_IMAGE=openjdk -FROM ${OPENJDK_IMAGE}:${JAVA_VERSION} AS pinot_build_env +ARG JDK_IMAGE=openjdk +FROM ${JDK_IMAGE}:${JAVA_VERSION} AS pinot_build_env LABEL MAINTAINER=dev@pinot.apache.org diff --git a/docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile b/docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile new file mode 100644 index 000000000000..9b3e61f8bcde --- /dev/null +++ b/docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +ARG JAVA_VERSION=11 +ARG JDK_IMAGE=amazoncorretto + +FROM ${JDK_IMAGE}:${JAVA_VERSION}-al2-jdk + +LABEL MAINTAINER=dev@pinot.apache.org + +RUN yum update -y && \ + yum groupinstall 'Development Tools' -y && \ + yum install -y procps vim less wget curl git python sysstat perf && \ + yum clean all + +RUN case `uname -m` in \ + x86_64) arch=x64; ;; \ + aarch64) arch=arm64; ;; \ + *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \ + esac \ + && mkdir -p /usr/local/lib/async-profiler \ + && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.5.1/async-profiler-2.5.1-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \ + && ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler + +CMD ["bash"] diff --git a/docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile b/docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile new file mode 100644 index 000000000000..50a08b4979ab --- /dev/null +++ b/docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ARG JAVA_VERSION=11 +ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk + +FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu + +LABEL MAINTAINER=dev@pinot.apache.org + +RUN apt-get update && \ + apt-get install -y --no-install-recommends vim less wget curl git python sysstat procps linux-tools-generic && \ + rm -rf /var/lib/apt/lists/* + +RUN case `uname -m` in \ + x86_64) arch=x64; ;; \ + aarch64) arch=arm64; ;; \ + *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \ + esac \ + && mkdir -p /usr/local/lib/async-profiler \ + && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.5.1/async-profiler-2.5.1-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \ + && ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler + +CMD ["bash"] diff --git a/docker/images/pinot-base/pinot-base-runtime/Dockerfile b/docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile similarity index 95% rename from docker/images/pinot-base/pinot-base-runtime/Dockerfile rename to docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile index 7a8ade3016a4..8210d1af47fc 100644 --- a/docker/images/pinot-base/pinot-base-runtime/Dockerfile +++ b/docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile @@ -17,9 +17,9 @@ # under the License. # ARG JAVA_VERSION=11 -ARG OPENJDK_IMAGE=openjdk +ARG JDK_IMAGE=openjdk -FROM ${OPENJDK_IMAGE}:${JAVA_VERSION}-jdk-slim +FROM ${JDK_IMAGE}:${JAVA_VERSION}-jdk-slim LABEL MAINTAINER=dev@pinot.apache.org