Adding Pinot base build/runtime image support for Amazon Corretto and…

… MS OpenJDK (#10422)

.github/workflows/build-pinot-docker-image.yml

@@ -47,13 +47,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v2
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx
     - uses: actions/checkout@v3
     - name: Build and push the Docker image

.github/workflows/build-presto-docker-image.yml

@@ -47,13 +47,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v2
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx
     - uses: actions/checkout@v3
     - name: Build and push the Docker image

.github/workflows/build-superset-docker-image.yml

@@ -43,13 +43,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v2
       name: Set up QEMU
-    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-buildx-action@v2
       name: Set up Docker Buildx
     - uses: actions/checkout@v3
     - name: Build and push the Docker image

docker/images/pinot-base/README.md

@@ -23,42 +23,32 @@
 
 This is the base docker image to build [Apache Pinot](https://github.com/apache/pinot).
 
-## How to build a docker image
+## Build and publish the docker image
 
-Arguments:
+Here is
+the [Github Action task](https://github.com/apachepinot/pinot-fork/actions/workflows/build-pinot-docker-base-image.yml)
+to build and publish pinot base docker images.
 
-`JAVA_VERSION`: The Java Build and Runtime image version. Default is `11`
+This task can be triggered manually to build the cross platform(amd64 and arm64v8) base image.
 
-`OPENJDK_IMAGE`: Base image to use for Pinot build and runtime, e.g. `arm64v8/openjdk`. Default is `openjdk`.
+The build shell is:
 
-Usage:
+For Amazon Corretto 11:
 
 ```SHELL
-docker build -t apachepinot/pinot-base-build:openjdk11 --no-cache --network=host --build-arg JAVA_VERSION=11 -f pinot-base-build/Dockerfile .
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-build/amazoncorretto.dockerfile --tag apachepinot/pinot-base-build:11-amazoncorretto --push .
 ```
 
 ```SHELL
-docker build -t apachepinot/pinot-base-runtime:openjdk11 --no-cache --network=host --build-arg JAVA_VERSION=11 -f pinot-base-runtime/Dockerfile .
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-runtime/amazoncorretto.dockerfile --tag apachepinot/pinot-base-runtime:11-amazoncorretto --push .
 ```
 
-Note that if you are not on arm64 machine, you can still build the image by turning on the experimental feature of docker, and add `--platform linux/arm64` into the `docker build ...` script, e.g.
-
-```SHELL
-docker build -t apachepinot/pinot-base-build:openjdk11-arm64v8 --platform linux/arm64 --no-cache --network=host --build-arg JAVA_VERSION=11 --build-arg OPENJDK_IMAGE=arm64v8/openjdk -f pinot-base-build/Dockerfile .
-```
+For MS OpenJDK, the build shell is:
 
 ```SHELL
-docker build -t apachepinot/pinot-base-runtime:openjdk11-arm64v8 --platform linux/arm64 --no-cache --network=host --build-arg JAVA_VERSION=11 --build-arg OPENJDK_IMAGE=arm64v8/openjdk -f pinot-base-runtime/Dockerfile .
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-build/amazoncorretto.dockerfile --tag apachepinot/pinot-base-build:11-ms-openjdk --push .
 ```
 
-## Publish the docker image
-
-Here is the [Github Action task](https://github.com/apachepinot/pinot-fork/actions/workflows/build-pinot-docker-base-image.yml) to build and publish pinot base docker images.
-
-This task can be triggered manually to build the cross platform(amd64 and arm64v8) base image.
-
-The build shell is:
-
 ```SHELL
-docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file Dockerfile --tag apachepinot/pinot-base-build:openjdk11 --push .
+docker buildx build --no-cache --platform=linux/arm64,linux/amd64 --file pinot-base-runtime/amazoncorretto.dockerfile --tag apachepinot/pinot-base-runtime:11-ms-openjdk --push .
 ```

docker/images/pinot-base/pinot-base-build/amazoncorretto.dockerfile

@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM debian:buster-slim
+
+ARG version=11.0.18.10-1
+# In addition to installing the Amazon corretto, we also install
+# fontconfig. The folks who manage the docker hub's
+# official image library have found that font management
+# is a common usecase, and painpoint, and have
+# recommended that Java images include font support.
+#
+# See:
+#  https://github.com/docker-library/official-images/blob/master/test/tests/java-uimanager-font/container.java
+
+LABEL [email protected]
+
+RUN set -eux \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl ca-certificates gnupg software-properties-common fontconfig java-common vim wget git automake bison flex g++ libboost-all-dev libevent-dev libssl-dev libtool make pkg-config\
+    && curl -fL https://apt.corretto.aws/corretto.key | apt-key add - \
+    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
+    && mkdir -p /usr/share/man/man1 || true \
+    && apt-get update \
+    && apt-get install -y java-11-amazon-corretto-jdk=1:$version \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV LANG C.UTF-8
+ENV JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto
+
+# install maven
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
+  && wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -P /tmp \
+  && tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \
+  && rm -f /tmp/apache-maven-*.tar.gz \
+  && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+ENV MAVEN_HOME /usr/share/maven
+ENV MAVEN_CONFIG /opt/.m2
+
+# install thrift
+RUN  wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \
+     tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \
+     base_dir=`pwd` && \
+     cd /tmp/thrift-0.12.0 && \
+     ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \
+     make install
+
+CMD ["-help"]

docker/images/pinot-base/pinot-base-build/ms-openjdk.dockerfile

@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+ARG JAVA_VERSION=11
+ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu AS pinot_build_env
+
+LABEL [email protected]
+
+# extra dependency for running launcher
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends vim wget curl git automake bison flex g++ libboost-all-dev libevent-dev \
+    libssl-dev libtool make pkg-config && \
+    rm -rf /var/lib/apt/lists/*
+
+# install maven
+RUN mkdir -p /usr/share/maven /usr/share/maven/ref \
+  && wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -P /tmp \
+  && tar -xzf /tmp/apache-maven-*.tar.gz -C /usr/share/maven --strip-components=1 \
+  && rm -f /tmp/apache-maven-*.tar.gz \
+  && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn
+ENV MAVEN_HOME /usr/share/maven
+ENV MAVEN_CONFIG /opt/.m2
+
+# install thrift
+RUN  wget http://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz -O /tmp/thrift-0.12.0.tar.gz && \
+     tar xfz /tmp/thrift-0.12.0.tar.gz --directory /tmp && \
+     base_dir=`pwd` && \
+     cd /tmp/thrift-0.12.0 && \
+     ./configure --with-cpp=no --with-c_glib=no --with-java=yes --with-python=no --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-php=no && \
+     make install
+
+CMD ["-help"]

docker/images/pinot-base/pinot-base-build/Dockerfile → docker/images/pinot-base/pinot-base-build/openjdk.dockerfile

@@ -17,8 +17,8 @@
 # under the License.
 #
 ARG JAVA_VERSION=11
-ARG OPENJDK_IMAGE=openjdk
-FROM ${OPENJDK_IMAGE}:${JAVA_VERSION} AS pinot_build_env
+ARG JDK_IMAGE=openjdk
+FROM ${JDK_IMAGE}:${JAVA_VERSION} AS pinot_build_env
 
 LABEL [email protected]
 

docker/images/pinot-base/pinot-base-runtime/amazoncorretto.dockerfile

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+ARG JAVA_VERSION=11
+ARG JDK_IMAGE=amazoncorretto
+
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-al2-jdk
+
+LABEL [email protected]
+
+RUN yum update -y && \
+    yum groupinstall 'Development Tools' -y && \
+    yum install -y procps vim less wget curl git python sysstat perf && \
+    yum clean all
+
+RUN case `uname -m` in \
+    x86_64) arch=x64; ;; \
+    aarch64) arch=arm64; ;; \
+    *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \
+  esac \
+  && mkdir -p /usr/local/lib/async-profiler \
+  && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.5.1/async-profiler-2.5.1-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \
+  && ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler
+
+CMD ["bash"]

docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ARG JAVA_VERSION=11
+ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk
+
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu
+
+LABEL [email protected]
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends vim less wget curl git python sysstat procps linux-tools-generic && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN case `uname -m` in \
+    x86_64) arch=x64; ;; \
+    aarch64) arch=arm64; ;; \
+    *) echo "platform=$(uname -m) un-supported, exit ..."; exit 1; ;; \
+  esac \
+  && mkdir -p /usr/local/lib/async-profiler \
+  && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.5.1/async-profiler-2.5.1-linux-${arch}.tar.gz | tar -xz --strip-components 1 -C /usr/local/lib/async-profiler \
+  && ln -s /usr/local/lib/async-profiler/profiler.sh /usr/local/bin/async-profiler
+
+CMD ["bash"]

docker/images/pinot-base/pinot-base-runtime/Dockerfile → docker/images/pinot-base/pinot-base-runtime/openjdk.dockerfile

@@ -17,9 +17,9 @@
 # under the License.
 #
 ARG JAVA_VERSION=11
-ARG OPENJDK_IMAGE=openjdk
+ARG JDK_IMAGE=openjdk
 
-FROM ${OPENJDK_IMAGE}:${JAVA_VERSION}-jdk-slim
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-jdk-slim
 
 LABEL [email protected]