Add amazon linux and arm64 builds. (#264)

* Build script: * Implement build QFS on amazonlinux 2023. * Github actions: * Add amazonlinux build to linux build matrix. * Make action runner type part of the linux build matrix. * Use ubuntu-24.04-arm runner for arm64 builds. * Github Actions: * Add ubuntu 24.04, debian 12, and rockylinux 9 arm64 build jobs. * Readme: * Add amazon linux 2023 and arm builds statuses. * Wiki: * Add links to binaries for amazon linux 2023, arm builds for debian 12, ubuntu 24.04, and rocky linux 9 to binary distributions page.
quantcast · Jan 26, 2025 · 7462c8b · 7462c8b
1 parent 03f6b45
commit 7462c8b
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 43 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -15,11 +15,12 @@ jobs:
   linux-build-docker:
     strategy:
       matrix:
-        distro: [ubuntu, centos, rockylinux, debian, i386/ubuntu]
+        distro: [ubuntu, centos, rockylinux, amazonlinux, debian, i386/ubuntu]
         exclude:
           - distro: ubuntu
           - distro: centos
           - distro: rockylinux
+          - distro: amazonlinux
           - distro: debian
           - distro: i386/ubuntu
         include:
@@ -28,87 +29,134 @@ jobs:
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
+          - distro: ubuntu
+            ver: 24.04
+            codecov: no
+            btype: release
+            buser: qfsbuild
+            runs_on: ubuntu-24.04-arm
           - distro: ubuntu
             ver: 22.04
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 20.04
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 18.04
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 18.04
             codecov: yes
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 18.04
             codecov: no
             btype: debug
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 18.04
             codecov: no
             btype: release
             buser: root
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 16.04
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: ubuntu
             ver: 14.04
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: debian
             ver: 12
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
+          - distro: debian
+            ver: 12
+            codecov: no
+            btype: release
+            buser: qfsbuild
+            runs_on: ubuntu-24.04-arm
           - distro: debian
             ver: 11
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: debian
             ver: 10
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: centos
             ver: 6
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: centos
             ver: 7
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: centos
             ver: 8
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-latest
+          - distro: rockylinux
+            ver: 9
+            codecov: no
+            btype: release
+            buser: qfsbuild
+            runs_on: ubuntu-latest
           - distro: rockylinux
             ver: 9
             codecov: no
             btype: release
             buser: qfsbuild
+            runs_on: ubuntu-24.04-arm
+          - distro: amazonlinux
+            ver: 2023
+            codecov: no
+            btype: release
+            buser: qfsbuild
+            runs_on: ubuntu-latest
+          - distro: amazonlinux
+            ver: 2023
+            codecov: no
+            btype: release
+            buser: qfsbuild
+            runs_on: ubuntu-24.04-arm
           - distro: i386/ubuntu
             ver: 18.04
             codecov: no
             btype: release
             buser: qfsbuild
-    runs-on: ubuntu-latest
+            runs_on: ubuntu-latest
+    runs-on: "${{ matrix.runs_on }}"
     env:
       BUILD_OS_NAME: linux
       BUILD_RUN_DOCKER: "yes"

diff --git a/README.md b/README.md
@@ -77,21 +77,25 @@ QFS used in Quantcast production cluster.
 QFS servers have been tested on 64-bit CentOS 6 extensively and run on Linux
 variants. The QFS client tools work on OS X and Cygwin as well.
 
-| Platform      | Build Status                                                                                                                                                          |
-| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Mac OS X      | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Ubuntu 14.04  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Ubuntu 18.04  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Ubuntu 20.04  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Ubuntu 22.04  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Ubuntu 24.04  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| CentOS 6      | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| CentOS 7      | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| CentOS 8      | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Rocky Linux 9 | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Debian 10     | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Debian 11     | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
-| Debian 12     | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Platform             | Build Status                                                                                                                                                          |
+| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Mac OS X x86_64      | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Mac OS X arm64       | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Ubuntu 14.04         | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Ubuntu 18.04         | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Ubuntu 20.04         | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Ubuntu 22.04         | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Ubuntu 24.04 x86_64  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Ubuntu 24.04 arm64   | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| CentOS 6             | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| CentOS 7             | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| CentOS 8             | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Rocky Linux 9 x86_64 | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Rocky Linux 9 arm64  | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Debian 10            | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Debian 11            | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Debian 12 x86_64     | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
+| Debian 12 arm64      | [![Build Status](https://github.com/quantcast/qfs/actions/workflows/build.yml/badge.svg?branch=master)](https://github.com/quantcast/qfs/actions/workflows/build.yml) |
 
 The implementation details and features of QFS are discussed in detail in the
 project [wiki](https://github.com/quantcast/qfs/wiki/Introduction-To-QFS).

diff --git a/travis/script.sh b/travis/script.sh
@@ -32,14 +32,19 @@ DEPS_UBUNTU=$DEPS_UBUNTU' python3-dev python3-venv'
 DEPS_UBUNTU22=$DEPS_UBUNTU' golang-go'
 
 DEPS_CENTOS='gcc-c++ make git boost-devel krb5-devel'
-DEPS_CENTOS=$DEPS_CENTOS' java-openjdk java-devel'
 DEPS_CENTOS=$DEPS_CENTOS' libuuid-devel unzip sudo which openssl fuse gdb'
 
 DEPS_CENTOS_PRIOR_TO_9=' curl fuse-devel'
 DEPS_CENTOS5=$DEPS_CENTOS' cmake28 openssl101e openssl101e-devel'$DEPS_CENTOS_PRIOR_TO_9
 DEPS_CENTOS=$DEPS_CENTOS' openssl-devel cmake chrpath python3-devel'
 DEPS_CENTOS8=$DEPS_CENTOS' diffutils hostname'
 DEPS_CENTOS9=$DEPS_CENTOS8' zlib-devel fuse3-devel'
+# amazonlinux 2023 is based on centos 9
+DEPS_CENTOS2023=$DEPS_CENTOS9' tar java-17-amazon-corretto java-17-amazon-corretto-devel'
+DEPS_CENTOS2023=$DEPS_CENTOS2023' maven-amazon-corretto17 golang-go'
+for ver in '' 5 8 9; do
+    eval DEPS_CENTOS"$ver"='$DEPS_CENTOS'"$ver' java-openjdk java-devel'"
+done
 DEPS_CENTOS=$DEPS_CENTOS$DEPS_CENTOS_PRIOR_TO_9
 DEPS_CENTOS8=$DEPS_CENTOS8$DEPS_CENTOS_PRIOR_TO_9
 
@@ -63,6 +68,7 @@ MYCMAKE_OPTIONS_CENTOS5=$MYCMAKE_OPTIONS_CENTOS5' -D _OPENSSL_LIBDIR=/usr/lib64/
 
 MYQFSHADOOP_VERSIONS_UBUNTU1404_CENTOS6='0.23.4 0.23.11  1.0.4  1.1.2  2.5.1  2.7.2'
 MYQFSHADOOP_VERSIONS_UBUNTU1804='1.0.4  1.1.2  2.7.2  2.7.7  2.8.5  2.9.2  2.10.1  3.1.4  3.2.2  3.3.1'
+MYQFSHADOOP_VERSIONS_CENTOS2023=$MYQFSHADOOP_VERSIONS_UBUNTU1804
 MYQFSHADOOP_VERSIONS_CENTOS5='0.23.4  0.23.11  1.0.4  1.1.2  2.5.1'
 
 MYBUILD_TYPE='release'
@@ -274,7 +280,7 @@ build_centos() {
     if [ -f "$MYCENTOSEPEL_RPM" ]; then
         $MYSUDO rpm -Uvh "$MYCENTOSEPEL_RPM"
     fi
-    if [ x"$1" = x'9' ]; then
+    if [ x"$1" = x'9' -o x"$1" = x'2023' ]; then
         YUM_OPTS=--nobest
     else
         YUM_OPTS=
@@ -302,6 +308,8 @@ build_centos() {
         QFS_MSTRESS_ON=false
     elif [ x"$1" = x'6' ]; then
         QFSHADOOP_VERSIONS=$MYQFSHADOOP_VERSIONS_UBUNTU1404_CENTOS6
+    elif [ x"$1" = x'2023' ]; then
+        QFSHADOOP_VERSIONS=$MYQFSHADOOP_VERSIONS_CENTOS2023
     fi
     if [ x"$1" = x'6' ]; then
         # Remove jre 1.8 as jdk is only 1.7
@@ -322,13 +330,21 @@ build_rockylinux() {
     build_centos ${1+"$@"}
 }
 
+build_amazonlinux() {
+    build_centos ${1+"$@"}
+}
+
 set_build_type() {
     if [ x"$1" != x ]; then
         MYBUILD_TYPE=$1
     fi
 }
 
 if [ $# -eq 5 -a x"$1" = x'build' ]; then
+    if [ x"$2" == x'amazonlinux' ]; then
+        # Install useradd, find, xargs
+        yum install -y shadow-utils findutils
+    fi
     set_build_type "$4"
     set_sudo "$5"
     rm -rf build