Support arm64/aarch64 Linux and Apple Silicon

- remove references to 32 bit macos/darwin
- normalize Redis server file names
- update `build-server-binaries.sh` to statically link openssl on macOS
- correctly detect macOS and Unix architectures

Co-authored-by: Nathan J Mehl <[email protected]>

README.md

@@ -38,10 +38,11 @@ RedisServer redisServer = new RedisServer("/path/to/your/redis", 6379);
 // 2) given os-independent matrix
 RedisExecProvider customProvider = RedisExecProvider.defaultProvider()
   .override(OS.UNIX, "/path/to/unix/redis")
-  .override(OS.WINDOWS, Architecture.x86, "/path/to/windows/redis")
-  .override(OS.Windows, Architecture.x86_64, "/path/to/windows/redis")
-  .override(OS.MAC_OS_X, Architecture.x86, "/path/to/macosx/redis")
-  .override(OS.MAC_OS_X, Architecture.x86_64, "/path/to/macosx/redis")
+  .override(OS.UNIX, Architecture.x86_64, "/path/to/unix/redis.x86_64")
+  .override(OS.UNIX, Architecture.arm64, "/path/to/unix/redis.arm64")
+  .override(OS.UNIX, Architecture.x86, "/path/to/unix/redis.i386")
+  .override(OS.MAC_OS_X, Architecture.x86_64, "/path/to/macosx/redis-x86_64")
+  .override(OS.MAC_OS_X, Architecture.arm64, "/path/to/macosx/redis.arm64")
 
 RedisServer redisServer = new RedisServer(customProvider, 6379);
 ```
@@ -141,10 +142,12 @@ Redis version
 
 By default, RedisServer runs an OS-specific executable enclosed in in the `embedded-redis` jar. The jar includes:
 
-- Redis 6.2.6 for Linux/Unix (amd64 and x86)
-- Redis 6.2.6 for macOS (amd64)
+- Redis 6.2.6 for Linux/Unix (i386, x86_64 and arm64)
+- Redis 6.2.6 for macOS (x86_64 and arm64e AKA Apple Silicon)
 
-The enclosed binaries are built from source from the [`6.2.6` tag](https://github.com/redis/redis/releases/tag/6.2.6) in the official Redis repository. The Linux binaries are statically-linked amd64 and x86 executables built using the `build-server-binaries.sh` script included in this repository at `/src/main/docker`. The macOS binaries are built according to the [instructions in the README](https://github.com/redis/redis/blob/4930d19e70c391750479951022e207e19111eb55/README.md#building-redis). Windows binaries are not included because Windows is not officially supported by Redis.
+The enclosed binaries are built from source from the [`6.2.6` tag](https://github.com/redis/redis/releases/tag/6.2.6) in the official Redis repository. The Linux and Darwin/macOS binaries are statically-linked amd64 and x86 executables built using the [build-server-binaries.sh](src/main/docker/build-server-binaries.sh) script included in this repository at `/src/main/docker`.  Windows binaries are not included because Windows is not officially supported by Redis.
+
+Note: the `build-server-binaries.sh` script attempts to build all of the above noted OS and architectures, which means that it expects the local Docker daemon to support all of them.  Docker Desktop on macOS and Windows supports multi-arch builds out of the box; Docker on Linux may require [additional configuration](https://docs.docker.com/buildx/working-with-buildx/).
 
 Callers may provide a path to a specific `redis-server` executable if needed.
 
@@ -182,8 +185,9 @@ Changelog
 ==============
 
 ### 0.8.2
- * Compiled Redis servers with TLS support
  * Updated to Redis 6.2.6
+ * Added native support for Apple Silicon (darwin/arm64) and Linux aarch64
+ * Compiled Redis servers with TLS support
 
 ### 0.8.1
 

pom.xml

@@ -95,6 +95,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
                 <configuration>
                     <source>8</source>
                     <target>8</target>

src/main/docker/.gitignore

@@ -0,0 +1,2 @@
+redis-server-*
+redis-*

src/main/docker/Dockerfile

@@ -0,0 +1,18 @@
+ARG ALPINE_VERSION=3.12
+FROM alpine:${ALPINE_VERSION}
+RUN apk add --no-cache gcc musl-dev libressl-dev make pkgconfig linux-headers
+
+WORKDIR /build
+
+ARG REDIS_VERSION
+ENV REDIS_VERSION=${REDIS_VERSION}
+COPY redis-${REDIS_VERSION}.tar.gz /redis-${REDIS_VERSION}.tar.gz
+RUN ls -l /
+
+ARG ARCH
+RUN tar zxf /redis-${REDIS_VERSION}.tar.gz && \
+    cd redis-${REDIS_VERSION} && \
+    make BUILD_TLS='yes' CC='gcc -static' LDFLAGS='-s' MALLOC='libc' && \
+    mv src/redis-server /build/redis-server-${REDIS_VERSION}-linux-${ARCH}
+
+CMD [ "/bin/sh" ]

src/main/docker/build-server-binaries.sh

@@ -1,16 +1,115 @@
 #!/bin/bash
 
+set -e
+
 REDIS_VERSION=6.2.6
+REDIS_TARBALL="redis-${REDIS_VERSION}.tar.gz"
+REDIS_URL="https://download.redis.io/releases/${REDIS_TARBALL}"
+
+echo $ARCH
+
+function copy_openssl_and_remove_dylibs() {
+  # To make macOS builds more portable, we want to statically link OpenSSL,
+  # which is not straightforward. To force static compilation, we copy
+  # the openssl libraries and remove dylibs, forcing static linking
+  OPENSSL_HOME="${1}"
+  ARCH=$2
+  OPENSSL_HOME_COPY="${3}/${ARCH}"
+  if [ -d "${OPENSSL_HOME}" ]; then
+    echo "*** Copying openssl libraries for static linking"
+    cp -RL "${OPENSSL_HOME}" "${OPENSSL_HOME_COPY}"
+    rm -f "${OPENSSL_HOME_COPY}"/lib/*.dylib
+  else
+    echo "*** WARNING: could not find openssl libraries at ${OPENSSL_HOME}; this build will almost certainly fail"
+  fi
+}
+
+if [ "$(dirname ${0})" != "." ]; then
+  echo "This script must be run from $(dirname ${0}). \`cd\` there and run again"
+  exit 1
+fi
+
+if ! [ -f "${REDIS_TARBALL}" ]; then
+  curl -o "${REDIS_TARBALL}" "${REDIS_URL}"
+fi
+
+all_linux=0
+if command -pv docker 2>/dev/null; then
+  for arch in x86_64 arm64 i386; do
+
+    echo "*** Building redis version ${REDIS_VERSION} for linux-${arch}"
+
+    set +e
+    docker build \
+      "--platform=linux/${arch}" \
+      --build-arg "REDIS_VERSION=${REDIS_VERSION}" \
+      --build-arg "ARCH=${arch}" \
+      -t "redis-server-builder-${arch}" \
+      .
+
+    if [[ $? -ne 0 ]]; then
+      echo "*** ERROR: could not build for linux-${arch}"
+      continue
+    fi
+    set -e
+
+    docker run -it --rm \
+      "--platform=linux/${arch}" \
+      -v "$(pwd)/":/mnt \
+      "redis-server-builder-${arch}" \
+      sh -c "cp /build/redis-server-${REDIS_VERSION}-linux-${arch} /mnt"
+
+    ((all_linux+=1))
+
+  done
+else
+  echo "*** WARNING: No docker command found. Cannot build for linux."
+fi
+
+if [[ "${all_linux}" -lt 3 ]]; then
+  echo "*** WARNING: was not able to build for all linux arches; see above for errors"
+fi
+
+if [[ "$(uname -s)" == "Darwin" ]]; then
+
+  tar zxf "${REDIS_TARBALL}"
+  cd "redis-${REDIS_VERSION}"
+
+  # temporary directory for openssl libraries for static linking.
+  # assumes standard Homebrew openssl install:
+  #   - arm64e at /opt/homebrew/opt/openssl
+  #   - x86_64 at /usr/local/opt/openssl
+  OPENSSL_TEMP=$(mktemp -d /tmp/embedded-redis-darwin-openssl.XXXXX)
+
+  # build for arm64 on apple silicon
+  if arch -arm64e true 2>/dev/null; then
+    copy_openssl_and_remove_dylibs /opt/homebrew/opt/openssl arm64e "${OPENSSL_TEMP}"
+    echo "*** Building redis version ${REDIS_VERSION} for darwin-arm64e (apple silicon)"
+    make clean
+    arch -arm64e make -j3 BUILD_TLS=yes OPENSSL_PREFIX="$OPENSSL_TEMP/arm64e"
+    mv src/redis-server "../redis-server-${REDIS_VERSION}-darwin-arm64"
+  else
+    echo "*** WARNING: could not build for darwin-arm64e; you probably want to do this on an apple silicon device"
+  fi
+
+  # build for x86_64 if we're on apple silicon or a recent macos on x86_64
+  if arch -x86_64 true 2>/dev/null; then
+    copy_openssl_and_remove_dylibs /usr/local/opt/openssl x86_64 "${OPENSSL_TEMP}"
+    echo "*** Building redis version ${REDIS_VERSION} for darwin-x86_64"
+    make clean
+    arch -x86_64 make -j3 BUILD_TLS=yes OPENSSL_PREFIX="$OPENSSL_TEMP/x86_64"
+    mv src/redis-server "../redis-server-${REDIS_VERSION}-darwin-x86_64"
+  else
+    echo "*** WARNING: you are on a version of macos that lacks /usr/bin/arch, you probably do not want this"
+    exit 1
+  fi
+  cd ..
 
-docker build --build-arg REDIS_VERSION=${REDIS_VERSION} -t redis-server-builder-amd64 -f ./build-server-amd64.docker .
-docker build --build-arg REDIS_VERSION=${REDIS_VERSION} -t redis-server-builder-x86   -f ./build-server-x86.docker .
+else
+  echo "*** WARNING: Cannot build for macos/darwin on a $(uname -s) host"
+fi
 
-docker run -it --rm \
-		-v "$(pwd)/":/redis-server-binaries \
-		redis-server-builder-amd64 \
-		sh -c "cd redis-${REDIS_VERSION}; make BUILD_TLS='yes' CC='gcc -static' LDFLAGS='-s' MALLOC='libc'; cp src/redis-server /redis-server-binaries/redis-server-${REDIS_VERSION}"
+ls -l redis-server-*
 
-docker run -it --rm \
-		-v "$(pwd)/":/redis-server-binaries \
-		redis-server-builder-x86 \
-		sh -c "cd redis-${REDIS_VERSION}; make BUILD_TLS='yes' CC='gcc -static' CFLAGS='-m32' LDFLAGS='-m32 -s' MALLOC='libc'; cp src/redis-server /redis-server-binaries/redis-server-${REDIS_VERSION}-32"
+echo "*** Moving built binaries to ../resources; you need to handle the rest yourself"
+mv redis-server-* ../resources/

src/main/java/redis/embedded/AbstractRedisInstance.java

@@ -76,7 +76,7 @@ private void awaitRedisServerReady() throws IOException {
                 }
             } while (!outputLine.matches(redisReadyPattern()));
         } finally {
-            IOUtils.closeQuietly(reader);
+            IOUtils.closeQuietly(reader, null);
         }
     }
 
@@ -127,7 +127,7 @@ public void run() {
             try {
                 readLines();
             } finally {
-                IOUtils.closeQuietly(reader);
+                IOUtils.closeQuietly(reader, null);
             }
         }
 

src/main/java/redis/embedded/RedisExecProvider.java

@@ -15,6 +15,8 @@ public class RedisExecProvider {
 
     private final Map<OsArchitecture, String> executables = Maps.newHashMap();
 
+    public static final String redisVersion = "6.2.6";
+
     public static RedisExecProvider defaultProvider() {
         return new RedisExecProvider();
     }
@@ -24,11 +26,12 @@ private RedisExecProvider() {
     }
 
     private void initExecutables() {
-        executables.put(OsArchitecture.UNIX_x86, "redis-server-6.2.6-32");
-        executables.put(OsArchitecture.UNIX_x86_64, "redis-server-6.2.6");
+        executables.put(OsArchitecture.UNIX_x86, "redis-server-" + redisVersion + "-linux-i386");
+        executables.put(OsArchitecture.UNIX_x86_64, "redis-server-" + redisVersion + "-linux-x86_64");
+        executables.put(OsArchitecture.UNIX_arm64, "redis-server-" + redisVersion + "-linux-arm64");
 
-        executables.put(OsArchitecture.MAC_OS_X_x86, "redis-server-6.2.6.app");
-        executables.put(OsArchitecture.MAC_OS_X_x86_64, "redis-server-6.2.6.app");
+        executables.put(OsArchitecture.MAC_OS_X_x86_64, "redis-server-" + redisVersion + "-darwin-x86_64");
+        executables.put(OsArchitecture.MAC_OS_X_arm64, "redis-server-" + redisVersion + "-darwin-arm64");
     }
 
     public RedisExecProvider override(OS os, String executable) {

src/main/java/redis/embedded/RedisSentinelBuilder.java

@@ -7,6 +7,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -139,7 +140,7 @@ private void resolveSentinelConf() throws IOException {
 
         File redisConfigFile = File.createTempFile(resolveConfigName(), ".conf");
         redisConfigFile.deleteOnExit();
-        Files.write(configString, redisConfigFile, Charset.forName("UTF-8"));
+        Files.asCharSink(redisConfigFile, StandardCharsets.UTF_8).write(configString);
         sentinelConf = redisConfigFile.getAbsolutePath();
     }
 

src/main/java/redis/embedded/RedisServerBuilder.java

@@ -8,6 +8,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -103,7 +104,7 @@ private void resolveConfAndExec() throws IOException {
         if (redisConf == null && redisConfigBuilder != null) {
             File redisConfigFile = File.createTempFile(resolveConfigName(), ".conf");
             redisConfigFile.deleteOnExit();
-            Files.write(redisConfigBuilder.toString(), redisConfigFile, Charset.forName("UTF-8"));
+            Files.asCharSink(redisConfigFile, StandardCharsets.UTF_8).write(redisConfigBuilder.toString());
             redisConf = redisConfigFile.getAbsolutePath();
         }
 

src/main/java/redis/embedded/util/Architecture.java

@@ -2,5 +2,6 @@
 
 public enum Architecture {
     x86,
-    x86_64
+    x86_64,
+    arm64
 }

src/main/java/redis/embedded/util/OSDetector.java

@@ -47,58 +47,46 @@ private static Architecture getWindowsArchitecture() {
     }
 
     private static Architecture getUnixArchitecture() {
-        BufferedReader input = null;
         try {
-            String line;
             Process proc = Runtime.getRuntime().exec("uname -m");
-            input = new BufferedReader(new InputStreamReader(proc.getInputStream()));
-            while ((line = input.readLine()) != null) {
-                if (line.length() > 0) {
-                    if (line.contains("64")) {
+            try (BufferedReader input = new BufferedReader(new InputStreamReader(proc.getInputStream()))) {
+                String machine = input.readLine();
+                switch (machine) {
+                    case "aarch64":
+                        return Architecture.arm64;
+                    case "x86_64":
                         return Architecture.x86_64;
-                    }
+                    default:
+                        throw new OsDetectionException("unsupported architecture: " + machine);
                 }
             }
         } catch (Exception e) {
-            throw new OsDetectionException(e);
-        } finally {
-            try {
-                if (input != null) {
-                    input.close();
-                }
-            } catch (Exception ignored) {
-                // ignore
+            if (e instanceof OsDetectionException) {
+                throw (OsDetectionException)e;
             }
+            throw new OsDetectionException(e);
         }
-
-        return Architecture.x86;
     }
 
     private static Architecture getMacOSXArchitecture() {
-        BufferedReader input = null;
         try {
-            String line;
-            Process proc = Runtime.getRuntime().exec("sysctl hw");
-            input = new BufferedReader(new InputStreamReader(proc.getInputStream()));
-            while ((line = input.readLine()) != null) {
-                if (line.length() > 0) {
-                    if ((line.contains("cpu64bit_capable")) && (line.trim().endsWith("1"))) {
+            Process proc = Runtime.getRuntime().exec("uname -m");
+            try (BufferedReader input = new BufferedReader(new InputStreamReader(proc.getInputStream()))) {
+                String machine = input.readLine();
+                switch (machine) {
+                    case "arm64":
+                        return Architecture.arm64;
+                    case "x86_64":
                         return Architecture.x86_64;
-                    }
+                    default:
+                        throw new OsDetectionException("unsupported architecture: " + machine);
                 }
             }
         } catch (Exception e) {
-            throw new OsDetectionException(e);
-        } finally {
-            try {
-                if (input != null) {
-                    input.close();
-                }
-            } catch (Exception ignored) {
-                // ignore
+            if (e instanceof OsDetectionException) {
+                throw (OsDetectionException)e;
             }
+            throw new OsDetectionException(e);
         }
-
-        return Architecture.x86;
     }
 }