build(lambda): concourse pipeline for AWS Lambda layer + container image

refs 82158

[skip CI]

packages/aws-lambda/layer/bin/publish-layer.sh

@@ -7,7 +7,14 @@
 
 set -eo pipefail
 
-cd `dirname $BASH_SOURCE`/..
+command -v npm >/dev/null 2>&1 || {
+  cat <<EOF >&2
+Node.js (and in particular npm) needs to be installed but it isn't.
+
+Aborting.
+EOF
+  exit 1
+}
 
 command -v aws >/dev/null 2>&1 || {
   cat <<EOF >&2
@@ -17,6 +24,16 @@ Aborting.
 EOF
   exit 1
 }
+
+command -v docker >/dev/null 2>&1 || {
+  cat <<EOF >&2
+Docker needs to be installed but it isn't.
+
+Aborting.
+EOF
+  exit 1
+}
+
 command -v jq >/dev/null 2>&1 || {
   cat <<EOF >&2
 The executable jq needs to be installed but it isn't.
@@ -26,12 +43,26 @@ EOF
   exit 1
 }
 
+command -v zip >/dev/null 2>&1 || {
+  cat <<EOF >&2
+The executable zip needs to be installed but it isn't.
+
+Aborting.
+EOF
+  exit 1
+}
+
+cd `dirname $BASH_SOURCE`/..
+
 PACKAGE_NAMES="@instana/aws-lambda instana-aws-lambda-auto-wrap"
 if [[ -z $LAYER_NAME ]]; then
   LAYER_NAME=instana-nodejs
 fi
+if [[ -z $CONTAINER_REGISTRY ]]; then
+  CONTAINER_REGISTRY=icr.io
+fi
 if [[ -z $DOCKER_IMAGE_NAME ]]; then
-  DOCKER_IMAGE_NAME=instana/aws-lambda-nodejs
+  DOCKER_IMAGE_NAME=$CONTAINER_REGISTRY/instana/aws-lambda-nodejs
 fi
 
 LICENSE=MIT
@@ -45,10 +76,22 @@ REGIONS=$'ap-northeast-1\nap-northeast-2\nap-south-1\nap-southeast-1\nap-southea
 
 if [[ -z $SKIP_DOCKER_IMAGE ]]; then
   echo Will build Docker image with name \"$DOCKER_IMAGE_NAME\".
+  if [[ -z $CONTAINER_REGISTRY_USER ]]; then
+    echo Missing mandatory environment variable CONTAINER_REGISTRY_USER.
+    exit 1
+  fi
+  if [[ -z $CONTAINER_REGISTRY_PASSWORD ]]; then
+    echo Missing mandatory environment variable CONTAINER_REGISTRY_PASSWORD.
+    exit 1
+  fi
 else
   echo Building the Docker image will be skipped.
 fi
 
+if [[ -z $AWS_ACCESS_KEY_ID ]] || [[ -z $AWS_SECRET_ACCESS_KEY ]]; then
+  echo Warning: AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are not set. This might be okay if you have set up AWS authentication via other means. If not, the AWS cli commands to publish the layer will fail.
+fi
+
 echo "step 1/8: fetching AWS regions (skipping, using fixed list of regions for now)"
 
 # Actually, this should give us the regions where the Lambda service is provided:
@@ -190,8 +233,10 @@ if [[ -z $SKIP_DOCKER_IMAGE ]]; then
   docker build . -t "$DOCKER_IMAGE_NAME"
   docker tag $DOCKER_IMAGE_NAME:latest $DOCKER_IMAGE_NAME:$VERSION
   echo " - pushing Docker image $DOCKER_IMAGE_NAME:"
+  docker login -u="$CONTAINER_REGISTRY_USER" -p="$CONTAINER_REGISTRY_PASSWORD" $CONTAINER_REGISTRY
   docker push $DOCKER_IMAGE_NAME:latest
   docker push $DOCKER_IMAGE_NAME:$VERSION
+  docker logout $CONTAINER_REGISTRY
 else
   echo "step 7/8: building docker images (skipping)"
 fi

packages/serverless/ci/dind-nodejs-aws-jq/Dockerfile

@@ -0,0 +1,73 @@
+# ----------------------------------------------------------------------------
+# Loosely based on
+#
+# https://github.com/karlkfi/concourse-dcind/blob/477674e4a27d79fa62099a86aa032017d4292d12/Dockerfile
+#
+# Modifications:
+# - Use a Node.js base image instead of raw alpine,
+# - add glibc because the AWS cli requires that (does not work with muslc)
+# - DOCKER_VERSION=20.10.11 instead of 19.03.2
+# - removed docker-compose and some related package (py-pip, python3-dev)
+# - removed docker squash support,
+# - added jq, and
+# - added/removed a bunch apk packages.
+#
+# Note: Needs to be run with --privileged.
+# ----------------------------------------------------------------------------
+
+FROM node:16.13.1-alpine3.14
+
+ENV DOCKER_CHANNEL=stable \
+    DOCKER_VERSION=20.10.11 \
+    GLIBC_VER=2.34-r0
+
+RUN apk --update --no-cache add \
+        bash \
+        ca-certificates \
+        cargo \
+        curl \
+        device-mapper \
+        gcc \
+        iptables \
+        libc-dev \
+        libffi-dev \
+        make \
+        musl-dev \
+        openssl-dev \
+        util-linux \
+        jq \
+        zip \
+        && \
+    apk upgrade && \
+    curl -fL "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz" | tar zx && \
+    mv /docker/* /bin/ && chmod +x /bin/docker* && \
+    rm -rf /var/cache/apk/* && \
+    rm -rf /root/.cache
+
+# Install glibc compatibility for alpine plus AWS cli
+RUN apk --no-cache add \
+        binutils \
+    && curl -sL https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub \
+    && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-${GLIBC_VER}.apk \
+    && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-bin-${GLIBC_VER}.apk \
+    && apk add --no-cache \
+        glibc-${GLIBC_VER}.apk \
+        glibc-bin-${GLIBC_VER}.apk \
+    && curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip \
+    && unzip awscliv2.zip \
+    && aws/install \
+    && rm -rf \
+        awscliv2.zip \
+        aws \
+        /usr/local/aws-cli/v2/*/dist/aws_completer \
+        /usr/local/aws-cli/v2/*/dist/awscli/data/ac.index \
+        /usr/local/aws-cli/v2/*/dist/awscli/examples \
+    && apk --no-cache del \
+        binutils \
+    && rm glibc-${GLIBC_VER}.apk \
+    && rm glibc-bin-${GLIBC_VER}.apk \
+    && rm -rf /var/cache/apk/*
+
+COPY entrypoint.sh /bin/entrypoint.sh
+
+ENTRYPOINT ["entrypoint.sh"]

packages/serverless/ci/dind-nodejs-aws-jq/entrypoint.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+
+# -----------------------------------------------------------------------------
+# Copied from
+# https://github.com/karlkfi/concourse-dcind/blob/477674e4a27d79fa62099a86aa032017d4292d12/entrypoint.sh
+# -----------------------------------------------------------------------------
+
+# Inspired by concourse/docker-image-resource:
+# https://github.com/concourse/docker-image-resource/blob/master/assets/common.sh
+
+set -o errexit -o pipefail -o nounset
+
+# Waits DOCKERD_TIMEOUT seconds for startup (default: 60)
+DOCKERD_TIMEOUT="${DOCKERD_TIMEOUT:-60}"
+# Accepts optional DOCKER_OPTS (default: --data-root /scratch/docker)
+DOCKER_OPTS="${DOCKER_OPTS:-}"
+
+# Constants
+DOCKERD_PID_FILE="/tmp/docker.pid"
+DOCKERD_LOG_FILE="/tmp/docker.log"
+
+sanitize_cgroups() {
+  local cgroup="/sys/fs/cgroup"
+
+  mkdir -p "${cgroup}"
+  if ! mountpoint -q "${cgroup}"; then
+    if ! mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup "${cgroup}"; then
+      echo >&2 "Could not make a tmpfs mount. Did you use --privileged?"
+      exit 1
+    fi
+  fi
+  mount -o remount,rw "${cgroup}"
+
+  # Skip AppArmor
+  # See: https://github.com/moby/moby/commit/de191e86321f7d3136ff42ff75826b8107399497
+  export container=docker
+
+  # Mount /sys/kernel/security
+  if [[ -d /sys/kernel/security ]] && ! mountpoint -q /sys/kernel/security; then
+    if ! mount -t securityfs none /sys/kernel/security; then
+      echo >&2 "Could not mount /sys/kernel/security."
+      echo >&2 "AppArmor detection and --privileged mode might break."
+    fi
+  fi
+
+  sed -e 1d /proc/cgroups | while read sys hierarchy num enabled; do
+    if [[ "${enabled}" != "1" ]]; then
+      # subsystem disabled; skip
+      continue
+    fi
+
+    grouping="$(cat /proc/self/cgroup | cut -d: -f2 | grep "\\<${sys}\\>")"
+    if [[ -z "${grouping}" ]]; then
+      # subsystem not mounted anywhere; mount it on its own
+      grouping="${sys}"
+    fi
+
+    mountpoint="${cgroup}/${grouping}"
+
+    mkdir -p "${mountpoint}"
+
+    # clear out existing mount to make sure new one is read-write
+    if mountpoint -q "${mountpoint}"; then
+      umount "${mountpoint}"
+    fi
+
+    mount -n -t cgroup -o "${grouping}" cgroup "${mountpoint}"
+
+    if [[ "${grouping}" != "${sys}" ]]; then
+      if [[ -L "${cgroup}/${sys}" ]]; then
+        rm "${cgroup}/${sys}"
+      fi
+
+      ln -s "${mountpoint}" "${cgroup}/${sys}"
+    fi
+  done
+
+  # Initialize systemd cgroup if host isn't using systemd.
+  # Workaround for https://github.com/docker/for-linux/issues/219
+  if ! [[ -d /sys/fs/cgroup/systemd ]]; then
+    mkdir "${cgroup}/systemd"
+    mount -t cgroup -o none,name=systemd cgroup "${cgroup}/systemd"
+  fi
+}
+
+# Setup container environment and start docker daemon in the background.
+start_docker() {
+  echo >&2 "Setting up Docker environment..."
+  mkdir -p /var/log
+  mkdir -p /var/run
+
+  sanitize_cgroups
+
+  # check for /proc/sys being mounted readonly, as systemd does
+  if grep '/proc/sys\s\+\w\+\s\+ro,' /proc/mounts >/dev/null; then
+    mount -o remount,rw /proc/sys
+  fi
+
+  local docker_opts="${DOCKER_OPTS:-}"
+
+  # Pass through `--garden-mtu` from gardian container
+  if [[ "${docker_opts}" != *'--mtu'* ]]; then
+    local mtu="$(cat /sys/class/net/$(ip route get 8.8.8.8|awk '{ print $5 }')/mtu)"
+    docker_opts+=" --mtu ${mtu}"
+  fi
+
+  # Use Concourse's scratch volume to bypass the graph filesystem by default
+  if [[ "${docker_opts}" != *'--data-root'* ]] && [[ "${docker_opts}" != *'--graph'* ]]; then
+    docker_opts+=' --data-root /scratch/docker'
+  fi
+
+  rm -f "${DOCKERD_PID_FILE}"
+  touch "${DOCKERD_LOG_FILE}"
+
+  echo >&2 "Starting Docker..."
+  dockerd ${docker_opts} &>"${DOCKERD_LOG_FILE}" &
+  echo "$!" > "${DOCKERD_PID_FILE}"
+}
+
+# Wait for docker daemon to be healthy
+# Timeout after DOCKERD_TIMEOUT seconds
+await_docker() {
+  local timeout="${DOCKERD_TIMEOUT}"
+  echo >&2 "Waiting ${timeout} seconds for Docker to be available..."
+  local start=${SECONDS}
+  timeout=$(( timeout + start ))
+  until docker info &>/dev/null; do
+    if (( SECONDS >= timeout )); then
+      echo >&2 'Timed out trying to connect to docker daemon.'
+      if [[ -f "${DOCKERD_LOG_FILE}" ]]; then
+        echo >&2 '---DOCKERD LOGS---'
+        cat >&2 "${DOCKERD_LOG_FILE}"
+      fi
+      exit 1
+    fi
+    if [[ -f "${DOCKERD_PID_FILE}" ]] && ! kill -0 $(cat "${DOCKERD_PID_FILE}"); then
+      echo >&2 'Docker daemon failed to start.'
+      if [[ -f "${DOCKERD_LOG_FILE}" ]]; then
+        echo >&2 '---DOCKERD LOGS---'
+        cat >&2 "${DOCKERD_LOG_FILE}"
+      fi
+      exit 1
+    fi
+    sleep 1
+  done
+  local duration=$(( SECONDS - start ))
+  echo >&2 "Docker available after ${duration} seconds."
+}
+
+# Gracefully stop Docker daemon.
+stop_docker() {
+  if ! [[ -f "${DOCKERD_PID_FILE}" ]]; then
+    return 0
+  fi
+  local docker_pid="$(cat ${DOCKERD_PID_FILE})"
+  if [[ -z "${docker_pid}" ]]; then
+    return 0
+  fi
+  echo >&2 "Terminating Docker daemon."
+  kill -TERM ${docker_pid}
+  local start=${SECONDS}
+  echo >&2 "Waiting for Docker daemon to exit..."
+  wait ${docker_pid}
+  local duration=$(( SECONDS - start ))
+  echo >&2 "Docker exited after ${duration} seconds."
+}
+
+start_docker
+trap stop_docker EXIT
+await_docker
+
+# do not exec, because exec disables traps
+if [[ "$#" != "0" ]]; then
+  "$@"
+else
+  bash --login
+fi
+