Setup X509 CodeBuild Project for Limbo Report Generation

.github/workflows/actions-ci.yml

@@ -591,21 +591,3 @@ jobs:
           run: |
             sudo pkg install -y git gmake cmake go ninja
             tests/ci/run_bsd_tests.sh
-  # Temporary to test the x509-limbo patch and building of the reporting tool.
-  # This will move into a separate project in the next PR. But doing this for now to
-  # cutdown the review size.
-  x509-limbo-tooling:
-    if: github.repository_owner == 'aws'
-    needs: [sanity-test-run]
-    name: x509-limbo tooling
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.13' 
-      - name: Verify x509-limbo patch and reporting tool
-        run: |
-          ./tests/ci/run_x509_limbo.sh

tests/ci/cdk/app.py

@@ -13,6 +13,7 @@
 from cdk.aws_lc_ec2_test_framework_ci_stack import AwsLcEC2TestingCIStack
 from cdk.linux_docker_image_batch_build_stack import LinuxDockerImageBatchBuildStack
 from cdk.windows_docker_image_build_stack import WindowsDockerImageBuildStack
+from cdk.aws_lc_github_ci_x509_stack import AwsLcGitHubX509CIStack
 from cdk.ecr_stack import EcrStack
 from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_X86_ECR_REPO
 
@@ -54,5 +55,6 @@
 AwsLcEC2TestingCIStack(app, "aws-lc-ci-ec2-test-framework", ec2_test_framework_build_spec_file, env=env)
 android_build_spec_file = "cdk/codebuild/github_ci_android_omnibus.yaml"
 AwsLcAndroidCIStack(app, "aws-lc-ci-devicefarm-android", android_build_spec_file, env=env)
+AwsLcGitHubX509CIStack(app, "aws-lc-ci-x509")
 
 app.synth()

tests/ci/cdk/cdk/aws_lc_github_ci_x509_stack.py

@@ -0,0 +1,93 @@
+from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_s3 as s3
+from constructs import Construct
+from util.build_spec_loader import BuildSpecLoader
+from util.metadata import (
+    GITHUB_PUSH_CI_BRANCH_TARGETS,
+    GITHUB_REPO_NAME,
+    GITHUB_REPO_OWNER,
+)
+
+
+class AwsLcGitHubX509CIStack(Stack):
+    def __init__(
+        self,
+        scope: Construct,
+        id: str,
+        **kwargs,
+    ) -> None:
+        super().__init__(scope, id, **kwargs)
+
+        git_hub_source = codebuild.Source.git_hub(
+            owner=GITHUB_REPO_OWNER,
+            repo=GITHUB_REPO_NAME,
+            webhook=True,
+            webhook_filters=[
+                codebuild.FilterGroup.in_event_of(
+                    codebuild.EventAction.PULL_REQUEST_CREATED,
+                    codebuild.EventAction.PULL_REQUEST_UPDATED,
+                    codebuild.EventAction.PULL_REQUEST_REOPENED,
+                ),
+                codebuild.FilterGroup.in_event_of(
+                    codebuild.EventAction.PUSH
+                ).and_branch_is(GITHUB_PUSH_CI_BRANCH_TARGETS),
+            ],
+            webhook_triggers_batch_build=True,
+        )
+
+        self.reports_bucket = s3.Bucket(
+            self,
+            "aws-lc-x509-reports",
+            block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
+            versioned=True,
+        )
+
+        self.reports_bucket.add_lifecycle_rule(
+            enabled=True,
+            prefix="x509-limbo/",
+            transitions=[
+                s3.Transition(
+                    storage_class=s3.StorageClass.INTELLIGENT_TIERING,
+                    transition_after=Duration.days(30),
+                ),
+            ],
+            noncurrent_version_transitions=[
+                s3.NoncurrentVersionTransition(
+                    storage_class=s3.StorageClass.INTELLIGENT_TIERING,
+                    transition_after=Duration.days(30),
+                ),
+            ],
+        )
+        self.reports_bucket.add_lifecycle_rule(
+            enabled=True,
+            prefix="x509-limbo/pr/",
+            expiration=Duration.days(30),
+            noncurrent_version_expiration=Duration.days(1),
+        )
+
+        # This is for the case of a manual build is triggered via CodeBuild console/API.
+        self.reports_bucket.add_lifecycle_rule(
+            enabled=True,
+            prefix=f"x509-limbo/{id}:",
+            expiration=Duration.days(30),
+            noncurrent_version_expiration=Duration.days(1),
+        )
+
+        self.codebuild_project = codebuild.Project(
+            self,
+            id,
+            project_name=id,
+            source=git_hub_source,
+            build_spec=BuildSpecLoader.load(
+                "cdk/codebuild/github_ci_x509_omnibus.yaml"
+            ),
+            environment=codebuild.BuildEnvironment(
+                build_image=codebuild.LinuxBuildImage.STANDARD_6_0,
+                compute_type=codebuild.ComputeType.LARGE,
+                privileged=True,
+            ),
+            artifacts=codebuild.Artifacts.s3(
+                bucket=self.reports_bucket,
+                package_zip=False,
+                include_build_id=False,
+            ),
+        )

tests/ci/cdk/cdk/codebuild/github_ci_x509_omnibus.yaml

@@ -0,0 +1,18 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+version: 0.2
+
+# Doc for batch https://docs.aws.amazon.com/codebuild/latest/userguide/batch-build-buildspec.html#build-spec.batch.build-list
+batch:
+  build-list:
+
+    - identifier: x509_limbo_report
+      buildspec: ./tests/ci/codebuild/common/run_x509_limbo_reports_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: true
+        compute-type: BUILD_GENERAL1_LARGE
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:amazonlinux-2023_x509_latest
+        variables:
+          AWS_LC_CI_TARGET: "tests/ci/run_x509_limbo.sh"

tests/ci/codebuild/common/run_x509_limbo_reports_target.yml

@@ -0,0 +1,23 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+version: 0.2
+
+env:
+  variables:
+    GOPROXY: https://proxy.golang.org,direct
+
+phases:
+  install:
+    commands:
+      - nohup /usr/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 &
+      - timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
+  build:
+    commands:
+      - "./${AWS_LC_CI_TARGET}"
+
+artifacts:
+  files:
+    - '**/*'
+  base-directory: x509-limbo-reports
+  name: x509-limbo/${CODEBUILD_WEBHOOK_TRIGGER:-$CODEBUILD_BUILD_ID}

tests/ci/docker_images/linux-x86/amazonlinux-2023_x509/Dockerfile

@@ -0,0 +1,28 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR ISC
+
+FROM amazonlinux-2023:clang-15x
+
+SHELL ["/bin/bash", "-c"]
+
+RUN set -ex && \
+    dnf -y upgrade --releasever=latest && dnf install -y \
+    docker \
+    openssl-devel \
+    patch \
+    jq && \
+    dnf clean packages && \
+    dnf clean metadata && \
+    dnf clean all && \
+    rm -rf /tmp/* && \
+    rm -rf /var/cache/dnf
+
+RUN curl -fsSL https://pyenv.run | bash
+
+ENV PATH="/root/.pyenv/bin:${PATH}"
+RUN eval "$(pyenv init -)" && \
+    pyenv install 3.13.1
+
+ENV PATH="/root/.pyenv/versions/3.13.1/bin:${PATH}"
+ENV CC=clang
+ENV CXX=clang++

tests/ci/docker_images/linux-x86/build_images.sh

@@ -40,6 +40,7 @@ docker build -t amazonlinux-2023:base -f amazonlinux-2023_base/Dockerfile ../dep
 docker build -t amazonlinux-2023:gcc-11x amazonlinux-2023_gcc-11x
 docker build -t amazonlinux-2023:clang-15x amazonlinux-2023_clang-15x
 docker build -t amazonlinux-2023:clang-15x-sanitizer amazonlinux-2023_clang-15x_sanitizer
+docker build -t amazonlinux-2023:x509 amazonlinux-2023_x509
 docker build -t amazonlinux-2023:cryptofuzz -f amazonlinux-2023_clang-15x_cryptofuzz/Dockerfile ../dependencies
 docker build -t ubuntu-16.04:gcc-5x -f ubuntu-16.04_gcc-5x/Dockerfile ../dependencies
 docker build -t centos-7:gcc-4x -f centos-7_gcc-4x/Dockerfile ../dependencies

tests/ci/docker_images/linux-x86/push_images.sh

@@ -46,5 +46,6 @@ tag_and_push_img 'amazonlinux-2023:gcc-11x' "${ECS_REPO}:amazonlinux-2023_gcc-11
 tag_and_push_img 'amazonlinux-2023:clang-15x' "${ECS_REPO}:amazonlinux-2023_clang-15x"
 tag_and_push_img 'amazonlinux-2023:clang-15x-sanitizer' "${ECS_REPO}:amazonlinux-2023_clang-15x_sanitizer"
 tag_and_push_img 'amazonlinux-2023:cryptofuzz' "${ECS_REPO}:amazonlinux-2023_clang-15x_cryptofuzz"
+tag_and_push_img 'amazonlinux-2023:x509' "${ECS_REPO}:amazonlinux-2023_x509"
 tag_and_push_img 'fedora-31:clang-9x' "${ECS_REPO}:fedora-31_clang-9x"
 tag_and_push_img 'ubuntu-10.04_gcc-4.1x' "${ECS_REPO}:ubuntu-10.04_gcc-4.1x"

tests/ci/run_x509_limbo.sh

@@ -13,6 +13,14 @@ source tests/ci/common_posix_setup.sh
 SCRATCH_DIR="${SYS_ROOT}/scratch"
 X509_CI_DIR="${SRC_ROOT}/tests/ci/x509"
 X509_LIMBO_SRC="${SCRATCH_DIR}/x509-limbo"
+BASE_COMMIT_SRC="${SYS_ROOT}/base-src"
+
+# If BASE_REF is set in the environment we will use that, this provides a mechanism for a user to manually kick off
+# a job via the CodeBuild console (otherwise CodeBuild won't let your override variables prefixed with CODEBUILD_).
+# Otherwise if CODEBUILD_WEBHOOK_BASE_REF is set we use (this would be in the case of pull requests).
+# Lastly if that isn't set then we fallback to CODEBUILD_WEBHOOK_PREV_COMMIT which will be set for a push event.
+# If none of those are set the script will fail here.
+BASE_REF="${BASE_REF:-${CODEBUILD_WEBHOOK_BASE_REF:-${CODEBUILD_WEBHOOK_PREV_COMMIT:?}}}"
 
 function build_reporting_tool() {
     pushd "${X509_CI_DIR}/limbo-report"
@@ -21,26 +29,65 @@ function build_reporting_tool() {
     popd # "${X509_CI_DIR}/limbo-report"
 }
 
-function clone_and_patch_x509_limbo() {
+function setup_x509_limbo() {
     git clone https://github.com/C2SP/x509-limbo.git "${X509_LIMBO_SRC}"
     pushd "${X509_LIMBO_SRC}"
     patch -p1 -i "${X509_CI_DIR}/x509-limbo.patch"
+    python3 -m venv .venv
+    source .venv/bin/activate
+    pip install -e .
     popd # "${X509_LIMBO_SRC}"
 }
 
 function run_aws_lc_harness() {
     pushd "${X509_LIMBO_SRC}"
-    AWS_LC_SRC_DIR="${SRC_ROOT}" make test-aws-lc
+    set +e
+    AWS_LC_SRC_DIR="${1}" make test-aws-lc
+    if [ ! -f "${X509_LIMBO_SRC}/results/aws-lc.json" ]; then
+        echo "Failed to run x509-limbo harness for AWS_LC_SRC_DIR=${1}"
+        exit 1
+    fi
+    set -e
     popd # "${X509_LIMBO_SRC}"
 }
 
+# Log Docker hub limit https://docs.docker.com/docker-hub/download-rate-limit/#how-can-i-check-my-current-rate
+TOKEN=$(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)
+curl --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest
+
+git worktree add "${BASE_COMMIT_SRC}" "${BASE_REF:?}"
+
 mkdir -p "${SCRATCH_DIR}"
 rm -rf "${SCRATCH_DIR:?}"/*
 pushd "${SCRATCH_DIR}"
 
 build_reporting_tool
-clone_and_patch_x509_limbo
-run_aws_lc_harness
+setup_x509_limbo
+
+REPORTS_DIR="${SRC_ROOT}/x509-limbo-reports"
+mkdir -p "${REPORTS_DIR}"
 
+# Build run x509-limbo on current src of event
+run_aws_lc_harness "${SRC_ROOT}"
+"${SCRATCH_DIR}/limbo-report" annotate "${X509_LIMBO_SRC}/limbo.json" "${X509_LIMBO_SRC}/results/aws-lc.json" > "${REPORTS_DIR}/base.json"
+"${SCRATCH_DIR}/limbo-report" annotate -csv "${X509_LIMBO_SRC}/limbo.json" "${X509_LIMBO_SRC}/results/aws-lc.json" > "${REPORTS_DIR}/base.csv"
+
+# Build run x509-limbo on the base src for event
+run_aws_lc_harness "${BASE_COMMIT_SRC}"
+"${SCRATCH_DIR}/limbo-report" annotate "${X509_LIMBO_SRC}/limbo.json" "${X509_LIMBO_SRC}/results/aws-lc.json" > "${REPORTS_DIR}/changes.json"
+"${SCRATCH_DIR}/limbo-report" annotate -csv "${X509_LIMBO_SRC}/limbo.json" "${X509_LIMBO_SRC}/results/aws-lc.json" > "${REPORTS_DIR}/changes.csv"
+
+# Produce diff report
+set +e
+"${SCRATCH_DIR}/limbo-report" diff "${REPORTS_DIR}/base.json" "${REPORTS_DIR}/changes.json" | tee "${REPORTS_DIR}/summary.txt"
+DIFF_RET_STATUS=${PIPESTATUS[0]}
+
+set -e
 popd # "${SCRATCH_DIR}"
-# rm -rf "${SCRATCH_DIR:?}"
+rm -rf "${SCRATCH_DIR:?}"
+
+if [ $DIFF_RET_STATUS -eq 0 ]; then
+    exit 0
+else
+    exit 1
+fi

tests/ci/setup.py

@@ -20,14 +20,14 @@
 
     install_requires=[
         # CDK dependencies.
-        "aws-cdk-lib==2.173.1",
+        "aws-cdk-lib==2.177.0",
         "constructs==10.4.2",
         # PyYAML is a YAML parser and emitter for Python. Used to read build_spec.yaml.
         "pyyaml==6.0.2",
         # A formatter for Python code.
         "yapf==0.43.0",
         # Introduced by benchmark framework.
-        "boto3==1.35.82",
+        "boto3==1.36.12",
         # Introduced by Android Device Farm CI.
         "requests",
         "arnparse==0.0.2",

tests/ci/x509/x509-limbo.patch

@@ -73,7 +73,7 @@ index 0000000..2305b42
 +
 +.PHONY: verify-aws-lc-source
 +verify-aws-lc-source:
-+	@if [ test -z "${AWS_LC_SRC_DIR}" ]; then \
++	@if [ -z "${AWS_LC_SRC_DIR}" ]; then \
 +		echo "AWS_LC_SRC_DIR environment variable is missing and is required to specify AWS-LC source location" && false; \
 +	fi
 +