From 0006a288ef782ebab4f73a0d1cc3706149e97831 Mon Sep 17 00:00:00 2001 From: Boquan Fang Date: Wed, 11 Dec 2024 17:01:53 -0800 Subject: [PATCH] ci: remove bench test from s2n-quic CI (#2418) --- .github/workflows/qns.yml | 105 ---------------------------- scripts/benchmark/run | 139 -------------------------------------- scripts/benchmark/run-all | 21 ------ 3 files changed, 265 deletions(-) delete mode 100755 scripts/benchmark/run delete mode 100755 scripts/benchmark/run-all diff --git a/.github/workflows/qns.yml b/.github/workflows/qns.yml index 96ac1336fd..2b24f8cc53 100644 --- a/.github/workflows/qns.yml +++ b/.github/workflows/qns.yml @@ -339,111 +339,6 @@ jobs: --required .github/interop/required.json \ web/logs/latest/result.json - bench: - runs-on: ubuntu-22.04 - needs: [env, s2n-quic-qns] - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Install rust toolchain - id: toolchain - run: | - rustup toolchain install stable - rustup override set stable - - - uses: camshaft/rust-cache@v1 - - - name: Install tshark - run: | - wget --no-verbose https://dnglbrstg7yg.cloudfront.net/tshark/v$WIRESHARK_VERSION/tshark - chmod +x tshark - sudo mv tshark /usr/bin - /usr/bin/tshark -v - - - name: Install gnuplot - run: | - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install -y gnuplot - - - uses: aws-actions/configure-aws-credentials@v4.0.2 - if: github.repository == github.event.pull_request.head.repo.full_name - with: - role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCEcrRole - role-session-name: S2nQuicGHAECRSession - aws-region: us-east-1 # Required for ECR - - # authenticate pull to avoid hitting pull quota - - name: Login to Amazon ECR Public - if: github.repository == github.event.pull_request.head.repo.full_name - id: login-ecr-public - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - - - name: Pull s2n-quic-qns:main - if: github.event.pull_request - run: docker pull public.ecr.aws/s2n/s2n-quic-qns:main - - - uses: actions/download-artifact@v4 - with: - name: s2n-quic-qns-debug - path: s2n-quic-qns-build/ - - - uses: actions/download-artifact@v4 - with: - name: s2n-quic-qns-release - path: s2n-quic-qns-build/ - - - name: Setup dockerfile - working-directory: s2n-quic-qns-build - run: | - cp ../quic/s2n-quic-qns/etc/Dockerfile . - cp ../quic/s2n-quic-qns/etc/run_endpoint.sh . - - - name: Run docker build - working-directory: s2n-quic-qns-build - env: - DOCKER_BUILDKIT: 1 - run: | - docker build . --file Dockerfile --target prebuilt --tag aws/s2n-quic-qns - - - name: Run script for pull request - if: github.event.pull_request - run: sudo env "PATH=$PATH" "BUILD_S2N_QUIC=false" "COMPARE_TO_MAIN=true" ./scripts/benchmark/run-all - - - name: Run script for push to main - if: github.event_name == 'push' - run: sudo env "PATH=$PATH" "BUILD_S2N_QUIC=false" ./scripts/benchmark/run-all - - - uses: aws-actions/configure-aws-credentials@v4.0.2 - if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name - with: - role-to-assume: arn:aws:iam::024603541914:role/GitHubOIDCRole - role-session-name: S2nQuicGHAS3Session - aws-region: us-west-2 - - - name: Upload results - if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name - id: s3 - run: | - TARGET="${{ github.sha }}/bench" - aws s3 sync target/benchmark/results "s3://s2n-quic-ci-artifacts/$TARGET" --acl private --follow-symlinks - URL="$CDN/$TARGET/index.html" - echo "URL=$URL" >> $GITHUB_OUTPUT - - - uses: ouzi-dev/commit-status-updater@v2.0.2 - if: github.event_name == 'push' || github.repository == github.event.pull_request.head.repo.full_name - with: - name: "bench / report" - status: "success" - url: "${{ steps.s3.outputs.URL }}" - - - name: Assert no crashes - run: | - ! grep -Rq 'The s2n-quic-qns application shut down unexpectedly' target/benchmark/results - h3spec: runs-on: ubuntu-22.04 needs: [s2n-quic-qns] diff --git a/scripts/benchmark/run b/scripts/benchmark/run deleted file mode 100755 index fc710c42e8..0000000000 --- a/scripts/benchmark/run +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -# - -set -e - -function ensure_executable() { - if ! command -v $1 &> /dev/null; then - echo "$1 needs to be installed" - exit - fi -} - -ensure_executable "docker compose" -ensure_executable "tshark" -ensure_executable "gnuplot" - -case "$(uname -s)" in - Linux) - sudo modprobe ip6table_filter - ;; - *) - # TODO possibly add other things for Darwin - ;; -esac - -ROOT_DIR=$(realpath "$(dirname $0)/../../") -OUTPUT_DIR="${ROOT_DIR}/target/benchmark/results" -CLIENT_PCAP="sim/trace_node_left.pcap" -DOWNLOAD_MB=1000 -UPLOAD_MB=0 -DURATION=15 -SCENARIO=${1:-"simple-p2p --delay=15ms --bandwidth=2Mbps --queue=25"} -NAME=${SCENARIO//[^a-zA-Z0-9]/-} - -mkdir -p ${OUTPUT_DIR/NAME} -TMP=$(mktemp -d -t s2n-quic-bench-XXXXXXXXXX) - -# Create the initial plot script for Gnuplot -cat < "${TMP}/plotcmd.p" - set grid - set autoscale - set title "${SCENARIO}" - set timefmt "%H:%M:%S" - set xdata time - set xlabel "Time measurement" - set ylabel "Megabits/second" - set term svg size 900,480 - set output "${OUTPUT_DIR}/${NAME}.svg" -EOT - -LOG_DIR="$OUTPUT_DIR/~logs/$NAME/s2n-quic" - -if [ "$BUILD_S2N_QUIC" != false ]; then - docker build . -f ./quic/s2n-quic-qns/etc/Dockerfile.build -t aws/s2n-quic -fi - -ROOT_DIR="$ROOT_DIR" \ -LOG_DIR="$LOG_DIR" \ -DOWNLOAD_B=$(($DOWNLOAD_MB * 1000000)) \ -UPLOAD_B=$(($UPLOAD_MB * 1000000)) \ -DURATION="$DURATION" \ -SCENARIO="$SCENARIO" \ -docker compose --file quic/s2n-quic-qns/benchmark/docker-compose.yml up --abort-on-container-exit --timeout 1 sim client server - -tshark -r "$LOG_DIR/$CLIENT_PCAP" -t u -qz io,stat,1,"SUM(ip.len)"ip.len\&\&ip.dst==193.167.0.100 | awk -F '[\\|\s]' '{ print $2 $3}' > "${TMP}/s2n-quic.nsv" -# Find the time of the first real data transfer packet -QUIC_OFFSET=$(tshark -r "$LOG_DIR/$CLIENT_PCAP" -T fields -e frame.time_epoch -c 100 -Y "quic.header_form == 0" | head -n 1) -jq -r '.intervals[].sum | [(.start | strftime("%H:%M:%S")), .bits_per_second/1000000] | @tsv' "$LOG_DIR/client/perf_client.json" > "${TMP}/s2n-quic.tsv" - -# Append to the plot script -cat <> "${TMP}/plotcmd.p" - plot "${TMP}/s2n-quic.nsv" using 1:(\$2/125000) title 's2n-quic Network' with lines lt 1, \ -"${TMP}/s2n-quic.tsv" using (timecolumn(1) + "${QUIC_OFFSET}"):2 title 's2n-quic Streams' with lines lt 2, -EOT -# Remove trailing newline -truncate -s -1 "${TMP}/plotcmd.p" - -if [ "$COMPARE_TO_MAIN" = true ] ; then - LOG_DIR="$OUTPUT_DIR/~logs/$NAME/s2n-quic-main" - - ROOT_DIR="$ROOT_DIR" \ - LOG_DIR="$LOG_DIR" \ - DOWNLOAD_B=$(($DOWNLOAD_MB * 1000000)) \ - UPLOAD_B=$(($UPLOAD_MB * 1000000)) \ - DURATION="$DURATION" \ - SCENARIO="$SCENARIO" \ - docker compose --file quic/s2n-quic-qns/benchmark/docker-compose.yml up --abort-on-container-exit --timeout 1 sim client server-main - - tshark -r "$LOG_DIR/$CLIENT_PCAP" -t u -qz io,stat,1,"SUM(ip.len)"ip.len\&\&ip.dst==193.167.0.100 | awk -F '[\\|\s]' '{ print $2 $3}' > "${TMP}/s2n-quic-main.nsv" - # Find the time of the first real data transfer packet - QUIC_MAIN_OFFSET=$(tshark -r "$LOG_DIR/$CLIENT_PCAP" -T fields -e frame.time_epoch -c 100 -Y "quic.header_form == 0" | head -n 1) - jq -r '.intervals[].sum | [(.start | strftime("%H:%M:%S")), .bits_per_second/1000000] | @tsv' "$LOG_DIR/client/perf_client.json" > "${TMP}/s2n-quic-main.tsv" - - # Append to the plot script - cat <> "${TMP}/plotcmd.p" -"${TMP}/s2n-quic-main.nsv" using 1:(\$2/125000) title 's2n-quic main Network' with lines lt 3, \ -"${TMP}/s2n-quic-main.tsv" using (timecolumn(1) + "${QUIC_MAIN_OFFSET}"):2 title 's2n-quic main Streams' with lines lt 4, -EOT - - # Remove trailing newline - truncate -s -1 "${TMP}/plotcmd.p" -fi - -LOG_DIR="$OUTPUT_DIR/~logs/$NAME/iperf" - -# iperf3 occasionally crashes, so retry a couple times -MAX_ATTEMPTS=3 -ATTEMPTS=1 -until [ "$ATTEMPTS" -gt $MAX_ATTEMPTS ] -do - echo "Attempt ${ATTEMPTS} of ${MAX_ATTEMPTS}" - ROOT_DIR="$ROOT_DIR" \ - LOG_DIR="$LOG_DIR" \ - SCENARIO="$SCENARIO" \ - DURATION="$DURATION" \ - docker compose --file quic/s2n-quic-qns/benchmark/docker-compose.yml up --abort-on-container-exit --timeout 1 --exit-code-from iperf_client sim iperf_client iperf_server && break - ATTEMPTS=$((ATTEMPTS+1)) -done - -tshark -r "$LOG_DIR/$CLIENT_PCAP" -t u -qz io,stat,1,"SUM(ip.len)"ip.len\&\&ip.dst==193.167.0.90 | awk -F '[\\|\s]' '{ print $2 $3}' > "${TMP}/iperf.nsv" -# Find the time of the first real data transfer packet -IPERF_OFFSET=$(tshark -r "$LOG_DIR/$CLIENT_PCAP" -T fields -e frame.time_epoch -c 100 -Y "ip.len > 1000" | head -n 1) -jq -r '.intervals[].sum | [(.start | strftime("%H:%M:%S")), .bits_per_second/1000000] | @tsv' "$LOG_DIR/client/iperf.json" > "${TMP}/iperf.tsv" - -# Append to the plot script -cat <> "${TMP}/plotcmd.p" -"${TMP}/iperf.nsv" using 1:(\$2/125000) title 'iperf TCP Network' with lines lt 5, \ -"${TMP}/iperf.tsv" using (timecolumn(1) + "${IPERF_OFFSET}"):2 title 'iperf TCP Streams' with lines lt 6 -EOT - -gnuplot "${TMP}/plotcmd.p" - -rm -rf $TMP - -echo "plot available in $OUTPUT_DIR/${NAME}.svg" diff --git a/scripts/benchmark/run-all b/scripts/benchmark/run-all deleted file mode 100755 index 4de9337fbd..0000000000 --- a/scripts/benchmark/run-all +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -# - -set -e - -./scripts/benchmark/run "simple-p2p --delay=15ms --bandwidth=2Mbps --queue=25" -./scripts/benchmark/run "simple-p2p --delay=15ms --bandwidth=5Mbps --queue=25" -./scripts/benchmark/run "simple-p2p --delay=15ms --bandwidth=10Mbps --queue=25" -./scripts/benchmark/run "simple-p2p --delay=15ms --bandwidth=10Mbps --queue=10" -./scripts/benchmark/run "simple-p2p --delay=15ms --bandwidth=10Mbps --queue=50" -./scripts/benchmark/run "simple-p2p --delay=15ms --bandwidth=10Mbps --queue=2048" -./scripts/benchmark/run "drop-rate --delay=15ms --bandwidth=10Mbps --queue=25 --rate_to_client=1 --rate_to_server=1" -./scripts/benchmark/run "drop-rate --delay=15ms --bandwidth=10Mbps --queue=25 --rate_to_client=2 --rate_to_server=2" -./scripts/benchmark/run "drop-rate --delay=15ms --bandwidth=10Mbps --queue=25 --rate_to_client=5 --rate_to_server=5" - -cd target/benchmark/results -tree -H "." -T "Benchmark Results" --noreport --charset utf-8 -I "*index.html" > index.html