Skip to content

Commit

Permalink
test: Retry in the case of network errors (#2606)
Browse files Browse the repository at this point in the history
# Description
This is an attempt to reduce test flakiness.  In particular, many tests die due to external network errors.  These failures can probably be addressed by retries.

Note:  This does not attempt to address internal network errors.  E.g. port conflicts (I have seen this only ~once~ twice on a random port) or `dfx start` failing to respond to http requests.
  • Loading branch information
bitdivine committed Sep 29, 2022
1 parent 38914bd commit b475418
Show file tree
Hide file tree
Showing 19 changed files with 103 additions and 1 deletion.
1 change: 1 addition & 0 deletions .curlrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
--retry-max-time 60
6 changes: 6 additions & 0 deletions .github/workflows/audit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
build_matrix:
runs-on: ubuntu-latest
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/build-frontend-canister.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ on:
- master
pull_request:

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
frontend:
runs-on: ubuntu-latest
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/consistent-sources.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
check:
runs-on: ubuntu-latest
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/deny.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
cargo-deny:
name: license-check:required
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
build_dfx:
runs-on: ${{ matrix.os }}
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/fmt.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
test:
name: fmt
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
test:
name: lint
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/motoko-updater.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
niv-updater:
name: 'Check for Motoko updates'
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/niv-updater.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
niv-updater:
name: 'Check for updates'
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
check:
runs-on: ubuntu-latest
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/prepare-dfx-assets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
prepare:
runs-on: ${{ matrix.os }}
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/publish-manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
publish-manifest:
runs-on: ubuntu-latest
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
build_dfx:
runs-on: ${{ matrix.os }}
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/shellcheck.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
check_macos:
# ubuntu-latest has shellcheck 0.4.6, while macos-latest has 0.7.1
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/unit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
test:
runs-on: ${{ matrix.os }}
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/update-docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
update_dfx_json_schema:
runs-on: ${{ matrix.os }}
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/update-replica-version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ concurrency:

env:
IC_RELEASES_API: "https://ic-api.internetcomputer.org/api/v3/subnet-replica-versions?limit=50&offset=0"

# When getting Rust dependencies, retry on network error:
CARGO_NET_RETRY: 10
# Use the local .curlrc
CURL_HOME: .

jobs:
update-replica:
runs-on: ubuntu-latest
Expand Down
1 change: 1 addition & 0 deletions .npmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fetch-retries=10

0 comments on commit b475418

Please sign in to comment.