Merge pull request erigontech#4 from maticnetwork/upstream-changes

Upstream changes

.env.example

@@ -0,0 +1,6 @@
+# host OS dedicated user (for docker especially)
+ERIGON_USER=erigon
+
+# UID, GID of user inside docker process which must exist also on host OS
+DOCKER_UID=1000     # random number [1001, 10000] chosen arbitrarily for example
+DOCKER_GID=1000     # can choose any valid #. 1000 tends to be taken by first user

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Continuous integration
+name: CI
 on:
   push:
     branches:
@@ -10,8 +10,15 @@ on:
       - devel
       - alpha
       - stable
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
 jobs:
   tests:
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
         os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
@@ -53,14 +60,13 @@ jobs:
         if: runner.os == 'Linux'
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.46
-          skip-pkg-cache: true
-          skip-build-cache: true
+          version: v1.47
 
       - name: Test
         run: make test
 
   tests-windows:
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
         os: [ windows-2022 ]
@@ -99,9 +105,63 @@ jobs:
         run: .\wmake.ps1 test
 
   docker:
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # fetch git tags for "git describe"
+
+      - name: make docker
+        run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+      # check with root permissions, should be cached from previous build
+      - name: sudo make docker
+        run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+  hive:
+    needs:
+      - tests
+      - tests-windows
+      - docker
     runs-on: ubuntu-20.04
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0 # fetch git tags for "git describe"
-      - run: make docker
+
+      - name: build erigon image
+        run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+      # check with root permissions, should be cached from previous build
+      - name: build erigon image (root permissions)
+        run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
+
+      - name: run hive
+        run: sudo mkdir /results && docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work gatewayfm/hive:latest --sim ethereum/engine --results-root=/work/results --client erigon_ci-$GITHUB_SHA --docker.output --loglevel 5
+
+      - name: parse hive output
+        run: docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}:/work --entrypoint /app/hivecioutput gatewayfm/hive:latest --resultsdir=/work/results --outdir=/work/results
+
+      - name: archive hive results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: hive-ci-output
+          path: results/*.xml
+
+  event_file:
+    needs:
+      - tests
+      - tests-windows
+      - docker
+    name: archive event file
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+    steps:
+      - name: upload
+        uses: actions/upload-artifact@v2
+        with:
+          name: event file
+          path: ${{ github.event_path }}

.github/workflows/coverage.yml

@@ -0,0 +1,55 @@
+name: Coverage
+on:
+  push:
+    branches:
+      - devel
+
+jobs:
+  coverage:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive --force
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.x
+
+      - name: install dependencies on Linux
+        if: runner.os == 'Linux'
+        run: sudo apt update && sudo apt install build-essential
+
+      - name: run coverage
+        run: echo "COVERAGE=$(make coverage)" >> $GITHUB_ENV
+
+      - name: set badge color
+        shell: bash
+        run: |
+          if [ ${{ env.COVERAGE }} -lt 40 ] 
+          then
+          echo "BADGE_COLOR=800000" >> $GITHUB_ENV
+          elif [ ${{ env.COVERAGE }} -lt 75 ]
+          then
+              echo "BADGE_COLOR=696969" >> $GITHUB_ENV
+          else
+              echo "BADGE_COLOR=31c653" >> $GITHUB_ENV
+          fi
+
+      - name: create badge
+        uses: emibcn/badge-action@d6f51ff11b5c3382b3b88689ae2d6db22d9737d1
+        with:
+          label: Coverage
+          status: ${{ env.COVERAGE }}
+          color: ${{ env.BADGE_COLOR }}
+          path: badge.svg
+
+      - name: upload badge to gist
+        if: >
+          github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'devel' ||
+          github.event_name != 'workflow_run' && github.ref == 'refs/heads/devel'
+        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        with:
+          token: ${{ secrets.GIST_TOKEN }}
+          gistURL: https://gist.githubusercontent.com/revittm/ee38e9beb22353eef6b88f2ad6ed7aa9
+          file: badge.svg

.github/workflows/hive-results.yml

@@ -0,0 +1,75 @@
+name: Hive results
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+
+jobs:
+  hive-results:
+    name: Hive results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
+
+    steps:
+      - name: download and extract artifacts
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          mkdir -p artifacts && cd artifacts
+          
+          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+          
+          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
+          do
+            IFS=$'\t' read name url <<< "$artifact"
+            gh api $url > "$name.zip"
+            unzip -d "$name" "$name.zip"
+          done
+
+      - name: publish hive test results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/**/*.xml"
+
+      - name: set badge color
+        shell: bash
+        run: |
+          case ${{ fromJSON( steps.test-results.outputs.json ).conclusion }} in
+            success)
+              echo "BADGE_COLOR=31c653" >> $GITHUB_ENV
+              ;;
+            failure)
+              echo "BADGE_COLOR=800000" >> $GITHUB_ENV
+              ;;
+            neutral)
+              echo "BADGE_COLOR=696969" >> $GITHUB_ENV
+              ;;
+          esac
+
+      - name: create badge
+        uses: emibcn/badge-action@d6f51ff11b5c3382b3b88689ae2d6db22d9737d1
+        with:
+          label: Hive
+          status: '${{ fromJSON( steps.test-results.outputs.json ).formatted.stats.tests }} tests, ${{ fromJSON( steps.test-results.outputs.json ).formatted.stats.runs }} runs: ${{ fromJSON( steps.test-results.outputs.json ).conclusion }}'
+          color: ${{ env.BADGE_COLOR }}
+          path: badge.svg
+
+      - name: upload badge to gist
+        if: >
+          github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'devel' ||
+          github.event_name != 'workflow_run' && github.ref == 'refs/heads/devel'
+        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        with:
+          token: ${{ secrets.GIST_TOKEN }}
+          gistURL: https://gist.githubusercontent.com/revittm/dc492845ba6eb694e6c7279224634b20
+          file: badge.svg

.gitignore

@@ -77,3 +77,6 @@ go.work
 docker-compose.*.yml
 
 dist
+.env
+coverage.out
+

.gitmodules

@@ -1,13 +1,6 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
-[submodule "turbo/snapshotsync/snapshothashes/erigon-snapshots"]
-	path = turbo/snapshotsync/snapshothashes/erigon-snapshots
-	url = https://github.com/ledgerwatch/erigon-snapshot.git
-[submodule "cmd/downloader/trackers/trackerslist"]
-	path = cmd/downloader/trackers/trackerslist
-	url = https://github.com/ngosang/trackerslist.git
-
 [submodule "libmdbx"]
 	path = libmdbx
 	url = https://github.com/torquem-ch/libmdbx.git

Dockerfile

@@ -14,29 +14,41 @@ RUN --mount=type=cache,target=/root/.cache \
 FROM docker.io/library/alpine:3.15
 
 RUN apk add --no-cache ca-certificates libstdc++ tzdata
+# copy compiled artifacts from builder
 COPY --from=builder /app/build/bin/* /usr/local/bin/
 
-ARG PUID=1000
-ARG PGID=1000
-RUN adduser -H -u ${PUID} -g ${PGID} -D erigon
-RUN mkdir -p /home/erigon
-RUN mkdir -p /home/erigon/.local/share/erigon
-RUN chown -R erigon:erigon /home/erigon
-
+# Setup user and group
+#
+# from the perspective of the container, uid=1000, gid=1000 is a sensible choice
+# (mimicking Ubuntu Server), but if caller creates a .env (example in repo root),
+# these defaults will get overridden when make calls docker-compose
+ARG UID=1000
+ARG GID=1000
+RUN adduser -D -u $UID -g $GID erigon
 USER erigon
-
-EXPOSE 8545 8551 8546 30303 30303/udp 42069 42069/udp 8080 9090 6060
+RUN mkdir -p ~/.local/share/erigon
+
+EXPOSE 8545 \
+       8551 \
+       8546 \
+       30303 \
+       30303/udp \
+       42069 \
+       42069/udp \
+       8080 \
+       9090 \
+       6060
 
 # https://github.com/opencontainers/image-spec/blob/main/annotations.md
 ARG BUILD_DATE
 ARG VCS_REF
 ARG VERSION
 LABEL org.label-schema.build-date=$BUILD_DATE \
-      org.label-schema.name="Erigon" \
       org.label-schema.description="Erigon Ethereum Client" \
+      org.label-schema.name="Erigon" \
+      org.label-schema.schema-version="1.0" \
       org.label-schema.url="https://torquem.ch" \
       org.label-schema.vcs-ref=$VCS_REF \
       org.label-schema.vcs-url="https://github.com/ledgerwatch/erigon.git" \
       org.label-schema.vendor="Torquem" \
-      org.label-schema.version=$VERSION \
-      org.label-schema.schema-version="1.0"
+      org.label-schema.version=$VERSION