From 887be9e714c5247b171a2000b54a0dacaa0149c8 Mon Sep 17 00:00:00 2001 From: Quentin McGaw Date: Wed, 3 Nov 2021 11:44:43 -0400 Subject: [PATCH 1/2] chore(ci): remove dependency on `scripts/ci.go` (#1964) - Remove unused `scripts/ci.sh` - Dockerfile: build Go binary without `scripts/ci.go` - Build Go binary with `-trimpath` - Makefile: - remove outdated `ci.go test` from Makefile - build binary without `scripts/ci.go` - build binary without changing directory - `install` binary by executing `build` target and just `mv` the built binary --- Dockerfile | 7 ++++- Makefile | 12 ++++---- scripts/ci.go | 81 --------------------------------------------------- scripts/ci.sh | 27 ----------------- 4 files changed, 11 insertions(+), 116 deletions(-) delete mode 100755 scripts/ci.go delete mode 100755 scripts/ci.sh diff --git a/Dockerfile b/Dockerfile index 82fbb4d2ad..327090342c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,12 @@ RUN cp /go/pkg/mod/github.com/wasmerio/go-ext-wasm@*/wasmer/libwasmer.so libwasm COPY . . # Build -RUN GOBIN=$GOPATH/src/github.com/ChainSafe/gossamer/bin go run scripts/ci.go install +ARG GO_BUILD_FLAGS +RUN go build \ + -trimpath \ + -o ./bin/gossamer \ + ${GO_BUILD_FLAGS} \ + ./cmd/gossamer # Final stage based on Debian FROM debian:${DEBIAN_VERSION} diff --git a/Makefile b/Makefile index 8cfa4bee33..3f0b633887 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,6 @@ proto: ## test: Runs `go test` on project test files. test: @echo " > \033[32mRunning tests...\033[0m " - #GOBIN=$(PWD)/bin go run scripts/ci.go test git lfs pull go test -short -coverprofile c.out ./... -timeout=30m @@ -80,11 +79,11 @@ deps: ## build: Builds application binary and stores it in `./bin/gossamer` build: @echo " > \033[32mBuilding binary...\033[0m " - GOBIN=$(PWD)/bin go run scripts/ci.go install + go build -trimpath -o ./bin/gossamer ./cmd/gossamer ## debug: Builds application binary with debug flags and stores it in `./bin/gossamer` build-debug: clean - cd cmd/gossamer && go build -gcflags=all="-N -l" -o ../../bin/gossamer && cd ../.. + go build -trimpath -gcflags=all="-N -l" -o ./bin/gossamer ./cmd/gossamer ## init: Initialise gossamer using the default genesis and toml configuration files init: @@ -120,9 +119,8 @@ docker-build: @echo " > \033[32mBuilding Docker Container...\033[0m " docker build -t $(FULLDOCKERNAME) -f Dockerfile . -gossamer: clean - cd cmd/gossamer && go build -o ../../bin/gossamer && cd ../.. +gossamer: clean build ## install: install the gossamer binary in $GOPATH/bin -install: - GOBIN=$(GOPATH)/bin go run scripts/ci.go install +install: build + mv ./bin/gossamer $(GOPATH)/bin/gossamer diff --git a/scripts/ci.go b/scripts/ci.go deleted file mode 100755 index 7a805020de..0000000000 --- a/scripts/ci.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2019 ChainSafe Systems (ON) Corp. -// This file is part of gossamer. -// -// The gossamer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The gossamer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the gossamer library. If not, see . - -//go:build none -// +build none - -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" -) - -func main() { - - log.SetFlags(log.Lshortfile) - - if _, err := os.Stat(filepath.Join("scripts", "ci.go")); os.IsNotExist(err) { - log.Fatal("should run build from root dir") - } - if len(os.Args) < 2 { - log.Fatal("cmd required, eg: install") - } - switch os.Args[1] { - case "install": - install(false) - case "install-debug": - install(true) - default: - log.Fatal("cmd not found", os.Args[1]) - } -} - -func install(debug bool) { - argsList := append([]string{"list"}, []string{"./..."}...) - - cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), argsList...) - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatalf("could not list packages: %v\n%s", err, string(out)) - } - var packages []string - for _, line := range strings.Split(string(out), "\n") { - if strings.Contains(line, "/gossamer/") { - packages = append(packages, strings.TrimSpace(line)) - } - } - - argsInstall := append([]string{"install"}) - cmd = exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), argsInstall...) - cmd.Args = append(cmd.Args, "-v") - if debug { - cmd.Args = append(cmd.Args, "-gcflags=\"all=-N -l\"") - } - cmd.Args = append(cmd.Args, packages...) - - fmt.Println("Build Gossamer", strings.Join(cmd.Args, " \\\n")) - cmd.Stderr, cmd.Stdout = os.Stderr, os.Stdout - - if err := cmd.Run(); err != nil { - log.Fatal("Error: Could not build Gossamer. ", "error: ", err, ", cmd: ", cmd) - } -} diff --git a/scripts/ci.sh b/scripts/ci.sh deleted file mode 100755 index 2ff381c431..0000000000 --- a/scripts/ci.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Copyright 2019 ChainSafe Systems (ON) Corp. -# This file is part of gossamer. -# -# The gossamer library is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# The gossamer library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with the gossamer library. If not, see . - -set -e - -echo ">> Running tests..." -go test -short -coverprofile c.out ./... -timeout=15m -#./cc-test-reporter after-build --exit-code $? -# echo ">> Running race condition test on runtime" -# go test -short -race ./lib/runtime -echo ">> Running race condition test on priority queue" -go test -short -race ./lib/transaction/ From dd80c09670b076cfbaa1bda2ec0c8b2d83253ad7 Mon Sep 17 00:00:00 2001 From: noot <36753753+noot@users.noreply.github.com> Date: Wed, 3 Nov 2021 16:54:25 -0400 Subject: [PATCH 2/2] fix(dot/state): fix deadlock, fixes bootstrap syncing (#1959) --- dot/core/service.go | 1 - dot/state/block.go | 8 +++++--- dot/state/block_finalisation.go | 4 +++- dot/sync/chain_processor.go | 17 ++++++++--------- lib/runtime/test_helpers.go | 13 +++++++++---- scripts/install-lint.sh | 2 +- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/dot/core/service.go b/dot/core/service.go index 289881e8dc..f79dd56f5e 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -324,7 +324,6 @@ func (s *Service) handleBlocksAsync() { } s.maintainTransactionPool(block) - case <-s.ctx.Done(): return } diff --git a/dot/state/block.go b/dot/state/block.go index 46fc641b46..09632c733a 100644 --- a/dot/state/block.go +++ b/dot/state/block.go @@ -221,6 +221,10 @@ func (bs *BlockState) getAndDeleteUnfinalisedBlock(hash common.Hash) (*types.Blo return block.(*types.Block), true } +func (bs *BlockState) deleteUnfinalisedBlock(hash common.Hash) { + bs.unfinalisedBlocks.Delete(hash) +} + // HasHeader returns if the db contains a header with the given hash func (bs *BlockState) HasHeader(hash common.Hash) (bool, error) { if bs.hasUnfinalisedBlock(hash) { @@ -327,6 +331,7 @@ func (bs *BlockState) GetBlockByHash(hash common.Hash) (*types.Block, error) { if err != nil { return nil, err } + return &types.Block{Header: *header, Body: *blockBody}, nil } @@ -360,9 +365,6 @@ func (bs *BlockState) HasBlockBody(hash common.Hash) (bool, error) { // GetBlockBody will return Body for a given hash func (bs *BlockState) GetBlockBody(hash common.Hash) (*types.Body, error) { - bs.RLock() - defer bs.RUnlock() - block, has := bs.getUnfinalisedBlock(hash) if has { return &block.Body, nil diff --git a/dot/state/block_finalisation.go b/dot/state/block_finalisation.go index aaeb9ffcfc..2e85ac37d4 100644 --- a/dot/state/block_finalisation.go +++ b/dot/state/block_finalisation.go @@ -226,7 +226,7 @@ func (bs *BlockState) handleFinalisedBlock(curr common.Hash) error { continue } - block, has := bs.getAndDeleteUnfinalisedBlock(hash) + block, has := bs.getUnfinalisedBlock(hash) if !has { return fmt.Errorf("failed to find block in unfinalised block map, block=%s", hash) } @@ -251,6 +251,8 @@ func (bs *BlockState) handleFinalisedBlock(curr common.Hash) error { if err = batch.Put(headerHashKey(block.Header.Number.Uint64()), hash.ToBytes()); err != nil { return err } + + bs.deleteUnfinalisedBlock(hash) } return batch.Flush() diff --git a/dot/sync/chain_processor.go b/dot/sync/chain_processor.go index 9333a4e600..bc7972ea45 100644 --- a/dot/sync/chain_processor.go +++ b/dot/sync/chain_processor.go @@ -115,11 +115,6 @@ func (s *chainProcessor) processBlockData(bd *types.BlockData) error { return ErrNilBlockData } - err := s.blockState.CompareAndSetBlockData(bd) - if err != nil { - return fmt.Errorf("failed to compare and set data: %w", err) - } - hasHeader, _ := s.blockState.HasHeader(bd.Hash) hasBody, _ := s.blockState.HasBlockBody(bd.Hash) if hasHeader && hasBody { @@ -164,8 +159,10 @@ func (s *chainProcessor) processBlockData(bd *types.BlockData) error { return nil } + logger.Debug("processing block data", "hash", bd.Hash) + if bd.Header != nil && bd.Body != nil { - if err = s.handleHeader(bd.Header); err != nil { + if err := s.handleHeader(bd.Header); err != nil { return err } @@ -176,9 +173,7 @@ func (s *chainProcessor) processBlockData(bd *types.BlockData) error { Body: *bd.Body, } - logger.Debug("processing block", "hash", bd.Hash) - - if err = s.handleBlock(block); err != nil { + if err := s.handleBlock(block); err != nil { logger.Error("failed to handle block", "number", block.Header.Number, "error", err) return err } @@ -191,6 +186,10 @@ func (s *chainProcessor) processBlockData(bd *types.BlockData) error { s.handleJustification(bd.Header, *bd.Justification) } + if err := s.blockState.CompareAndSetBlockData(bd); err != nil { + return fmt.Errorf("failed to compare and set data: %w", err) + } + return nil } diff --git a/lib/runtime/test_helpers.go b/lib/runtime/test_helpers.go index c6e969e69d..59da6fd68d 100644 --- a/lib/runtime/test_helpers.go +++ b/lib/runtime/test_helpers.go @@ -89,9 +89,6 @@ func GetRuntimeBlob(testRuntimeFilePath, testRuntimeURL string) (n int64, err er if err != nil { return 0, err } - defer func() { - _ = out.Close() - }() /* #nosec */ resp, err := http.Get(testRuntimeURL) @@ -103,7 +100,15 @@ func GetRuntimeBlob(testRuntimeFilePath, testRuntimeURL string) (n int64, err er }() n, err = io.Copy(out, resp.Body) - return n, err + if err != nil { + return 0, err + } + + if err = out.Close(); err != nil { + return 0, err + } + + return n, nil } // TestRuntimeNetwork ... diff --git a/scripts/install-lint.sh b/scripts/install-lint.sh index 30b7c6baaa..bd1e4cf175 100755 --- a/scripts/install-lint.sh +++ b/scripts/install-lint.sh @@ -6,7 +6,7 @@ fi if ! command -v golangci-lint &> /dev/null then - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.41.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.43.0 fi export PATH=$PATH:$(go env GOPATH)/bin \ No newline at end of file