feat(rollup): add circuits capacity checker (ethereum#321)

* add proof for predeployed storages * reverse inneeded code * update for mainbranch merging * add pertx storage trace * dummy tx proof * add txstorage trace * add coinbase storage as trace * enable (sync) tracing by default * basic * init cgo framework * fix break loop * integrate the right zkevm version (ethereum#323) * finish rust codes * use dylib (ethereum#325) * flip * ? * use cdylib * revert * fix * apply_tx * rename * fixing types fixing types fixing types * clean up * ExecutionResults (ethereum#328) * filling * filling * more * clean up * filling * coinbase * add * MPTWitness * ExecutionResults WIP * L1fee L1fee * sender * to * Failed & ReturnValue * createdAcc & after * remove MPTWitness * txStorageTrace * add FeeRecipient * add StorageTrace * fix FFI types * better logger * cargo fmt * fix * add build tags * update Makefile * fix library * improve ld path * correctly deal with circuit_capacity_checker returned result * fix return value * update cargo (ethereum#333) * update cargo * update * update go * refactor * raname `circuits capacity checker` to `circuit capacity checker` * some refactorings * [Fix] storage proof generation in capacity checker (ethereum#348) * make per-tx storage and deletion proof work * format * fix misplaced markdeletion --------- Co-authored-by: HAOYUatHZ <[email protected]> * docker (ethereum#363) * update Dockerfile * build: update go version to 1.19 * update * fix * fix * try * simplify * revert go version update l2geth Dockerfiles * fix * fix coinbase * fix (ethereum#369) * format * Update version.go * address comments * Capacity refactor (ethereum#374) * init * id * support multiple instances * fix id * fix conflicts * refactor to use same codes (ethereum#379) * re-init * WIP * WIP * refactor * go * minor * fix storage proof of l1gas price oracle * move 1 * move 2 * move 3 * move 4 * move 5 move 5 * move 6 move 6 * move 7 * move 8 * move 9 * move 10 * clean up clean up --------- Co-authored-by: Ho Vei <[email protected]> * finish basic * minor * config capacity check in block_validator (ethereum#380) * init * done ref * fix tests fix tests fix tests fix tests * add more comments * apply_block * improve logs * cargo fmt * Capacity big refactor (ethereum#383) * CreateTraceEnv * WIP * draft more fix * for test * fortet * clean up * add more comments * goimports -local github.com/scroll-tech/go-ethereum -w . * fix typos * attempt 1 * attempt 2 * attempt 3 * gogogo * clean up * fix * fix * rename * minor * fix * minor * minor * improve doc * use dedicated `checkCircuitCapacity` flag (ethereum#394) * refactor * fix * add lock * [feat] capacity checking: upgrade libzkp (ethereum#395) * upgrade * upgrade libzkp * write RowConsumption (ethereum#396) * write RowConsumption * name alignments * revert some formatting * add lock to CircuitCapacityChecker in BlockValidator * remove mutex pointer * improve github workflow * improve * store row consumption in mining (ethereum#397) * prepare * finish * add more logs * mark `ApplyBlock` as ready * update libzkp (ethereum#401) * fix * Capacity detail (ethereum#402) * fix(block-validation): consider skipping in ValidateL1Messages (ethereum#405) * fix(block-validation): consider skipping in ValidateL1Messages * fix(block): consider skipping in L1MessageCount * fix l1 validation tests * fix NumL1Messages * fix impl.go return types fix * better error handling (ethereum#407) * add add * add * add * add * add * cargo fmt * add * update * add * WIP * minor * gogogo * gogogo * fix * fix * fix * cargo clippy * improve * improve * creation lock (ethereum#408) * creation lock * update * Debug log (ethereum#409) * add more logs * more * more * fix * improve * Update cmd/utils/flags.go Co-authored-by: Péter Garamvölgyi <[email protected]> * refactor worker.commit() * avoid re-calculate * txpool ccc err handling (ethereum#411) * more explicit error comments * add more logs * fix unnecessary commit * add more logs * fix `ineffassign` * add more comments * log id for `NewCircuitCapacityChecker` (ethereum#414) add log to `NewCircuitCapacityChecker` * Persist skip info for block where all L1 msgs are skipped (ethereum#415) persist skip info for block where all L1 msgs are skipped * Update version.go --------- Co-authored-by: Ho Vei <[email protected]> Co-authored-by: Zhang Zhuo <[email protected]> Co-authored-by: Péter Garamvölgyi <[email protected]>
tanishqjasoria · Aug 1, 2023 · 2f9edf7 · 2f9edf7
1 parent 134e1b8
commit 2f9edf7
Show file tree

Hide file tree

Showing 56 changed files with 5,986 additions and 689 deletions.
diff --git a/.github/workflows/l2geth_ci.yml b/.github/workflows/l2geth_ci.yml
@@ -13,7 +13,7 @@ on:
       - ready_for_review
 name: CI
 jobs:
-  build:
+  build-mock-ccc-geth: # build geth with mock circuit capacity checker
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
@@ -23,6 +23,25 @@ jobs:
         go-version: 1.18.x
     - name: Checkout code
       uses: actions/checkout@v2
+    - name: Build
+      run: |
+        make nccc_geth
+  build-geth: # build geth with circuit capacity checker
+    if: github.event_name == 'push' # will only be triggered when pushing to main & staging & develop & alpha
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.19.x
+    - name: Install rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly-2022-12-10
+        override: true
+        components: rustfmt, clippy
+    - name: Checkout code
+      uses: actions/checkout@v2
     - name: Build
       run: |
         make geth

diff --git a/Dockerfile b/Dockerfile
@@ -3,19 +3,39 @@ ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
 
-# Build Geth in a stock Go builder container
-FROM golang:1.18-alpine as builder
+# Build libzkp dependency
+FROM scrolltech/go-rust-builder:go-1.19-rust-nightly-2022-12-10 as chef
+WORKDIR app
+
+FROM chef as planner
+COPY ./rollup/circuitcapacitychecker/libzkp/ .
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM chef as zkp-builder
+COPY ./rollup/circuitcapacitychecker/libzkp/rust-toolchain ./
+COPY --from=planner /app/recipe.json recipe.json
+RUN cargo chef cook --release --recipe-path recipe.json
 
-RUN apk add --no-cache gcc musl-dev linux-headers git
+COPY ./rollup/circuitcapacitychecker/libzkp .
+RUN cargo build --release
+RUN find ./ | grep libzktrie.so | xargs -I{} cp {} /app/target/release/
+
+# Build Geth in a stock Go builder container
+FROM scrolltech/go-rust-builder:go-1.19-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
-RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth
+COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
+COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
+RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
-FROM alpine:latest
+FROM ubuntu:20.04
 
-RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
+COPY --from=zkp-builder /app/target/release/libzkp.so /usr/local/lib/
+COPY --from=zkp-builder /app/target/release/libzktrie.so /usr/local/lib/
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
 
 EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

diff --git a/Dockerfile.alltools b/Dockerfile.alltools
@@ -4,7 +4,7 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.18-alpine as builder
+FROM golang:1.19-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 

diff --git a/Makefile b/Makefile
@@ -2,17 +2,26 @@
 # with Go source code. If you know what GOPATH is then you probably
 # don't need to bother with make.
 
-.PHONY: geth android ios evm all test clean
+.PHONY: geth android ios evm all test clean libzkp
 
 GOBIN = ./build/bin
 GO ?= latest
 GORUN = env GO111MODULE=on go run
 
-geth:
+
+libzkp:
+	cd $(PWD)/rollup/circuitcapacitychecker/libzkp && make libzkp
+
+nccc_geth: ## geth without circuit capacity checker
 	$(GORUN) build/ci.go install ./cmd/geth
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
+geth: libzkp
+	$(GORUN) build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
+	@echo "Done building."
+	@echo "Run \"$(GOBIN)/geth\" to launch geth."
+
 all:
 	$(GORUN) build/ci.go install
 

diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go
@@ -79,7 +79,7 @@ type SimulatedBackend struct {
 func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil, false)
 
 	backend := &SimulatedBackend{
 		database:   database,

diff --git a/build/ci.go b/build/ci.go
@@ -199,9 +199,10 @@ func main() {
 
 func doInstall(cmdline []string) {
 	var (
-		dlgo = flag.Bool("dlgo", false, "Download Go and build with it")
-		arch = flag.String("arch", "", "Architecture to cross build for")
-		cc   = flag.String("cc", "", "C compiler to cross build with")
+		dlgo      = flag.Bool("dlgo", false, "Download Go and build with it")
+		arch      = flag.String("arch", "", "Architecture to cross build for")
+		cc        = flag.String("cc", "", "C compiler to cross build with")
+		buildtags = flag.String("buildtags", "", "Tags for go build")
 	)
 	flag.CommandLine.Parse(cmdline)
 
@@ -215,6 +216,9 @@ func doInstall(cmdline []string) {
 	// Configure the build.
 	env := build.Env()
 	gobuild := tc.Go("build", buildFlags(env)...)
+	if len(*buildtags) != 0 {
+		gobuild.Args = append(gobuild.Args, "-tags", *buildtags)
+	}
 
 	// arm64 CI builders are memory-constrained and can't handle concurrent builds,
 	// better disable it. This check isn't the best, it should probably

diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
@@ -812,6 +812,12 @@ var (
 		Name:  "l1.sync.startblock",
 		Usage: "L1 block height to start syncing from. Should be set to the L1 message queue deployment block number.",
 	}
+
+	// Circuit capacity check settings
+	CircuitCapacityCheckEnabledFlag = cli.BoolFlag{
+		Name:  "ccc",
+		Usage: "Enable circuit capacity check during block validation",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1493,6 +1499,12 @@ func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) {
 	}
 }
 
+func setCircuitCapacityCheck(ctx *cli.Context, cfg *ethconfig.Config) {
+	if ctx.GlobalIsSet(CircuitCapacityCheckEnabledFlag.Name) {
+		cfg.CheckCircuitCapacity = ctx.GlobalBool(CircuitCapacityCheckEnabledFlag.Name)
+	}
+}
+
 // CheckExclusive verifies that only a single instance of the provided flags was
 // set by the user. Each flag might optionally be followed by a string type to
 // specialize it further.
@@ -1558,6 +1570,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setMiner(ctx, &cfg.Miner)
 	setWhitelist(ctx, cfg)
 	setLes(ctx, cfg)
+	setCircuitCapacityCheck(ctx, cfg)
 
 	// Cap the cache allowance and tune the garbage collector
 	mem, err := gopsutil.VirtualMemory()
@@ -2006,7 +2019,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 
 	// TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only.
 	// Disable transaction indexing/unindexing by default.
-	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil)
+	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil, false)
 	if err != nil {
 		Fatalf("Can't create BlockChain: %v", err)
 	}

diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go
@@ -55,7 +55,7 @@ func TestReimportMirroredState(t *testing.T) {
 	genesis := genspec.MustCommit(db)
 
 	// Generate a batch of blocks, each properly signed
-	chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil)
+	chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, false)
 	defer chain.Stop()
 
 	blocks, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, 3, func(i int, block *core.BlockGen) {
@@ -89,7 +89,7 @@ func TestReimportMirroredState(t *testing.T) {
 	db = rawdb.NewMemoryDatabase()
 	genspec.MustCommit(db)
 
-	chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil)
+	chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, false)
 	defer chain.Stop()
 
 	if _, err := chain.InsertChain(blocks[:2]); err != nil {
@@ -102,7 +102,7 @@ func TestReimportMirroredState(t *testing.T) {
 	// Simulate a crash by creating a new chain on top of the database, without
 	// flushing the dirty states out. Insert the last block, triggering a sidechain
 	// reimport.
-	chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil)
+	chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil, false)
 	defer chain.Stop()
 
 	if _, err := chain.InsertChain(blocks[2:]); err != nil {

diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go
@@ -450,7 +450,7 @@ func TestClique(t *testing.T) {
 			batches[len(batches)-1] = append(batches[len(batches)-1], block)
 		}
 		// Pass all the headers through clique and ensure tallying succeeds
-		chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil)
+		chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil, false)
 		if err != nil {
 			t.Errorf("test %d: failed to create test chain: %v", i, err)
 			continue

diff --git a/consensus/errors.go b/consensus/errors.go
@@ -45,8 +45,7 @@ var (
 
 	// ErrInvalidL1MessageOrder is returned if a block contains L1 messages in the wrong
 	// order. Possible scenarios are: (1) L1 messages do not follow their QueueIndex order,
-	// (2) the block skipped one or more L1 messages, (3) L1 messages are not included in
-	// a contiguous block at the front of the block.
+	// (2) L1 messages are not included in a contiguous block at the front of the block.
 	ErrInvalidL1MessageOrder = errors.New("invalid L1 message order")
 
 	// ErrUnknownL1Message is returned if a block contains an L1 message that does not

diff --git a/core/bench_test.go b/core/bench_test.go
@@ -201,7 +201,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 
 	// Time the insertion of the new chain.
 	// State and blocks are stored in the same DB.
-	chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
+	chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil, false)
 	defer chainman.Stop()
 	b.ReportAllocs()
 	b.ResetTimer()
@@ -316,7 +316,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
-		chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
+		chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil, false)
 		if err != nil {
 			b.Fatalf("error creating chain: %v", err)
 		}