Add updated batch submitter (#334)

* Added state dump utilities to contracts package

* Linted files

* Have dump replace addresses

* Cleaned up state dump imports

* Linted files

* Added potential fix for ganache

* use published js-vm

* small changes for jsvm and ganache

* Remove outdated dockerfiles

* update dependency

* Remove old contracts & packages

* Add engine (node 10) to pkg.json

* Remove unused actions

* Fix lint error

* Remove node 11 in CI

* Add initial batch-submitter structure

* Add batch encoder

* Fix batch encoder

* Rename toBytes->toVerifiedBytes

* Add buidler & helpers

TODO: move the helpers into a standalone package. This shouldn't be
duplicated.

* Add initial batch submitter

* First pass batch submitter

* Use r s v not v r s

* Begin adding mockchain provider

* Fix imports & errors with cherry-pick

* Use git repo for contracts v2

* Increase gasLimit & gasPrice encoding size

* use new contracts package

* update yarn.lock

* Refactor coders & add Mockchain blocks

* Begin integrating batch submitter

* Add expected types to mock

* Add logger

* Clean up batch submitter config

* Add exec file for running batch submitter

* Add queue origin decoding

* Update to use optimism provider

* Fix linting errors

* Remove unused helpers

* Make confirmations configurable

* Fix small bugs & improve naming

* Make batch size configurable

* Remove more helpers

* Update txType & add EthSign encoding

* Re-introduce needed buidler constants

* Fix bugs, improve tests, & add retries

* Fix lint errors

* Remove Object.assign for env vars

Co-authored-by: Kelvin Fichter <[email protected]>
Co-authored-by: Kevin Ho <[email protected]>
Co-authored-by: B T <[email protected]>

packages/batch-submitter/README.md

@@ -0,0 +1,23 @@
+# Batch Submitter
+
+Contains an executable batch submitter service which watches L1 and a local L2 node and submits batches to the
+`CanonicalTransactionChain` & `StateCommitmentChain` based on its local information.
+
+## Configuration
+All configuration is done via environment variables.
+
+## Building & Running
+1. Make sure dependencies are installed just run `yarn` in the base directory
+2. Build `yarn build`
+3. Run `yarn start`
+
+## Controlling log output verbosity
+Before running, set the `DEBUG` environment variable to specify the verbosity level. It must be made up of comma-separated values of patterns to match in debug logs. Here's a few common options:
+* `debug*` - Will match all debug statements -- very verbose
+* `info*` - Will match all info statements -- less verbose, useful in most cases
+* `warn*` - Will match all warnings -- recommended at a minimum
+* `error*` - Will match all errors -- would not omit this
+
+Examples:
+* Everything but debug: `export DEBUG=info*,error*,warn*`
+* Most verbose: `export DEBUG=info*,error*,warn*,debug*`

packages/batch-submitter/buidler-env.d.ts

@@ -0,0 +1,2 @@
+/// <reference types="@nomiclabs/buidler-ethers/src/type-extensions" />
+/// <reference types="@nomiclabs/buidler-waffle/src/type-extensions" />

packages/batch-submitter/buidler.config.ts

@@ -0,0 +1,29 @@
+import { usePlugin, BuidlerConfig } from '@nomiclabs/buidler/config'
+
+import {
+  DEFAULT_ACCOUNTS_BUIDLER,
+  RUN_OVM_TEST_GAS,
+} from './test/helpers/constants'
+
+usePlugin('@nomiclabs/buidler-ethers')
+usePlugin('@nomiclabs/buidler-waffle')
+
+import '@eth-optimism/smock/build/src/buidler-plugins/compiler-storage-layout'
+
+const config: BuidlerConfig = {
+  networks: {
+    buidlerevm: {
+      accounts: DEFAULT_ACCOUNTS_BUIDLER,
+      blockGasLimit: RUN_OVM_TEST_GAS * 2,
+    },
+  },
+  mocha: {
+    timeout: 50000,
+  },
+  solc: {
+    version: '0.7.0',
+    optimizer: { enabled: true, runs: 200 },
+  },
+}
+
+export default config

packages/batch-submitter/exec/run-batch-submitter.js

@@ -0,0 +1,3 @@
+const batchSubmitter = require("../build/src/exec/run-batch-submitter")
+
+batchSubmitter.run()

packages/batch-submitter/index.ts

@@ -0,0 +1,3 @@
+const rootPath = __dirname
+
+export { rootPath }

packages/batch-submitter/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@eth-optimism/batch-submitter",
+  "version": "0.0.1-alpha.1",
+  "description": "[Optimism] Batch submission for sequencer & aggregators",
+  "main": "build/index.js",
+  "files": [
+    "build/**/*.js"
+  ],
+  "scripts": {
+    "start": "node ./exec/run-batch-submitter.js",
+    "build": "tsc -p .",
+    "clean": "rimraf build/",
+    "fix": "prettier --config ../../prettier-config.json --write 'index.ts' 'src/**/*.ts'",
+    "lint": "tslint --format stylish --project .",
+    "test": "buidler test --show-stack-traces"
+  },
+  "engines": {
+    "node": "10"
+  },
+  "keywords": [
+    "optimism",
+    "ethereum",
+    "sequencer",
+    "aggregator"
+  ],
+  "homepage": "https://github.com/ethereum-optimism/optimism-monorepo/tree/master/packages/batch-submitter#readme",
+  "license": "MIT",
+  "author": "Optimism",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ethereum-optimism/optimism-monorepo.git"
+  },
+  "dependencies": {
+    "@eth-optimism/contracts": "^0.0.2-alpha.2",
+    "@eth-optimism/core-utils": "^0.0.1-alpha.30",
+    "@eth-optimism/provider": "^0.0.1-alpha.6",
+    "@ethersproject/abstract-provider": "^5.0.5",
+    "@ethersproject/providers": "^5.0.14",
+    "@nomiclabs/buidler": "^1.4.4",
+    "@nomiclabs/buidler-ethers": "^2.0.0",
+    "@nomiclabs/buidler-waffle": "^2.0.0",
+    "chai": "^4.2.0",
+    "ethers": "5.0.0",
+    "rimraf": "^2.6.3"
+  },
+  "devDependencies": {
+    "@eth-optimism/smock": "^0.0.2",
+    "@types/mocha": "^5.2.7",
+    "@types/node": "^12.0.7",
+    "ethereum-waffle": "3.0.0",
+    "mocha": "^6.1.4",
+    "ts-node": "^8.2.0",
+    "typescript": "^3.5.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/batch-submitter/src/batch-submitter.ts

@@ -0,0 +1,234 @@
+/* External Imports */
+import { BigNumber, Signer } from 'ethers'
+import {
+  TransactionResponse,
+  TransactionReceipt,
+} from '@ethersproject/abstract-provider'
+import { getLogger } from '@eth-optimism/core-utils'
+import { OptimismProvider } from '@eth-optimism/provider'
+
+const log = getLogger('oe:batch-submitter:core')
+
+/* Internal Imports */
+import {
+  CanonicalTransactionChainContract,
+  encodeAppendSequencerBatch,
+  BatchContext,
+  AppendSequencerBatchParams,
+} from './transaciton-chain-contract'
+import {
+  EIP155TxData,
+  CreateEOATxData,
+  TxType,
+  ctcCoder,
+  EthSignTxData,
+} from './coders'
+import { L2Block, BatchElement, Batch, QueueOrigin } from '.'
+
+export class BatchSubmitter {
+  public blockCache: {
+    [blockNumber: number]: BatchElement
+  } = {}
+
+  constructor(
+    readonly txChain: CanonicalTransactionChainContract,
+    readonly signer: Signer,
+    readonly l2Provider: OptimismProvider,
+    readonly l2ChainId: number,
+    readonly maxTxSize: number,
+    readonly defaultBatchSize: number,
+    readonly numConfirmations: number
+  ) {}
+
+  public async submitNextBatch(): Promise<TransactionReceipt> {
+    const startBlock = parseInt(await this.txChain.getTotalElements(), 16) + 1
+    const endBlock = Math.min(
+      startBlock + this.defaultBatchSize,
+      await this.l2Provider.getBlockNumber()
+    )
+    log.info(
+      `Attempting to submit next batch. Start l2 tx index: ${startBlock} - end index: ${endBlock}`
+    )
+    if (startBlock === endBlock) {
+      log.info(`No txs to submit. Skipping...`)
+      return
+    }
+
+    const batchParams = await this._generateSequencerBatchParams(
+      startBlock,
+      endBlock
+    )
+    const txRes = await this.txChain.appendSequencerBatch(batchParams)
+    const receipt = await txRes.wait(this.numConfirmations)
+    log.info('Submitted batch!')
+    log.debug('Tx receipt:', receipt)
+    return receipt
+  }
+
+  public async _generateSequencerBatchParams(
+    startBlock: number,
+    endBlock: number
+  ): Promise<AppendSequencerBatchParams> {
+    // Get all L2Blocks between the given range
+    const blocks: Batch = []
+    for (let i = startBlock; i < endBlock; i++) {
+      if (!this.blockCache.hasOwnProperty(i)) {
+        this.blockCache[i] = await this._getL2BatchElement(i)
+      }
+      blocks.push(this.blockCache[i])
+    }
+    let sequencerBatchParams = await this._getSequencerBatchParams(
+      startBlock,
+      blocks
+    )
+    let encoded = encodeAppendSequencerBatch(sequencerBatchParams)
+    while (encoded.length / 2 > this.maxTxSize) {
+      blocks.splice(Math.ceil((blocks.length * 2) / 3)) // Delete 1/3rd of all of the blocks
+      sequencerBatchParams = await this._getSequencerBatchParams(
+        startBlock,
+        blocks
+      )
+      encoded = encodeAppendSequencerBatch(sequencerBatchParams)
+    }
+    return sequencerBatchParams
+  }
+
+  public async _getSequencerBatchParams(
+    shouldStartAtIndex: number,
+    blocks: Batch
+  ): Promise<AppendSequencerBatchParams> {
+    const totalElementsToAppend = blocks.length
+
+    // Generate contexts
+    const contexts: BatchContext[] = []
+    let lastBlockIsSequencerTx = false
+    const groupedBlocks: Array<{
+      sequenced: BatchElement[]
+      queued: BatchElement[]
+    }> = []
+    for (const block of blocks) {
+      if (
+        (lastBlockIsSequencerTx === false && block.isSequencerTx === true) ||
+        groupedBlocks.length === 0
+      ) {
+        groupedBlocks.push({
+          sequenced: [],
+          queued: [],
+        })
+      }
+      const cur = groupedBlocks.length - 1
+      block.isSequencerTx
+        ? groupedBlocks[cur].sequenced.push(block)
+        : groupedBlocks[cur].queued.push(block)
+      lastBlockIsSequencerTx = block.isSequencerTx
+    }
+    for (const groupedBlock of groupedBlocks) {
+      contexts.push({
+        numSequencedTransactions: groupedBlock.sequenced.length,
+        numSubsequentQueueTransactions: groupedBlock.queued.length,
+        timestamp:
+          groupedBlock.sequenced.length > 0
+            ? groupedBlock.sequenced[0].timestamp
+            : 0,
+        blockNumber:
+          groupedBlock.sequenced.length > 0
+            ? groupedBlock.sequenced[0].blockNumber
+            : 0,
+      })
+    }
+
+    // Generate sequencer transactions
+    const transactions: string[] = []
+    for (const block of blocks) {
+      if (!block.isSequencerTx) {
+        continue
+      }
+      let encoding: string
+      if (block.sequencerTxType === TxType.EIP155) {
+        encoding = ctcCoder.eip155TxData.encode(block.txData as EIP155TxData)
+      } else if (block.sequencerTxType === TxType.EthSign) {
+        encoding = ctcCoder.ethSignTxData.encode(block.txData as EthSignTxData)
+      } else if (block.sequencerTxType === TxType.createEOA) {
+        encoding = ctcCoder.createEOATxData.encode(
+          block.txData as CreateEOATxData
+        )
+      }
+      transactions.push(encoding)
+    }
+
+    return {
+      shouldStartAtBatch: shouldStartAtIndex - 1,
+      totalElementsToAppend,
+      contexts,
+      transactions,
+    }
+  }
+
+  public async _getL2BatchElement(blockNumber: number): Promise<BatchElement> {
+    const block = (await this.l2Provider.getBlockWithTransactions(
+      blockNumber
+    )) as L2Block
+    const txType = block.transactions[0].meta.txType
+
+    if (this._isSequencerTx(block)) {
+      if (txType === TxType.EIP155 || txType === TxType.EthSign) {
+        return this._getDefaultEcdsaTxBatchElement(block)
+      } else if (txType === TxType.createEOA) {
+        return this._getCreateEoaBatchElement(block)
+      } else {
+        throw new Error('Unsupported Tx Type!')
+      }
+    } else {
+      return {
+        stateRoot: block.stateRoot,
+        isSequencerTx: false,
+        sequencerTxType: undefined,
+        txData: undefined,
+        timestamp: block.timestamp,
+        blockNumber: block.transactions[0].meta.l1BlockNumber,
+      }
+    }
+  }
+
+  private _getDefaultEcdsaTxBatchElement(block: L2Block): BatchElement {
+    const tx: TransactionResponse = block.transactions[0]
+    const txData: EIP155TxData = {
+      sig: {
+        v: '0' + (tx.v - this.l2ChainId * 2 - 8 - 27).toString(),
+        r: tx.r,
+        s: tx.s,
+      },
+      gasLimit: BigNumber.from(tx.gasLimit).toNumber(),
+      gasPrice: BigNumber.from(tx.gasPrice).toNumber(),
+      nonce: tx.nonce,
+      target: tx.to ? tx.to : '00'.repeat(20),
+      data: tx.data,
+    }
+    return {
+      stateRoot: block.stateRoot,
+      isSequencerTx: true,
+      sequencerTxType: block.transactions[0].meta.txType,
+      txData,
+      timestamp: block.timestamp,
+      blockNumber: block.transactions[0].meta.l1BlockNumber,
+    }
+  }
+
+  private _getCreateEoaBatchElement(block: L2Block): BatchElement {
+    const txData: CreateEOATxData = ctcCoder.createEOATxData.decode(
+      block.transactions[0].data
+    )
+    return {
+      stateRoot: block.stateRoot,
+      isSequencerTx: true,
+      sequencerTxType: block.transactions[0].meta.txType,
+      txData,
+      timestamp: block.timestamp,
+      blockNumber: block.transactions[0].meta.l1BlockNumber,
+    }
+  }
+
+  public _isSequencerTx(block: L2Block): boolean {
+    return block.transactions[0].meta.queueOrigin === QueueOrigin.Sequencer
+  }
+}