feat: bump dependencies to latest master (#35)

* ci: disable yarn install cache

* chore: use yarn test:ci to only test changes since master

* ci: use separate job for linting contracts

* feat(contracts): update to ethereum-optimism/contracts@06cdfb4

* feat(hardhat-ovm): replace env var with OVM config network

ethereum-optimism/plugins#35

* feat(batch-submitter): update to ethereum-optimism/batch-submitter@8cd92d4

* feat(dtl): update to https://github.com/ethereum-optimism/data-transport-layer/commit/b1e340a32bd6986ba2c381a10229d49c0b274648

More unit tests and slight type refactoring

* chore: regenerate yarn lock

* fix(integration-tests): use `--network optimism` instead of env var

* chore: yarn lint

.github/workflows/integration.yml

@@ -41,8 +41,7 @@ jobs:
 
       - name: Install integration test dependencies
         working-directory: ./integration-tests
-        # only install dependencies if the cache was invalidated
-        if: steps.yarn-cache.outputs.cache-hit != 'true'
+        # if: steps.yarn-cache.outputs.cache-hit != 'true'
         run: yarn install
 
       - name: Build deps for the integration tests

.github/workflows/ts-packages.yml

@@ -15,8 +15,8 @@ on:
         - 'packages/**/*.ts'
 
 jobs:
-  build-test-lint:
-    name: Run job on ${{matrix.node}}
+  test:
+    name: Run tests on ${{matrix.node}}
     runs-on: ubuntu-latest
 
     strategy:
@@ -48,14 +48,42 @@ jobs:
 
       - name: Install Dependencies
         # only install dependencies if there was a change in the deps
-        if: steps.yarn-cache.outputs.cache-hit != 'true'
+        # if: steps.yarn-cache.outputs.cache-hit != 'true'
         run: yarn install
 
       - name: Build
         run: yarn build
 
-      - name: Lint
-        run: yarn lint
-
       - name: Test
-        run: yarn test
+        run: yarn test:ci
+
+  lint:
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Fetch history
+        run: git fetch
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        # only install dependencies if there was a change in the deps
+        # if: steps.yarn-cache.outputs.cache-hit != 'true'
+        run: yarn install
+
+      - name: Lint
+        run: yarn lint:ci

integration-tests/hardhat.config.ts

@@ -8,6 +8,12 @@ const config: HardhatUserConfig = {
   mocha: {
     timeout: 100000,
   },
+  networks: {
+    optimism: {
+      url: 'http://localhost:8545',
+      ovm: true,
+    },
+  },
 }
 
 export default config

integration-tests/package.json

@@ -8,7 +8,7 @@
     "lint": "yarn lint:fix && yarn lint:check",
     "lint:check": "tslint --format stylish --project .",
     "lint:fix": "prettier --config ./prettier-config.json --write 'test/**/*.ts'",
-    "test:integration": "TARGET=ovm hardhat test"
+    "test:integration": "hardhat --network optimism test"
   },
   "devDependencies": {
     "@eth-optimism/contracts": "^0.1.11",

ops/envs/batches.env

@@ -2,9 +2,9 @@ ADDRESS_MANAGER_ADDRESS=
 
 DEBUG=info*,error*,warn*,debug*
 
-MAX_TX_SIZE=90000
-MIN_TX_SIZE=0
-MAX_BATCH_SIZE=50
+MAX_L1_TX_SIZE=90000
+MIN_L1_TX_SIZE=0
+MAX_TX_BATCH_COUNT=50
 
 POLL_INTERVAL=15000
 NUM_CONFIRMATIONS=0

package.json

@@ -13,8 +13,10 @@
   "scripts": {
       "clean": "yarn lerna run clean",
       "build": "yarn lerna run build",
-      "test": "yarn lerna run test --parallel --since origin/master",
-      "lint": "yarn lerna run lint  --parallel --since origin/master",
+      "test": "yarn lerna run test --parallel",
+      "test:ci": "yarn lerna run test --parallel --since origin/master",
+      "lint": "yarn lerna run lint",
+      "lint:ci": "yarn lerna run lint  --parallel --since origin/master",
       "lint:fix": "yarn lerna run lint:fix"
   }
 }

packages/batch-submitter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eth-optimism/batch-submitter",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "[Optimism] Batch submission for sequencer & aggregators",
   "main": "dist/index",
   "types": "dist/index",
@@ -10,7 +10,7 @@
   "scripts": {
     "start": "node ./exec/run-batch-submitter.js",
     "build": "tsc -p ./tsconfig.build.json",
-    "clean": "rimraf dist/ ./tsconfig.build.tsbuildinfo",
+    "clean": "rimraf cache/ dist/ ./tsconfig.build.tsbuildinfo",
     "lint": "yarn lint:fix && yarn lint:check",
     "lint:check": "tslint --format stylish --project .",
     "lint:fix": "prettier --config prettier-config.json --write \"hardhat.config.ts\" \"{src,exec,test}/**/*.ts\"",

packages/batch-submitter/src/batch-submitter/batch-submitter.ts

@@ -131,22 +131,35 @@ export abstract class BatchSubmitter {
   }
 
   protected _shouldSubmitBatch(batchSizeInBytes: number): boolean {
+    const currentTimestamp = Date.now()
     const isTimeoutReached =
       this.lastBatchSubmissionTimestamp + this.maxBatchSubmissionTime <=
-      Date.now()
+      currentTimestamp
     if (batchSizeInBytes < this.minTxSize) {
       if (!isTimeoutReached) {
         this.log.info(
           'Skipping batch submission. Batch too small & max submission timeout not reached.',
           {
             batchSizeInBytes,
             minTxSize: this.minTxSize,
+            lastBatchSubmissionTimestamp: this.lastBatchSubmissionTimestamp,
+            currentTimestamp,
           }
         )
         return false
       }
-      this.log.info('Timeout reached.')
+      this.log.info('Timeout reached, proceeding with batch submission.', {
+        batchSizeInBytes,
+        lastBatchSubmissionTimestamp: this.lastBatchSubmissionTimestamp,
+        currentTimestamp,
+      })
+      return true
     }
+    this.log.info('Sufficient batch size, proceeding with batch submission.', {
+      batchSizeInBytes,
+      lastBatchSubmissionTimestamp: this.lastBatchSubmissionTimestamp,
+      currentTimestamp,
+    })
     return true
   }
 

packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts

@@ -1,4 +1,5 @@
 /* External Imports */
+import { Promise as bPromise } from 'bluebird'
 import { Contract, Signer } from 'ethers'
 import { TransactionReceipt } from '@ethersproject/abstract-provider'
 import { getContractFactory } from '@eth-optimism/contracts'
@@ -108,14 +109,13 @@ export class StateBatchSubmitter extends BatchSubmitter {
     const startBlock: number =
       (await this.chainContract.getTotalElements()).toNumber() + BLOCK_OFFSET
     // We will submit state roots for txs which have been in the tx chain for a while.
-    const callBlockNumber: number =
-      (await this.signer.provider.getBlockNumber()) - this.finalityConfirmations
     const totalElements: number =
       (await this.ctcContract.getTotalElements()).toNumber() + BLOCK_OFFSET
     const endBlock: number = Math.min(
       startBlock + this.maxBatchSize,
       totalElements
     )
+
     if (startBlock >= endBlock) {
       if (startBlock > endBlock) {
         this.log.error(
@@ -142,7 +142,7 @@ export class StateBatchSubmitter extends BatchSubmitter {
       'appendStateBatch',
       [batch, startBlock]
     )
-    if (!this._shouldSubmitBatch(tx.length * 2)) {
+    if (!this._shouldSubmitBatch(tx.length / 2)) {
       return
     }
 
@@ -172,21 +172,22 @@ export class StateBatchSubmitter extends BatchSubmitter {
     startBlock: number,
     endBlock: number
   ): Promise<Bytes32[]> {
-    const batch: Bytes32[] = []
-
-    for (let i = startBlock; i < endBlock; i++) {
-      const block = (await this.l2Provider.getBlockWithTransactions(
-        i
-      )) as L2Block
-      if (block.transactions[0].from === this.fraudSubmissionAddress) {
-        batch.push(
-          '0xbad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1'
-        )
-        this.fraudSubmissionAddress = 'no fraud'
-      } else {
-        batch.push(block.stateRoot)
-      }
-    }
+    const blockRange = endBlock - startBlock
+    const batch: Bytes32[] = await bPromise.map(
+      [...Array(blockRange).keys()],
+      async (i: number) => {
+        this.log.debug('Fetching L2BatchElement', { blockNo: startBlock + i })
+        const block = (await this.l2Provider.getBlockWithTransactions(
+          startBlock + i
+        )) as L2Block
+        if (block.transactions[0].from === this.fraudSubmissionAddress) {
+          this.fraudSubmissionAddress = 'no fraud'
+          return '0xbad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1bad1'
+        }
+        return block.stateRoot
+      },
+      { concurrency: 100 }
+    )
 
     let tx = this.chainContract.interface.encodeFunctionData(
       'appendStateBatch',
@@ -199,6 +200,7 @@ export class StateBatchSubmitter extends BatchSubmitter {
         startBlock,
       ])
     }
+
     return batch
   }
 }

packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts

@@ -237,7 +237,12 @@ export class TransactionBatchSubmitter extends BatchSubmitter {
       batchParams,
       wasBatchTruncated,
     ] = await this._generateSequencerBatchParams(startBlock, endBlock)
-    const batchSizeInBytes = encodeAppendSequencerBatch(batchParams).length * 2
+    const batchSizeInBytes = encodeAppendSequencerBatch(batchParams).length / 2
+
+    // Only submit batch if one of the following is true:
+    // 1. it was truncated
+    // 2. it is large enough
+    // 3. enough time has passed since last submission
     if (!wasBatchTruncated && !this._shouldSubmitBatch(batchSizeInBytes)) {
       return
     }
@@ -278,7 +283,7 @@ export class TransactionBatchSubmitter extends BatchSubmitter {
         this.log.debug('Fetching L2BatchElement', { blockNo: startBlock + i })
         return this._getL2BatchElement(startBlock + i)
       },
-      { concurrency: 50 }
+      { concurrency: 100 }
     )
 
     // Fix our batches if we are configured to. TODO: Remove this.

packages/batch-submitter/src/exec/run-batch-submitter.ts

@@ -32,11 +32,13 @@ interface RequiredEnvVars {
   // The layer one address manager address
   ADDRESS_MANAGER_ADDRESS: 'ADDRESS_MANAGER_ADDRESS'
   // The minimum size in bytes of any L1 transactions generated by the batch submitter.
-  MIN_TX_SIZE: 'MIN_TX_SIZE'
+  MIN_L1_TX_SIZE: 'MIN_L1_TX_SIZE'
   // The maximum size in bytes of any L1 transactions generated by the batch submitter.
-  MAX_TX_SIZE: 'MAX_TX_SIZE'
+  MAX_L1_TX_SIZE: 'MAX_L1_TX_SIZE'
   // The maximum number of L2 transactions that can ever be in a batch.
-  MAX_BATCH_SIZE: 'MAX_BATCH_SIZE'
+  MAX_TX_BATCH_COUNT: 'MAX_TX_BATCH_COUNT'
+  // The maximum number of L2 state roots that can ever be in a batch.
+  MAX_STATE_BATCH_COUNT: 'MAX_STATE_BATCH_COUNT'
   // The maximum amount of time (seconds) that we will wait before submitting an under-sized batch.
   MAX_BATCH_SUBMISSION_TIME: 'MAX_BATCH_SUBMISSION_TIME'
   // The delay in milliseconds between querying L2 for more transactions / to create a new batch.
@@ -62,9 +64,10 @@ const requiredEnvVars: RequiredEnvVars = {
   L1_NODE_WEB3_URL: 'L1_NODE_WEB3_URL',
   L2_NODE_WEB3_URL: 'L2_NODE_WEB3_URL',
   ADDRESS_MANAGER_ADDRESS: 'ADDRESS_MANAGER_ADDRESS',
-  MIN_TX_SIZE: 'MIN_TX_SIZE',
-  MAX_TX_SIZE: 'MAX_TX_SIZE',
-  MAX_BATCH_SIZE: 'MAX_BATCH_SIZE',
+  MIN_L1_TX_SIZE: 'MIN_L1_TX_SIZE',
+  MAX_L1_TX_SIZE: 'MAX_L1_TX_SIZE',
+  MAX_TX_BATCH_COUNT: 'MAX_TX_BATCH_COUNT',
+  MAX_STATE_BATCH_COUNT: 'MAX_STATE_BATCH_COUNT',
   MAX_BATCH_SUBMISSION_TIME: 'MAX_BATCH_SUBMISSION_TIME',
   POLL_INTERVAL: 'POLL_INTERVAL',
   NUM_CONFIRMATIONS: 'NUM_CONFIRMATIONS',
@@ -144,9 +147,9 @@ export const run = async () => {
   const txBatchSubmitter = new TransactionBatchSubmitter(
     sequencerSigner,
     l2Provider,
-    parseInt(requiredEnvVars.MIN_TX_SIZE, 10),
-    parseInt(requiredEnvVars.MAX_TX_SIZE, 10),
-    parseInt(requiredEnvVars.MAX_BATCH_SIZE, 10),
+    parseInt(requiredEnvVars.MIN_L1_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_L1_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_TX_BATCH_COUNT, 10),
     parseInt(requiredEnvVars.MAX_BATCH_SUBMISSION_TIME, 10) * 1_000,
     parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10),
     parseInt(requiredEnvVars.RESUBMISSION_TIMEOUT, 10) * 1_000,
@@ -164,9 +167,9 @@ export const run = async () => {
   const stateBatchSubmitter = new StateBatchSubmitter(
     sequencerSigner,
     l2Provider,
-    parseInt(requiredEnvVars.MIN_TX_SIZE, 10),
-    parseInt(requiredEnvVars.MAX_TX_SIZE, 10),
-    parseInt(requiredEnvVars.MAX_BATCH_SIZE, 10),
+    parseInt(requiredEnvVars.MIN_L1_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_L1_TX_SIZE, 10),
+    parseInt(requiredEnvVars.MAX_STATE_BATCH_COUNT, 10),
     parseInt(requiredEnvVars.MAX_BATCH_SUBMISSION_TIME, 10) * 1_000,
     parseInt(requiredEnvVars.NUM_CONFIRMATIONS, 10),
     parseInt(requiredEnvVars.RESUBMISSION_TIMEOUT, 10) * 1_000,