Merge branch 'master' into aaronmgdr/build4

* master: (61 commits)
  [Wallet] Handle `/v/<code` deep links for phone verifications (#1776)
  Patch tslint auto fix for macOS Catalina until tslint#6.x is out (#1802)
  Fix typos (#1855)
  Update documentation sidebar (#1861)
  [Wallet] Update forno dev documentation (#1818)
  [Docs] Correct typo
  Validator UX docs for baklava network (#1688) (#1849)
  Various improvements to the CLI, allow voters to revote for a group (#1840)
  Serialize to hex string (#1848)
  Update faucet to pull from the reserve when possible (#1844)
  Fix elect validators migration, deploy integration (#1847)
  Configurable genesis balances (#1838)
  Deploy new version of Celostats and minnor change from celo-blockchain (#1714)
  Make default node URL consistent in celocli (#1805)
  Onboarding feedback (#1811)
  return at least true, when the function is returning bool (#1825)
  Minor Contractkit changes (#1819)
  Point end-to-end tests back to master (#1824)
  Update migration config to correct protocol parameters (#1822)
  Update with new istanbul lookback window size flag (#1820)
  ...

# Conflicts:
#	yarn.lock

.circleci/config.yml

@@ -35,6 +35,8 @@ e2e-defaults: &e2e-defaults
   <<: *defaults
   docker:
     - image: celohq/circleci
+  environment:
+    CELO_BLOCKCHAIN_BRANCH_TO_TEST: master
 
 general:
   artifacts:
@@ -44,6 +46,8 @@ general:
 jobs:
   install_dependencies:
     <<: *defaults
+    # Source: https://circleci.com/docs/2.0/configuration-reference/#resource_class
+    resource_class: medium+
     steps:
       - restore_cache:
           keys:
@@ -251,7 +255,7 @@ jobs:
           name: Run test itself
 
           command: |
-            cd ~/src/packages/mobile 
+            cd ~/src/packages/mobile
             # detox sometimes without releasing the terminal and thus making the CI timout
             # 480s = 8 minutes
             timeout 480 yarn test:detox || echo "failed, try again"
@@ -398,7 +402,7 @@ jobs:
       - run:
           name: Generate DevChain
           command: |
-            (cd packages/contractkit && yarn test:prepare)
+            (cd packages/contractkit && yarn test:reset)
       - run:
           name: Run Tests
           command: yarn --cwd=packages/contractkit test
@@ -525,7 +529,7 @@ jobs:
           command: |
             set -e
             cd packages/celotool
-            ./ci_test_transfers.sh checkout master
+            ./ci_test_transfers.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST}
 
   end-to-end-geth-blockchain-parameters-test:
     <<: *e2e-defaults
@@ -543,7 +547,7 @@ jobs:
           command: |
             set -e
             cd packages/celotool
-            ./ci_test_blockchain_parameters.sh checkout master
+            ./ci_test_blockchain_parameters.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST}
 
   end-to-end-geth-governance-test:
     <<: *e2e-defaults
@@ -563,7 +567,7 @@ jobs:
           command: |
             set -e
             cd packages/celotool
-            ./ci_test_governance.sh checkout master
+            ./ci_test_governance.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST}
 
   end-to-end-geth-sync-test:
     <<: *e2e-defaults
@@ -582,7 +586,7 @@ jobs:
           command: |
             set -e
             cd packages/celotool
-            ./ci_test_sync.sh checkout master
+            ./ci_test_sync.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST}
 
   end-to-end-geth-validator-order-test:
     <<: *e2e-defaults
@@ -600,7 +604,7 @@ jobs:
           command: |
             set -e
             cd packages/celotool
-            ./ci_test_validator_order.sh checkout master
+            ./ci_test_validator_order.sh checkout ${CELO_BLOCKCHAIN_BRANCH_TO_TEST}
 
   web:
     working_directory: ~/app

.env

@@ -12,31 +12,35 @@ CLUSTER_DOMAIN_NAME="celo-networks-dev"
 TESTNET_PROJECT_NAME="celo-testnet"
 
 BLOCKSCOUT_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/blockscout"
-BLOCKSCOUT_DOCKER_IMAGE_TAG="5fba4843b3e78b5ab75d01766214cb24c6a40649"
+BLOCKSCOUT_DOCKER_IMAGE_TAG="909682b7435fc3e05849211d96fb1dfbf76306f2"
 BLOCKSCOUT_WEB_REPLICAS=3
 BLOCKSCOUT_DB_SUFFIX=
 
 ETHSTATS_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/ethstats"
-ETHSTATS_DOCKER_IMAGE_TAG="0ffe524c625ea59e4492dc92c2e638689c36e4b0"
+ETHSTATS_DOCKER_IMAGE_TAG="cd037ea1e18848466452ba9890c1f1bcd3f61009"
+ETHSTATS_TRUSTED_ADDRESSES=""
+ETHSTATS_BANNED_ADDRESSES=""
+
+FAUCET_GENESIS_ACCOUNTS=2
 
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/geth"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`
-GETH_NODE_DOCKER_IMAGE_TAG="7ccc0a81920e6e09663855fc0ed46a98d634e74a"
+GETH_NODE_DOCKER_IMAGE_TAG="master"
 
 GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/geth-all"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`
-GETH_BOOTNODE_DOCKER_IMAGE_TAG="7ccc0a81920e6e09663855fc0ed46a98d634e74a"
+GETH_BOOTNODE_DOCKER_IMAGE_TAG="master"
 
 CELOTOOL_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-CELOTOOL_DOCKER_IMAGE_TAG="celotool-dfdc3e8b26e98aa294b27e2b5621c184488a10db"
+CELOTOOL_DOCKER_IMAGE_TAG="celotool-c8e3392aa2ca44ff83b4035700ece5fd12ed2b84"
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-d3d165a7db548d175cd703c86c20c1657c04368d"
 
-ATTESTATION_SERVICE_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-ATTESTATION_SERVICE_DOCKER_IMAGE_TAG="attestation-service-74f329b014c40c7af19cf89b4c0d080c344d4a1c"
+ATTESTATION_SERVICE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/celo-monorepo"
+ATTESTATION_SERVICE_DOCKER_IMAGE_TAG="attestation-service-c8e3392aa2ca44ff83b4035700ece5fd12ed2b84"
 
 GETH_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet-production/geth-exporter"
 GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
@@ -46,6 +50,7 @@ NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
 BLOCK_TIME=1
 EPOCH=1000
+LOOKBACK=12
 ISTANBUL_REQUEST_TIMEOUT_MS=3000
 
 # "og" -> our original 4 validators, "${n}" -> for deriving n validators from the MNEMONIC

.env.alfajores

@@ -17,6 +17,8 @@ BLOCKSCOUT_WEB_REPLICAS=3
 BLOCKSCOUT_DB_SUFFIX="7"
 BLOCKSCOUT_SUBNETWORK_NAME="Alfajores"
 
+FAUCET_GENESIS_ACCOUNTS=2
+
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/geth"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`

.env.integration

@@ -11,29 +11,36 @@ CLUSTER_DOMAIN_NAME="celo-testnet"
 TESTNET_PROJECT_NAME="celo-testnet"
 
 BLOCKSCOUT_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/blockscout"
-BLOCKSCOUT_DOCKER_IMAGE_TAG="ad86714d629c01272e0651dec1fb6a968c3cec71"
+BLOCKSCOUT_DOCKER_IMAGE_TAG="909682b7435fc3e05849211d96fb1dfbf76306f2"
 BLOCKSCOUT_WEB_REPLICAS=3
-BLOCKSCOUT_DB_SUFFIX="25"
+BLOCKSCOUT_DB_SUFFIX="29"
 BLOCKSCOUT_SUBNETWORK_NAME="Integration"
 
+ETHSTATS_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/ethstats"
+ETHSTATS_DOCKER_IMAGE_TAG="cd037ea1e18848466452ba9890c1f1bcd3f61009"
+ETHSTATS_TRUSTED_ADDRESSES=""
+ETHSTATS_BANNED_ADDRESSES=""
+
+FAUCET_GENESIS_ACCOUNTS=2
+
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/geth"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`
-GETH_NODE_DOCKER_IMAGE_TAG="c1ae452c707f8bee91a9a0bf49193e78e9c8512e"
+GETH_NODE_DOCKER_IMAGE_TAG="09a217ff58a95214cbc5189c933359707f4fdaf2"
 
 GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/geth-all"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`
-GETH_BOOTNODE_DOCKER_IMAGE_TAG="c1ae452c707f8bee91a9a0bf49193e78e9c8512e"
+GETH_BOOTNODE_DOCKER_IMAGE_TAG="09a217ff58a95214cbc5189c933359707f4fdaf2"
 
 CELOTOOL_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-CELOTOOL_DOCKER_IMAGE_TAG="celotool-2cb725c36b69e7ae608875610af080f4f3fa79bd"
+CELOTOOL_DOCKER_IMAGE_TAG="celotool-5bea6d30cbe6aa4272b32a4d2cfed5567f422ea9"
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-2cb725c36b69e7ae608875610af080f4f3fa79bd"
+TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-5bea6d30cbe6aa4272b32a4d2cfed5567f422ea9"
 
-ATTESTATION_SERVICE_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-ATTESTATION_SERVICE_DOCKER_IMAGE_TAG="attestation-service-2cb725c36b69e7ae608875610af080f4f3fa79bd"
+ATTESTATION_SERVICE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/celo-monorepo"
+ATTESTATION_SERVICE_DOCKER_IMAGE_TAG="attestation-service-c8e3392aa2ca44ff83b4035700ece5fd12ed2b84"
 
 GETH_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet-production/geth-exporter"
 GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
@@ -45,7 +52,6 @@ BLOCK_TIME=3
 EPOCH=1000
 ISTANBUL_REQUEST_TIMEOUT_MS=3000
 
-# "og" -> our original 4 tx nodes, "${n}" -> for deriving n tx nodes from the MNEMONIC
 # NOTE: we only create static IPs when TX_NODES is set to "og"
 VALIDATORS=20
 TX_NODES=2
@@ -59,7 +65,6 @@ GETH_NODES_BACKUP_CRONJOB_ENABLED=true
 CONTRACT_CRONJOBS_ENABLED=true
 CLUSTER_CREATION_FLAGS="--enable-autoscaling --min-nodes 3 --max-nodes 8 --machine-type=n1-standard-4"
 
-
 GETH_NODE_CPU_REQUEST=400m
 GETH_NODE_MEMORY_REQUEST=2.5G
 

.gitignore

@@ -80,4 +80,7 @@ lerna-debug.log
 !.env.mnemonic*.enc
 
 .terraform/
-tsconfig.tsbuildinfo
+tsconfig.tsbuildinfo
+
+# git mergetool
+*.orig

SETUP.md

@@ -1,6 +1,7 @@
 # Celo Engineering Setup
 
 - [Celo Engineering Setup](#celo-engineering-setup)
+  - [Reading](#reading)
   - [Getting Everything Installed](#getting-everything-installed)
     - [MacOS](#macos)
       - [Xcode](#xcode)
@@ -26,6 +27,10 @@
 
 This is a living document! Please edit and update it as part of your onboarding process :-)
 
+## Reading
+
+Review the README from each directory in [packages](packages/).
+
 ## Getting Everything Installed
 
 Follow these steps to get everything that you need installed to develop and

dockerfiles/attestation-service/Dockerfile

@@ -8,6 +8,7 @@ COPY scripts/ scripts/
 # Copy only pkg.json
 COPY packages/typescript/package.json packages/typescript/
 COPY packages/utils/package.json packages/utils/
+COPY packages/dev-utils/package.json packages/dev-utils/
 COPY packages/protocol/package.json packages/protocol/
 COPY packages/contractkit/package.json packages/contractkit/
 COPY packages/attestation-service/package.json packages/attestation-service/
@@ -17,6 +18,7 @@ RUN yarn install --frozen-lockfile --network-timeout 100000 && yarn cache clean
 # Copy the rest
 COPY packages/typescript packages/typescript/
 COPY packages/utils packages/utils/
+COPY packages/dev-utils packages/dev-utils/
 COPY packages/protocol packages/protocol/
 COPY packages/contractkit packages/contractkit/
 COPY packages/attestation-service packages/attestation-service/

dockerfiles/celotool/Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update && \
 COPY lerna.json package.json yarn.lock ./
 COPY scripts/ scripts/
 COPY packages/utils/package.json packages/utils/
+COPY packages/dev-utils/package.json packages/dev-utils/
 COPY packages/typescript/package.json packages/typescript/
 COPY packages/walletkit/package.json packages/walletkit/
 COPY packages/contractkit/package.json packages/contractkit/
@@ -26,6 +27,7 @@ COPY packages/celotool/package.json packages/celotool/
 RUN yarn install --network-timeout 100000 --frozen-lockfile && yarn cache clean
 
 COPY packages/utils packages/utils/
+COPY packages/dev-utils packages/dev-utils/
 COPY packages/typescript packages/typescript/
 COPY packages/walletkit packages/walletkit/
 COPY packages/contractkit packages/contractkit/

packages/attestation-service/README.md

@@ -0,0 +1,45 @@
+# Attestation Service
+
+A service run by validators on the Celo network to send SMS messages, enabling attestations of user phone numbers and their accounts on the Celo network.
+
+### Configuration
+
+You can use the following environment variables to configure the attestation service:
+
+- `DATABASE_URL` - The URL under which your database is accessible, currently supported are `postgres://`, `mysql://` and `sqlite://`
+- `CELO_PROVIDER` - The URL under which a celo blockchain node is reachable, i.e. something like `https://integration-forno.celo-testnet.org`
+- `ACCOUNT_ADDRESS` - The address of the account on the `Accounts` smart contract
+- `ATTESTATION_KEY` - The private key with which attestations should be signed. You could use your account key for attestations, but really you should authorize a dedicated attestation key
+- `APP_SIGNATURE` - The hash with which clients can auto-read SMS messages on android
+- `SMS_PROVIDERS` - A comma-separated list of providers you want to configure, we currently support:
+
+`nexmo`
+
+- `NEXMO_KEY` - The API key to the Nexmo API
+- `NEXMO_SECRET` - The API secret to the Nexmo API
+- `NEXMO_BLACKLIST` - A comma-sperated list of country codes you do not want to serve
+
+`twilio`
+
+- `TWILIO_ACCOUNT_SID` - The SID of your Twilio account
+- `TWILIO_MESSAGE_SERVICE_SID` - The SID of the messaging service you want to use. The messaging service should have at least 1 phone number associated with it.
+- `TWILIO_AUTH_TOKEN` - The auth token for your Twilio account
+
+### Operations
+
+This service uses `bunyan` for structured logging with JSON lines. You can pipe STDOUT to `yarn run bunyan` for a more human friendly output. The `LOG_LEVEL` environment variable can specify desired log levels. With `LOG_FORMAT=stackdriver` you can get stackdriver specific format to recover information such as error traces etc.
+
+This service exposes prometheus metrics under `/metrics`.
+
+### Running locally
+
+After checking out the source, you should create a local sqlite database by running:
+
+```sh
+yarn run db:create:dev
+yarn run db:migrate:dev
+```
+
+You will also have to set the environment variables in `.env.development`
+
+Then start the service with `yarn run dev` (you'll have to add the appropriate credentials for the text providers)

packages/attestation-service/config/.env.development

@@ -3,5 +3,15 @@ CELO_PROVIDER=https://integration-forno.celo-testnet.org
 ACCOUNT_ADDRESS=0xE6e53b5fc2e18F51781f14a3ce5E7FD468247a15
 ATTESTATION_KEY=x
 APP_SIGNATURE=x
+SMS_PROVIDERS=twilio,nexmo
 NEXMO_KEY=x
-NEXMO_SECRET=x
+NEXMO_SECRET=x
+NEXMO_BLACKLIST=
+TWILIO_ACCOUNT_SID=x
+TWILIO_MESSAGING_SERVICE_SID=x
+TWILIO_AUTH_TOKEN=x
+TWILIO_BLACKLIST=
+# Options: default, stackdriver
+LOG_FORMAT=
+# Options: fatal, error, warn, info, debug, trace
+LOG_LEVEL=

packages/attestation-service/index.d.ts

@@ -1 +1,2 @@
 declare module 'nexmo'
+declare module 'express-request-id'

packages/attestation-service/migrations/20191015211858-create-attestation.js

@@ -23,6 +23,14 @@ module.exports = {
           allowNull: false,
           type: Sequelize.STRING,
         },
+        status: {
+          allowNull: false,
+          type: Sequelize.STRING,
+        },
+        smsProvider: {
+          allowNull: false,
+          type: Sequelize.STRING,
+        },
         createdAt: {
           allowNull: false,
           type: Sequelize.DATE,

packages/attestation-service/nodemon.json

@@ -1,6 +1,6 @@
 {
   "ignore": ["**/*.test.ts", "**/*.spec.ts", ".git", "node_modules"],
   "watch": ["src"],
-  "exec": "yarn start",
+  "exec": "yarn start-ts",
   "ext": "ts"
 }

packages/attestation-service/package.json

@@ -31,6 +31,8 @@
     "@celo/utils": "^0.1.0",
     "bignumber.js": "^7.2.0",
     "body-parser": "1.19.0",
+    "bunyan": "1.8.12",
+    "bunyan-gke-stackdriver": "0.1.2",
     "debug": "^4.1.1",
     "dotenv": "8.0.0",
     "eth-lib": "^0.2.8",
@@ -39,18 +41,24 @@
     "nexmo": "2.4.2",
     "web3": "1.0.0-beta.37",
     "express": "4.17.1",
+    "express-rate-limit": "5.0.0",
+    "express-request-id": "1.4.1",
     "mysql2": "2.0.0-alpha1",
     "pg": "7.12.1",
     "pg-hstore": "2.3.3",
+    "prom-client": "11.2.0",
     "sequelize": "5.13.1",
     "sequelize-cli": "5.5.0",
     "sqlite3": "4.0.9",
+    "twilio": "^3.23.2",
     "yargs": "13.3.0"
   },
   "devDependencies": {
     "@celo/protocol": "1.0.0",
+    "@types/bunyan": "1.8.4",
     "@types/dotenv": "4.0.3",
     "@types/debug": "^4.1.5",
+    "@types/express-rate-limit": "2.9.3",
     "@types/web3": "^1.0.18",
     "ts-node": "8.3.0",
     "nodemon": "1.19.1",