Merge branch 'master' into aaronmgdr/biscuit

* master: (61 commits)
  Remove locales as website is now just in English (#1050)
  Add MetadataURL to account struct (#1103)
  Allow validators to use any valid combination of gold commitments as stake (#885)
  Fix blockscout websocket jsonrpc url (#1096)
  [Wallet] Preliminary iOS support (#1098)
  [Wallet] Set security fee description translation in Spanish (#1097)
  Exclude generated in vscode file watcher setting (#1082)
  Update .env and .env.integration files (#1087)
  Allow a testnet to run without ethstats (#1085)
  Collect exchange rate time series using notification service (#1020)
  Return to preview view when Fee Education is closed (#1068)
  [Wallet] Pin Setup Flow v2 (#1054)
  Added a variable for electoral threshold (#1023)
  [celotool]Store .env config on GCS after deployment (#1086)
  Group size limit (#1035)
  Fix governance unit tests (#1084)
  Add getExchangeRate to ContractKit (#1083)
  [CLI]Unlock till the geth exits (#1070)
  Add Quorum and Refactor Governance (#430)
  Shuffle elected validators using block randomness (#1033)
  ...

.circleci/config.yml

@@ -94,6 +94,7 @@ jobs:
           name: Build packages
           command: |
             # separate build to avoid ENOMEN in CI :(
+            yarn build --scope @celo/utils
             yarn build --scope @celo/protocol
             yarn build --scope docs
             yarn build --scope @celo/walletkit
@@ -194,6 +195,22 @@ jobs:
             JEST_JUNIT_OUTPUT: test-results/jest/junit.xml
 
   protocol-test:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/app
+      - run:
+          name: Check if the test should run
+          command: |
+            FILES_TO_CHECK="${PWD}/packages/protocol,${PWD}/packages/utils,${PWD}/.circleci/config.yml"
+            ./scripts/ci_check_if_test_should_run_v2.sh ${FILES_TO_CHECK}
+      - run:
+          name: test
+          # Flaky tests - run them twice
+          command: yarn --cwd packages/protocol test || yarn --cwd packages/protocol test
+
+  # Slow and flaky compared to protocol-test which runs without coverage
+  protocol-test-with-code-coverage:
     <<: *defaults
     steps:
       - attach_workspace:
@@ -528,7 +545,7 @@ workflows:
           requires:
             - lint-checks
             - walletkit-test
-  nightly:
+  npm-install-testing-cron-workflow:
     triggers:
       - schedule:
           # 7 PM in UTC = noon in PDT.
@@ -543,3 +560,25 @@ workflows:
       - test-utils-npm-package-install
       - test-walletkit-npm-package-install
       - test-celocli-npm-package-install
+  protocol-testing-with-code-coverage-cron-workflow:
+    triggers:
+      - schedule:
+          # 1 PM in UTC = 6 AM in PDT.
+          # Best for this slow test (~3 hours) to run during SF early morning.
+          cron: "0 13 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - install_dependencies
+      - lint-checks:
+          requires:
+            - install_dependencies
+      - walletkit-test:
+          requires:
+            - install_dependencies
+      - protocol-test-with-code-coverage:
+          requires:
+            - lint-checks
+            - walletkit-test

.env

@@ -11,36 +11,41 @@ CLUSTER_DOMAIN_NAME="celo-networks-dev"
 
 TESTNET_PROJECT_NAME="celo-testnet"
 
-BLOCKSCOUT_DOCKER_IMAGE_REPOSITORY= "gcr.io/celo-testnet/blockscout"
+BLOCKSCOUT_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/blockscout"
 BLOCKSCOUT_WEB_DOCKER_IMAGE_TAG="web-f6c3e0888d1d0ef72dc8bf870808702b7fd13730"
 BLOCKSCOUT_INDEXER_DOCKER_IMAGE_TAG="indexer-f6c3e0888d1d0ef72dc8bf870808702b7fd13730"
 BLOCKSCOUT_WEB_REPLICAS=3
 BLOCKSCOUT_DB_SUFFIX=
 
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/geth"
-GETH_NODE_DOCKER_IMAGE_TAG="f7095b78003062db9536e1d070772d20a3f81e93"
+# When upgrading change this to latest commit hash from the master of the geth repo
+# `geth $ git show | head -n 1`
+GETH_NODE_DOCKER_IMAGE_TAG="7fbd6f3574f1c1c1e657c152fc63fb771adab3af"
 
 GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/geth-all"
-GETH_BOOTNODE_DOCKER_IMAGE_TAG="f7095b78003062db9536e1d070772d20a3f81e93"
+# When upgrading change this to latest commit hash from the master of the geth repo
+# `geth $ git show | head -n 1`
+GETH_BOOTNODE_DOCKER_IMAGE_TAG="7fbd6f3574f1c1c1e657c152fc63fb771adab3af"
 
 CELOTOOL_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-CELOTOOL_DOCKER_IMAGE_TAG="celotool-552b1accf90404fdcd886670d150af0a5cae116f"
+CELOTOOL_DOCKER_IMAGE_TAG="celotool-dfdc3e8b26e98aa294b27e2b5621c184488a10db"
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-552b1accf90404fdcd886670d150af0a5cae116f"
+TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-d3d165a7db548d175cd703c86c20c1657c04368d"
 
 GETH_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet-production/geth-exporter"
 GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
 
 # Genesis Vars
 NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=1
-EPOCH=30000
+EPOCH=1000
 # "og" -> our original 4 validators, "${n}" -> for deriving n validators from the MNEMONIC
 VALIDATORS="3"
 STATIC_IPS_FOR_GETH_NODES=false
+# Whether tx_nodes/validators stateful set should use ssd persistent disks
+GETH_NODES_SSD_DISKS=true
 
 GETHTX1_NODE_ID="1182aa8c9dbb96cd1aa71b74e2b6b481085971e08b210bab3b64c39d54876d4b1370f3f2c3cc3c0f52806a0e5772aa3fe937b4ceda8b97c5bf647a34170555e4"
 GETHTX2_NODE_ID="b1d8deee4e5f4faf9b7e8e7fbd5e19545632d9023ff10de55e2e7a37464c52d1d6fb3ac8cb011757558b37309b83d915de19ef86eb27fe13209cc02d0098ee1f"

.env.alfajores

@@ -39,7 +39,6 @@ GETH_EXPORTER_DOCKER_IMAGE_TAG="6df683de7ae30d3fbca384abb14599d0e8130d35"
 # Genesis Vars
 NETWORK_ID=44782
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=5
 EPOCH=720   // 1 hour
 
@@ -48,6 +47,8 @@ EPOCH=720   // 1 hour
 VALIDATORS=70
 TX_NODES=10
 STATIC_IPS_FOR_GETH_NODES=true
+# Whether tx_nodes/validators stateful set should use ssd persistent disks
+GETH_NODES_SSD_DISKS=true
 
 ADMIN_RPC_ENABLED=false
 
@@ -58,7 +59,7 @@ CLUSTER_CREATION_FLAGS="--enable-autoscaling --min-nodes 3 --max-nodes 8 --machi
 
 
 GETH_NODE_CPU_REQUEST=400m
-GETH_NODE_MEMORY_REQUEST=2.5G
+GETH_NODE_MEMORY_REQUEST=4.5G
 
 VERIFICATION_POOL_URL="https://us-central1-celo-testnet-production.cloudfunctions.net/handleVerificationRequestalfajores/v0.1/sms/"
 VERIFICATION_REWARDS_URL="https://us-central1-celo-testnet-production.cloudfunctions.net/handleVerificationRequestalfajores/v0.1/rewards/"

.env.alfajoresstaging

@@ -43,7 +43,6 @@ GETH_EXPORTER_DOCKER_IMAGE_TAG="6df683de7ae30d3fbca384abb14599d0e8130d35"
 # Genesis Vars
 NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=5
 # Minimum epoch length is 1 day
 EPOCH=17280
@@ -53,6 +52,8 @@ EPOCH=17280
 VALIDATORS=70
 TX_NODES=10
 STATIC_IPS_FOR_GETH_NODES=true
+# Whether tx_nodes/validators stateful set should use ssd persistent disks
+GETH_NODES_SSD_DISKS=true
 
 ADMIN_RPC_ENABLED=false
 

.env.integration

@@ -20,15 +20,15 @@ BLOCKSCOUT_SUBNETWORK_NAME="Integration"
 GETH_NODE_DOCKER_IMAGE_REPOSITORY="us.gcr.io/celo-testnet/geth"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`
-GETH_NODE_DOCKER_IMAGE_TAG="288e163082235106e5473a075a544af4b3f55fb6"
+GETH_NODE_DOCKER_IMAGE_TAG="7fbd6f3574f1c1c1e657c152fc63fb771adab3af"
 
 GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/geth-all"
 # When upgrading change this to latest commit hash from the master of the geth repo
 # `geth $ git show | head -n 1`
-GETH_BOOTNODE_DOCKER_IMAGE_TAG="288e163082235106e5473a075a544af4b3f55fb6"
+GETH_BOOTNODE_DOCKER_IMAGE_TAG="7fbd6f3574f1c1c1e657c152fc63fb771adab3af"
 
 CELOTOOL_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
-CELOTOOL_DOCKER_IMAGE_TAG="celotool-74feb4952ea540279405e61e1243b63852b878c0"
+CELOTOOL_DOCKER_IMAGE_TAG="celotool-dfdc3e8b26e98aa294b27e2b5621c184488a10db"
 
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_REPOSITORY="gcr.io/celo-testnet/celo-monorepo"
 TRANSACTION_METRICS_EXPORTER_DOCKER_IMAGE_TAG="transaction-metrics-exporter-d3d165a7db548d175cd703c86c20c1657c04368d"
@@ -39,7 +39,6 @@ GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
 # Genesis Vars
 NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=3
 EPOCH=1000
 
@@ -48,6 +47,8 @@ EPOCH=1000
 VALIDATORS=20
 TX_NODES=2
 STATIC_IPS_FOR_GETH_NODES=false
+# Whether tx_nodes/validators stateful set should use ssd persistent disks
+GETH_NODES_SSD_DISKS=true
 
 ADMIN_RPC_ENABLED=false
 

.env.integrationtesting

@@ -32,7 +32,6 @@ GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
 # Genesis Vars
 NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=5
 
 # "og" -> our original 4 tx nodes, "${n}" -> for deriving n tx nodes from the MNEMONIC

.env.pilot

@@ -39,7 +39,6 @@ GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
 # Genesis Vars
 NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=5
 EPOCH=17280   // Minimum epoch length is 1 day
 
@@ -48,6 +47,8 @@ EPOCH=17280   // Minimum epoch length is 1 day
 VALIDATORS=10
 TX_NODES=2
 STATIC_IPS_FOR_GETH_NODES=false
+# Whether tx_nodes/validators stateful set should use ssd persistent disks
+GETH_NODES_SSD_DISKS=true
 
 ADMIN_RPC_ENABLED=false
 
@@ -77,4 +78,4 @@ PROMTOSD_EXPORT_INTERVAL="5m"
 
 AUCTION_CRON_SPEC="*/5 * * * *"
 
-SMS_RETRIEVER_HASH_CODE=l5k6LvdPDXS
+SMS_RETRIEVER_HASH_CODE=1SlgTw9pFW5

.env.pilotstaging

@@ -36,7 +36,6 @@ GETH_EXPORTER_DOCKER_IMAGE_TAG="ed7d21bd50592709173368cd697ef73c1774a261"
 # Genesis Vars
 NETWORK_ID=1101
 CONSENSUS_TYPE="istanbul"
-PREDEPLOYED_CONTRACTS="REGISTRY"
 BLOCK_TIME=5
 EPOCH=17280   // Minimum epoch length is 1 day
 
@@ -45,6 +44,8 @@ EPOCH=17280   // Minimum epoch length is 1 day
 VALIDATORS=10
 TX_NODES=2
 STATIC_IPS_FOR_GETH_NODES=false
+# Whether tx_nodes/validators stateful set should use ssd persistent disks
+GETH_NODES_SSD_DISKS=true
 
 ADMIN_RPC_ENABLED=false
 
@@ -74,4 +75,4 @@ PROMTOSD_EXPORT_INTERVAL="5m"
 
 AUCTION_CRON_SPEC="*/5 * * * *"
 
-SMS_RETRIEVER_HASH_CODE=l5k6LvdPDXS
+SMS_RETRIEVER_HASH_CODE=1SlgTw9pFW5

.prettierignore

@@ -5,37 +5,26 @@ package.json
 **/privacy/*
 .git/
 
-packages/blockchain-api/dist
+packages/*/dist
+packages/*/lib
 
-packages/cli/lib
 
-packages/contractkit/lib/
-
-packages/walletkit/lib/
 packages/walletkit/.artifacts/
 packages/walletkit/contracts/
 packages/walletkit/types/
-packages/dappkit/lib/
+
 packages/docs/_book
-packages/faucet/dist/
 packages/mobile/src/geth/*.json
-packages/notification-service/dist/
 
 packages/protocol/build/
 packages/protocol/types/
-packages/protocol/lib/**/*.js
+!packages/protocol/lib/**/*.ts
 packages/protocol/scripts/**/*.js
 packages/protocol/migrations/**/*.js
 packages/protocol/tests/**/*.js
 
-packages/transaction-metrics-exporter/src/contracts/*.ts
-packages/dappkit/lib/
-packages/utils/lib/
-
-packages/verification-pool-api/lib/
 packages/verification-pool-api/contracts/*.ts
 
-packages/web/dist
 packages/web/.next
 
 # prettier eats Latex underscore escapting and doesn't seem to have an option to disable

.vscode/settings.json

@@ -6,6 +6,11 @@
     "**/*.js": { "when": "$(basename).ts" },
     "**/*.js.map": true
   },
+  "files.watcherExclude": {
+    "**/.git/objects/**": true,
+    "**/.git/subtree-cache/**": true,
+    "**/node_modules/*/**": true
+  },
   "typescript.preferences.importModuleSpecifier": "non-relative",
   "typescript.updateImportsOnFileMove.enabled": "always",
   "editor.codeActionsOnSave": {

README-dev.md

@@ -0,0 +1,71 @@
+# README GUIDE FOR CELO DEVELOPERS
+
+## How to publish a new npm package
+
+First checkout the alfajores branch.
+
+```
+celo-monorepo $ git checkout alfajores
+celo-monorepo $ yarn
+```
+
+Before publishing a new celocli package, test in isolation using Docker. This confirms that it is locally installable and does not have implict dependency on rest of the celo-monorepo or have an implicit dependency which is an explicit dependency of another celo-monorepo package.
+
+```
+# To test utils package, change $PWD/packages/cli to $PWD/packages/utils
+# To test contractkit package, change $PWD/packages/contractkit to $PWD/packages/contractkit
+celo-monorepo $ docker run -v $PWD/packages/cli:/tmp/npm_package -it --entrypoint bash node:8
+root@e0d56700584f:/# mkdir /tmp/tmp1 && cd /tmp/tmp1
+root@e0d56700584f:/tmp/tmp1# npm install /tmp/npm_package/
+```
+
+After testing, exit the docker container, and publish the package. Do note that all our packages are prefixed with "@celo/" and only members listed [here](https://www.npmjs.com/settings/celo/members) can publish new packages or update the existing ones.
+
+```
+# Publish the package publicly
+celo-monorepo/packages/cli $ yarn publish --access=public
+# Increment the version number, after testing, we will push that commit to GitHub
+```
+
+Let's say the published package version number 0.0.15, verify that it is installable
+
+```
+/tmp/tmp1 $ npm install @celo/[email protected]
+```
+
+Once you publish do some manual tests, for example, after publishing `celocli`
+
+```
+# Docker for an isolated environment again
+celo-monorepo $ docker run -it --entrypoint bash node:8
+root@e0d56700584f:/# mkdir /tmp/tmp1 && cd /tmp/tmp1
+root@e0d56700584f:/tmp/tmp1# npm install @celo/[email protected]
+/tmp/tmp1# ./node_modules/.bin/celocli
+CLI Tool for transacting with the Celo protocol
+
+VERSION
+  @celo/celocli/0.0.20 linux-x64 node-v8.16.1
+
+USAGE
+  $ celocli [COMMAND]
+
+COMMANDS
+  account         Manage your account, send and receive Celo Gold and Celo Dollars
+  bonds           Manage bonded deposits to participate in governance and earn rewards
+  config          Configure CLI options which persist across commands
+  exchange        Commands for interacting with the Exchange
+  help            display help for celocli
+  node            Manage your full node
+  validator       View validator information and register your own
+  validatorgroup  View validator group information and cast votes
+
+root@f8c51e3c7bc3:/tmp/tmp1# ./node_modules/.bin/celocli account:new
+This is not being stored anywhere, so, save the mnemonic somewhere to use this account at a later point
+
+mnemonic: wall school patrol also peasant enroll build merit health reduce junior obtain awful sword warfare sponsor honey display resemble bubble trend elevator ostrich assist
+privateKey: a9531609ca3d1c224e0742a4bb9b9e2fae67cc9d872797869092804e1500d67c
+publicKey: 0429b83753806f2b61ddab2e8a139214c3c8a5dfd0557557830b13342f2490bad6f61767e1b0707be51685e5e13683e6fa276333cbdb06f07768a09b8070e27259accountAddress: 0xf63e0F60DFcd84090D2863e0Dfa452B871fdC6d7
+```
+
+Now push your changes `git push origin alfajores`.
+If you don't have access to the repo, you might have to open a PR.