diff --git a/.travis.yml b/.travis.yml index 8589273392ab..36644fb497e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -115,7 +115,22 @@ jobs: - terraform validate $tf_validation_cli_options - terraform plan $tf_plan_cli_options - - stage: (Devnet)Terraform apply + - stage: (Devnet) Build, and push images + if: branch = dev-upgrade AND type = push AND tag IS blank + services: + - docker + install: skip + before_script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker --version # document the version travis is using + - docker build -t xdc-devnet -f cicd/devnet/Dockerfile . + script: + - pip install --user awscli # install aws cli w/o sudo + - export PATH=$PATH:$HOME/.local/bin # put aws in the path + - docker tag xdc-devnet:latest xinfinorg/devnet:latest # Always push to the latest + - docker push xinfinorg/devnet:latest + + - stage: Terraform apply if: branch = dev-upgrade AND type = push AND tag IS blank dist: xenial language: bash @@ -131,22 +146,11 @@ jobs: # Terraform init and then apply changes to environment - terraform init $tf_init_cli_options - terraform apply $tf_apply_cli_options - - - stage: (Devnet) Build, push and deploy - if: branch = dev-upgrade AND type = push AND tag IS blank - services: - - docker - install: skip - before_script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker --version # document the version travis is using - - docker build -t xdc-devnet -f cicd/devnet/Dockerfile . - script: - - pip install --user awscli # install aws cli w/o sudo - - export PATH=$PATH:$HOME/.local/bin # put aws in the path - - docker tag xdc-devnet:latest xinfinorg/devnet:latest # Always push to the latest - - docker push xinfinorg/devnet:latest - - sleep 10 # Wait for 10 second before asking the ecs to update the images - - aws ecs update-service --region us-east-1 --cluster devnet --service devnet-group-1 --force-new-deployment #TODO: Temporary solution until we have proper automated scripts ready - - \ No newline at end of file + - sleep 20 + - | + source cicd/devnet/terraform/.env + for ((i=0;i<$num_of_nodes;i++)); do + echo "Force deploy xdc-$i" + sleep 5 && aws ecs update-service --region us-east-1 --cluster devnet-xdcnode-cluster --service ecs-service-xdc$i --force-new-deployment; + done + diff --git a/README.md b/README.md index e092a27db84f..282e77517f8d 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ If at a later stage if some predecided amount of owners ( investors ) vote that ### For developers +#### Continues integration & delivery +See https://github.com/XinFinOrg/XDPoSChain/tree/dev-upgrade/cicd + + ### To contribute Simple create a pull request along with proper reasoning, we'll get back to you. diff --git a/cicd/README.md b/cicd/README.md index ac46caee3ecf..15106703cfbf 100644 --- a/cicd/README.md +++ b/cicd/README.md @@ -1,7 +1,10 @@ # CI/CD pipeline for XDC This directory contains CI/CD scripts used for each of the XDC environments. -### Devnet +## How to deploy more nodes +Adjust the number of variable `num_of_nodes` under file `.env`. (**Maximum supported is 58**) + +## Devnet Each PR merged into `dev-upgrade` will trigger below actions: - Tests - Terraform to apply infrascture changes(if any) @@ -9,20 +12,33 @@ Each PR merged into `dev-upgrade` will trigger below actions: - Docker push to docker hub. https://hub.docker.com/repository/docker/xinfinorg/devnet - Deployment of the latest XDC image(from above) to devnet run by AWS ECS -In order to allow pipeline able to push and deploy via ECR and ECS, we require below environment variables to be injected into the CI pipeline: -1. ECR_REPO_NAME -2. ECR_BASE_URI -3. AWS_ACCESS_KEY_ID -4. AWS_SECRET_ACCESS_KEY - -#### How to spin up more nodes in devnet -NOTE: The terraform managed auto deployment is still under deployment. The current best way to spin up new nodes is done by below: -1. `docker pull xinfinorg/devnet:latest` -2. `docker run -it -e PRIVATE_KEYS={{Wallet-Private-key-Here}} xinfinorg/devnet:latest` - +### First time set up an new environment +1. Pre-generate a list of node private keys in below format +``` +{ + "xdc0": { + "pk": {{PRIVATE KEY}} + }, + "xdc1": {...}, + "xdc{{NUMBER}}: {...} +} +``` +2. Access to aws console, create a bucket with name `terraform-devnet-bucket` +3. Upload the file from step 1 into the above bucket with name `node-config.json` +4. In order to allow pipeline able to push and deploy via ECR and ECS, we require below environment variables to be injected into the CI pipeline: + 1. DOCKER_USERNAME + 2. DOCKER_PASSWORD + 3. AWS_ACCESS_KEY_ID + 4. AWS_SECRET_ACCESS_KEY + +You are all set! -### Testnet -**WIP** +## Testnet +*** WIP *** +Testnet release build are triggered by cutting a "pre-release" tag which matches the name of `TESTNET-{{release-version}}` from dev-upgrade or master branch. +An example can be found here: https://github.com/XinFinOrg/XDPoSChain/releases/tag/Testnet-v2.0.0 +For more information, refer to github documentation on the release: https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases -### Mainnet -**WIP** \ No newline at end of file +## Mainnet +*** WIP *** +Mainnet release are triggered by making a normal release tag with name starting with `v` (stands for version) from the master branch. \ No newline at end of file diff --git a/cicd/devnet/Dockerfile b/cicd/devnet/Dockerfile index 192f84596aff..12c6f4b53b6a 100644 --- a/cicd/devnet/Dockerfile +++ b/cicd/devnet/Dockerfile @@ -31,6 +31,6 @@ EXPOSE 8545 # ws EXPOSE 8555 # port -EXPOSE 30304 +EXPOSE 30303 ENTRYPOINT ["bash","/work/start.sh"] \ No newline at end of file diff --git a/cicd/devnet/bootnodes.list b/cicd/devnet/bootnodes.list index e3065b9f8130..6341b3c18ec3 100644 --- a/cicd/devnet/bootnodes.list +++ b/cicd/devnet/bootnodes.list @@ -1,2 +1,2 @@ -enode://1c20e6b46ce608c1fe739e78611225b94e663535b74a1545b1667eac8ff75ed43216306d123306c10e043f228e42cc53cb2728655019292380313393eaaf6e23@194.233.77.19:30301 -enode://1c20e6b46ce608c1fe739e78611225b94e663535b74a1545b1667eac8ff75ed43216306d123306c10e043f228e42cc53cb2728655019292380313393eaaf6e23@66.94.98.186:30301 \ No newline at end of file +enode://c31bf87732529ef2d92e247f546fb139b8ab7476fc4e2d14fe4ce38631890c11ca9f87fed8c1b3e505e116d9a2674644bbef7ec296e518b688cc692e2aef885d@194.233.77.19:30303 +enode://a2e685e0daf718d4697f49eaa7c8e8bdfa25678d5b24afd8caa684261a90c513062b1fe9140effa04b7a98b0c971607b6413fe993344875e97137c00a7993efb@66.94.121.151:30303 \ No newline at end of file diff --git a/cicd/devnet/start.sh b/cicd/devnet/start.sh index cca09fa0f755..8784c438a8ea 100755 --- a/cicd/devnet/start.sh +++ b/cicd/devnet/start.sh @@ -2,7 +2,7 @@ echo "Preparing to start the XDC chain, it's likely to take up to 1 minute" # Sleep for > 30 as we need to wait for the ECS tasks container being killed by fargate. Otherwise it will ended up with two same nodes running on a single /work/xdcchain directory -sleep 45 +sleep 60 if [ ! -d /work/xdcchain/XDC/chaindata ] then @@ -41,10 +41,11 @@ if test -z "$LOG_LEVEL" then echo "Log level not set, default to verbosity of 3" else + echo "Log level found, set to $LOG_LEVEL" log_level=$LOG_LEVEL fi -netstats="aws_${wallet}:xinfin_xdpos_hybrid_network_stats@devnetstats.apothem.network:2000" +netstats="${NODE_NAME}-${wallet}:xinfin_xdpos_hybrid_network_stats@devnetstats.apothem.network:2000" INSTANCE_IP=$(curl https://checkip.amazonaws.com) echo "Running a node with wallet: ${wallet} at IP: ${INSTANCE_IP}" @@ -53,7 +54,7 @@ echo "Starting nodes with $bootnodes ..." XDC --ethstats ${netstats} --gcmode=archive \ --bootnodes ${bootnodes} --syncmode full \ --datadir /work/xdcchain --networkid 551 \ --port 30304 --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \ +-port 30303 --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \ --rpcport 8545 \ --rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,XDPoS \ --rpcvhosts "*" --unlock "${wallet}" --password /work/.pwd --mine \ diff --git a/cicd/devnet/terraform/.env b/cicd/devnet/terraform/.env new file mode 100644 index 000000000000..52d0028f41e2 --- /dev/null +++ b/cicd/devnet/terraform/.env @@ -0,0 +1,2 @@ +num_of_nodes=2 +log_level=1 \ No newline at end of file diff --git a/cicd/devnet/terraform/container-definition.tpl b/cicd/devnet/terraform/container-definition.tpl index 6d3d231f85cd..7bc2bf78e389 100644 --- a/cicd/devnet/terraform/container-definition.tpl +++ b/cicd/devnet/terraform/container-definition.tpl @@ -3,7 +3,9 @@ "name": "tfXdcNode", "image": "xinfinorg/${xdc_environment}:latest", "environment": [ - {"name": "PRIVATE_KEYS", "value": "${private_keys}"} + {"name": "PRIVATE_KEYS", "value": "${private_keys}"}, + {"name": "LOG_LEVEL", "value": "${log_level}"}, + {"name": "NODE_NAME", "value": "${node_name}"} ], "essential": true, "logConfiguration": { @@ -15,11 +17,6 @@ } }, "portMappings": [ - { - "hostPort": 80, - "protocol": "tcp", - "containerPort": 80 - }, { "hostPort": 8555, "protocol": "tcp", @@ -31,9 +28,9 @@ "containerPort": 8545 }, { - "hostPort": 30304, + "hostPort": 30303, "protocol": "tcp", - "containerPort": 30304 + "containerPort": 30303 } ], "mountPoints": [ diff --git a/cicd/devnet/terraform/ecs.tf b/cicd/devnet/terraform/ecs.tf index cfcd82af7f29..65f69bdf0d40 100644 --- a/cicd/devnet/terraform/ecs.tf +++ b/cicd/devnet/terraform/ecs.tf @@ -1,16 +1,18 @@ data template_file devnet_container_definition { - for_each = var.devnet_node_kyes + for_each = local.devnetNodeKyes template = "${file("${path.module}/container-definition.tpl")}" vars = { xdc_environment = "devnet" - private_keys = "${each.value.pk}", + node_name = "${each.key}" + private_keys = "${each.value.pk}" cloudwatch_group = "tf-${each.key}" + log_level = "${local.logLevel}" } } resource "aws_ecs_task_definition" "devnet_task_definition_group" { - for_each = var.devnet_node_kyes + for_each = local.devnetNodeKyes family = "devnet-${each.key}" requires_compatibilities = ["FARGATE"] @@ -38,4 +40,44 @@ resource "aws_ecs_task_definition" "devnet_task_definition_group" { tags = { Name = "TfDevnetEcs-${each.key}" } +} + +data "aws_ecs_task_definition" "devnet_ecs_task_definition" { + for_each = local.devnetNodeKyes + task_definition = aws_ecs_task_definition.devnet_task_definition_group[each.key].family +} + +resource "aws_ecs_cluster" "devnet_ecs_cluster" { + name = "devnet-xdcnode-cluster" + tags = { + Name = "TfDevnetEcsCluster" + } +} + +resource "aws_ecs_service" "devnet_ecs_service" { + for_each = local.devnetNodeKyes + name = "ecs-service-${each.key}" + cluster = aws_ecs_cluster.devnet_ecs_cluster.id + task_definition = "${aws_ecs_task_definition.devnet_task_definition_group[each.key].family}:${max(aws_ecs_task_definition.devnet_task_definition_group[each.key].revision, data.aws_ecs_task_definition.devnet_ecs_task_definition[each.key].revision)}" + launch_type = "FARGATE" + scheduling_strategy = "REPLICA" + desired_count = 1 + force_new_deployment = true + + network_configuration { + subnets = [aws_subnet.devnet_subnet.id] + assign_public_ip = true + security_groups = [ + aws_default_security_group.devnet_xdcnode_security_group.id + ] + } + + deployment_circuit_breaker { + enable = true + rollback = false + } + + tags = { + Name = "TfDevnetEcsService-${each.key}" + } } \ No newline at end of file diff --git a/cicd/devnet/terraform/efs.tf b/cicd/devnet/terraform/efs.tf index b45e8e0bd9c3..bae9e2daa7b8 100644 --- a/cicd/devnet/terraform/efs.tf +++ b/cicd/devnet/terraform/efs.tf @@ -1,5 +1,28 @@ # EFS +resource "aws_security_group" "devnet_efs_security_group" { + name = "TfDevnetEfsSecurityGroup" + description = "Allow HTTP in and out of devnet EFS" + vpc_id = aws_vpc.devnet_vpc.id + + ingress { + from_port = 2049 + to_port = 2049 + protocol = "TCP" + security_groups = [aws_default_security_group.devnet_xdcnode_security_group.id] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + tags = { + Name = "TfDevnetEfs" + } +} + resource "aws_efs_file_system" "devnet_efs" { creation_token = "efs" performance_mode = "generalPurpose" @@ -17,7 +40,7 @@ resource "aws_efs_mount_target" "devnet_efs_efs_mount_target" { } resource "aws_efs_access_point" "devnet_efs_access_point" { - for_each = var.devnet_node_kyes + for_each = local.devnetNodeKyes file_system_id = aws_efs_file_system.devnet_efs.id root_directory { path = "/${each.key}/database" diff --git a/cicd/devnet/terraform/main.tf b/cicd/devnet/terraform/main.tf index d1328e248e82..5b643347ec44 100644 --- a/cicd/devnet/terraform/main.tf +++ b/cicd/devnet/terraform/main.tf @@ -13,24 +13,6 @@ provider "aws" { region = "us-east-1" } -# This bucket had to be created before you can run the terraform init -resource "aws_s3_bucket" "terraform_s3_bucket" { - bucket = "terraform-devnet-bucket" - versioning { - enabled = true - } -} - -# Bucket need to be created first. If first time run terraform init, need to comment out the below section -terraform { - backend "s3" { - bucket = "terraform-devnet-bucket" - key = "tf/terraform.tfstate" - region = "us-east-1" - encrypt = true - } -} - resource "aws_vpc" "devnet_vpc" { cidr_block = "10.0.0.0/16" instance_tenancy = "default" @@ -99,31 +81,7 @@ resource "aws_default_security_group" "devnet_xdcnode_security_group" { } } -resource "aws_security_group" "devnet_efs_security_group" { - name = "TfDevnetEfsSecurityGroup" - description = "Allow HTTP in and out of devnet EFS" - vpc_id = aws_vpc.devnet_vpc.id - - ingress { - from_port = 2049 - to_port = 2049 - protocol = "TCP" - security_groups = [aws_default_security_group.devnet_xdcnode_security_group.id] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - tags = { - Name = "TfDevnetEfs" - } -} - # IAM policies - data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" { statement { actions = ["sts:AssumeRole"] @@ -154,7 +112,8 @@ resource "aws_iam_role_policy_attachment" "devnet_xdc_ecs_tasks_execution_role" # Logs resource "aws_cloudwatch_log_group" "devnet_cloud_watch_group" { - for_each = var.devnet_node_kyes + for_each = local.devnetNodeKyes + name = "tf-${each.key}" retention_in_days = 14 # Logs are only kept for 14 days tags = { diff --git a/cicd/devnet/terraform/s3.tf b/cicd/devnet/terraform/s3.tf new file mode 100644 index 000000000000..ce955302f833 --- /dev/null +++ b/cicd/devnet/terraform/s3.tf @@ -0,0 +1,24 @@ + + +# This bucket had to be created before you can run the terraform init +resource "aws_s3_bucket" "terraform_s3_bucket" { + bucket = "terraform-devnet-bucket" + versioning { + enabled = true + } +} + +# Bucket need to be created first. If first time run terraform init, need to comment out the below section +terraform { + backend "s3" { + bucket = "terraform-devnet-bucket" + key = "tf/terraform.tfstate" + region = "us-east-1" + encrypt = true + } +} + +data "aws_s3_bucket_object" "devnet_xdc_node_config" { + bucket = "terraform-devnet-bucket" + key = "node-config.json" +} \ No newline at end of file diff --git a/cicd/devnet/terraform/variables.tf b/cicd/devnet/terraform/variables.tf index e9fdcde317e0..62fd689b09b3 100644 --- a/cicd/devnet/terraform/variables.tf +++ b/cicd/devnet/terraform/variables.tf @@ -1,19 +1,18 @@ -variable "devnet_node_kyes" { - description = "Array of nodes keys." - type = map(any) - - /** - Below is the list of private keys you need to specify. It follows the pattern of - {{Name of the node}}: { +locals { + /** + Load the nodes data from s3 + Below is the the format the config needs to follow: + {{Name of the node, in a pattern of 'xdc'+ number. i.e xdc50}}: { pk: {{Value of the node private key}}, ... any other configuration we want to pass. } Note: No `n` is allowed in the node name **/ - default = { - xdc1 = { - pk = "3efdb44088929167487da052125162b48d8d54fe8f7b7db11b5d5cc3b9a1c14b", - isChaosNode = false # This is a placeholder, config not supported yet + predefinedNodesConfig = jsondecode(data.aws_s3_bucket_object.devnet_xdc_node_config.body) + envs = { for tuple in regexall("(.*)=(.*)", file(".env")) : tuple[0] => tuple[1] } + logLevel = local.envs["log_level"] + keyNames =[for i in range(tonumber(local.envs["num_of_nodes"])) : "xdc${i}"] + devnetNodeKyes = { + for i in local.keyNames: i => local.predefinedNodesConfig[i] } - } } \ No newline at end of file