ci: automatically creates an EC2 instance for builds (#220)

* ci: run on aws * ci: run always * branching draft - deploy EC2 from CI * commands inside of workflow * use terraform * go to the right directory for terraform * where am I * go to the right directory for terraform * node error * node error * checkout action * switch lib * setup terraform CLI * setup terraform CLI * setup terraform CLI * where am I * infra dir * infra dir * infra dir * infra dir * apply infra * separate run commands * separate run commands * separate run commands * workspace select * workspace select * fix terraform file * using ssh key * adding security group * add key * add connection * fix pem path * add host * add key * add key * remove agent * fix paths * remove key * with custom AMI * key name * test * new AMI image * add plan * add plan * add ssh type * user data * user data * agent to false * bootstrap runner * variable name * with user data * test * wrap * aws image * update access to instance * access token * fix path * runner image * runner not blocking * runner not blocking * runner not blocking * runner not blocking * run bash * run bash * run * run * run * run * run * add run conf * run conf * run conf * run build on the newly created instance * run build on the newly created instance * sleep between steps * sleep between steps * runs on aws * runs on aws * runs on aws * check build * check build * check build * check build * check build * check build * check build * needs * needs * needs * needs * needs * needs * ref * ref * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * dynamic branch name * with destroy * with destroy * with destroy * with destroy * with destroy * with destroy * with destroy * with wait * test destroy * test destroy * test destroy * test destroy * variabilise s3 bucket key * trigger * trigger * trigger * trigger * trigger * trigger * trigger * trigger * trigger * trigger with build * trigger with build * removed init.sh * ☠️ test sabotage * to hash commit * to hash commit * to hash commit * to hash commit * to hash commit * to hash commit * destroy runs anyway * release * release * release * always run destroy * Revert "☠️ test sabotage" This reverts commit bd294be * merge master * Revert "merge master" This reverts commit 6cd0675 * merged ci changes from master * new EC2 instance image with git 2.31 * remove redundant init * remove redundant init Co-authored-by: Lumir Mrkva <[email protected]> Co-authored-by: Ayoub <[email protected]> Co-authored-by: Lumir Mrkva <[email protected]> (cherry picked from commit 8762b4a)
galacticcouncil · Apr 28, 2021 · d562765 · d562765
1 parent a2066a3
commit d562765
Show file tree

Hide file tree

Showing 5 changed files with 183 additions and 6 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -1,12 +1,63 @@
-name: Build
-
+name: build-and-deploy
+#${GITHUB_REF##*/}
 on:
   push:
 
 jobs:
-  build:
-    runs-on: aws
+  pop-runner-instance:
+    name: create and setup infrastructure for branch
+    runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      EC2_SECRET: ${{ secrets.EC2_PWD }}
+      S3_BACKEND_KEY_NAME: tf-state-${{ github.sha }}
+    steps:
+      - name: checkout code
+        uses: actions/[email protected]
+      - name: setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 0.12.25
+          #- name: Declare some variables
+          # id: vars
+          #shell: bash
+          #run: |
+          # echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+          #echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      - name: rewrite tf state key
+        id: tf-state-key
+        working-directory: ./infrastructure
+        run: sed -i 's/BRANCH_ACTOR_KEY/${{ env.S3_BACKEND_KEY_NAME }}/g' infra.tf
+      - name: Terraform Init
+        id: init
+        working-directory: ./infrastructure
+        run: terraform init
+        continue-on-error: false
 
+      - name: Terraform plan
+        id: plan
+        working-directory: ./infrastructure
+        run: terraform plan -var="ec2_pwd=$EC2_SECRET" -var="branch_name=${{ github.sha }}"
+        continue-on-error: false
+      - name: Terraform apply
+        id: apply
+        working-directory: ./infrastructure
+        run: terraform apply -var="ec2_pwd=$EC2_SECRET" -var="branch_name=${{ github.sha }}" -auto-approve
+        continue-on-error: false
+      - name: Sleep for 20 seconds
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '20s'
+
+  build:
+    runs-on: ${{ github.sha }}
+    needs: pop-runner-instance
     steps:
       - uses: actions/checkout@v2
       - name: Install Rust
@@ -17,15 +68,66 @@ jobs:
           target: wasm32-unknown-unknown
           override: true
           default: true
-      - run: ./scripts/init.sh
+      - name: Install clippy
+        run: rustup component add clippy
+      - name: Run clippy
+        run: cargo clippy --release -- -D warnings
       - name: Tests
         run: time cargo test --all --release --verbose --locked
       - name: Benchmarking tests
         run: cd node && cargo test --release --features runtime-benchmarks -p pallet-exchange-benchmarking -p pallet-amm -p pallet-multi-payment-benchmarking
       - name: Build release
         run: time cargo build --release --verbose
-      - name: Upload release
+      - name: Version info
+        run: ./target/release/hydra-dx --version
+      - name: Upload release binary
         uses: actions/upload-artifact@v2
         with:
           name: hack-hydra-dx
           path: target/release/hack-hydra-dx
+      - name: Upload release wasm
+        uses: actions/upload-artifact@v2
+        with:
+          name: hydra_dx_runtime.wasm
+          path: target/release/wbuild/hack-hydra-dx-runtime/hack_hydra_dx_runtime.compact.wasm
+
+
+  destroy-runner-instance:
+    name: destroy infra
+    runs-on: ubuntu-latest
+    needs: build
+    if: always()
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      EC2_SECRET: ${{ secrets.EC2_PWD }}
+      S3_BACKEND_KEY_NAME: tf-state-${{ github.sha }}
+    steps:
+      - name: checkout code
+        uses: actions/[email protected]
+      - name: setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 0.12.25
+          continue-on-error: false
+      - name: rewrite tf state key
+        id: tf-state-key
+        working-directory: ./infrastructure
+        run: sed -i 's/BRANCH_ACTOR_KEY/${{ env.S3_BACKEND_KEY_NAME }}/g' infra.tf
+
+      - name: Terraform Init
+        id: init
+        working-directory: ./infrastructure
+        run: terraform init
+        continue-on-error: false
+
+      - name: Terraform apply
+        id: apply
+        working-directory: ./infrastructure
+        run: terraform destroy -var="ec2_pwd=$EC2_SECRET" -var="branch_name=${{ github.sha }}" -auto-approve
+        continue-on-error: false
diff --git a/infrastructure/config_script.sh b/infrastructure/config_script.sh
@@ -0,0 +1,7 @@
+curl -o actions-runner-linux-x64-2.277.1.tar.gz -L https://github.com/actions/runner/releases/download/v2.277.1/actions-runner-linux-x64-2.277.1.tar.gz
+
+tar xzf ./actions-runner-linux-x64-2.277.1.tar.gz
+TOKEN=$(bash get_token.sh $1)
+echo $TOKEN
+bash config.sh --url https://github.com/galacticcouncil/HydraDX-node --token $TOKEN --name "$2" --work _work --labels aws
+bash ./run.sh
diff --git a/infrastructure/get_token.sh b/infrastructure/get_token.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+header1="Accept: application/vnd.github.v3+json"
+header2="Authorization: token $1"
+#apiUrl="https://api.github.com/orgs/galacticcouncil/actions/runners/registration-token"
+apiUrl="https://api.github.com/repos/galacticcouncil/HydraDX-node/actions/runners/registration-token"
+
+cmd="$(curl -X POST -H "$header1" -H "$header2" "$apiUrl" | jq -r '.token')"
+echo "${cmd}"
diff --git a/infrastructure/infra.tf b/infrastructure/infra.tf
@@ -0,0 +1,55 @@
+# Setting Up Remote State
+variable "branch_name" {
+  description = ""
+}
+terraform {
+  required_version = ">= 0.12.24"
+
+  backend "s3" {
+    bucket = "hydradx-ci-backend-state"
+    key    = "BRANCH_ACTOR_KEY"
+    region = "eu-west-1"
+  }
+}
+
+provider "aws" {
+  region = var.aws_region
+}
+
+variable "aws_region" {
+  description = "The AWS region to create resources in."
+  default     = "eu-west-1"
+}
+
+variable "ec2_pwd" {
+  description = ""
+}
+
+
+resource "aws_instance" "runner-aws" {
+    ami = "ami-04f71779ae623a544"
+    instance_type = "c5ad.4xlarge"
+    subnet_id = "subnet-0ba99ac0d4aea3dc6"
+    key_name = "aws-ec2-key"
+    vpc_security_group_ids = ["sg-05f1a5d51f4d92cae"]
+
+    tags = {
+        Type = "Github_Self_Runner"
+    }
+    connection {
+        type = "ssh"
+        user = "ubuntu"
+        host = aws_instance.runner-aws.public_ip
+        password = var.ec2_pwd
+        timeout = "3m"
+    }
+    provisioner "file" {
+      source      = "run_conf.sh"
+      destination = "/home/ubuntu/run_conf.sh"
+    }
+    provisioner "remote-exec" {
+        inline = [
+        "tmux new -d 'bash run_conf.sh ${var.branch_name}'"
+        ]
+    }
+}
diff --git a/infrastructure/run_conf.sh b/infrastructure/run_conf.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+bash_source=$(source /home/ubuntu/config_script.sh)
+echo $bash_source
+bashrun=$(bash -c "config_script.sh $1 &")
+echo $bashrun