Merge pull request apache#16 from riskive/ZFE-74287-create-small-inst…

…ance Deploy insights to nomad
rguerraZero · Aug 29, 2023 · 7b5a3ae · 7b5a3ae
2 parents 8281326 + 26abeb0
commit 7b5a3ae
Show file tree

Hide file tree

Showing 23 changed files with 1,038 additions and 21 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -20,4 +20,6 @@ steps:
       - mkdir ~/.ssh && echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519 && chmod 0600 ~/.ssh/id_ed25519
       - ssh-keyscan -H github.com >> ~/.ssh/known_hosts
       - make -C .terra/superset init
-      - make -C .terra/superset validate
+      - make -C .terra/superset validate
+      - make -C .terra/insights init
+      - make -C .terra/insights validate
diff --git a/.terra/insights/Makefile b/.terra/insights/Makefile
@@ -0,0 +1,53 @@
+AWS_REGION:=$(if $(AWS_REGION),$(AWS_REGION),us-west-2)
+ENVIRONMENT:=qa
+GIT_SHA="$(shell git rev-parse --short=7 HEAD)"
+TERRAFORM:=terraform
+S3_BUCKET_PREFIX:=$(if $(S3_BUCKET_PREFIX),$(S3_BUCKET_PREFIX),zf-terraform)
+
+tf_files := $(shell find . -name "*.tf")
+
+all: plan
+
+check-fmt:
+	$(TERRAFORM) fmt -check=true
+
+format:
+	$(TERRAFORM) fmt
+
+init:
+	$(TERRAFORM) init \
+		-backend-config="bucket=$(S3_BUCKET_PREFIX)-$(ENVIRONMENT)" \
+		-backend-config="region=$(AWS_REGION)"
+
+validate: ${tf_files}
+	$(TERRAFORM) validate
+
+plan: .terraform/terraform.tfstate ${tf_files}
+	$(TERRAFORM) plan \
+		-var "env=$(ENVIRONMENT)" \
+		-var "git_sha=$(GIT_SHA)" \
+		-out terra.plan
+
+apply: plan
+	$(TERRAFORM) apply terra.plan
+
+clean:
+	rm -f *.backup *.plan
+	rm -f .terraform/terraform.tfstate
+
+clobber: clean
+	rm -rf .terraform
+
+plan_destroy: .terraform/terraform.tfstate ${tf_files}
+	$(TERRAFORM) plan -destroy \
+		-var "env=$(ENVIRONMENT)" \
+		-var "git_sha=$(GIT_SHA)" \
+		-out terra_destroy.plan
+
+destroy: plan_destroy
+	$(TERRAFORM) destroy \
+		-var "env=$(ENVIRONMENT)" \
+		-var "git_sha=$(GIT_SHA)" \
+		-auto-approve
+
+.PHONY: all check-fmt format init validate plan apply clean clobber destroy plan_destroy
diff --git a/.terra/insights/celery_beat/celery_beat.nomad.hcl b/.terra/insights/celery_beat/celery_beat.nomad.hcl
@@ -0,0 +1,106 @@
+group "celery-beat-group" {
+  count = "${count}"
+
+  meta {
+    group = "$${NOMAD_GROUP_NAME}"
+    lang  = "python"
+  }
+
+  task "celery-beat" {
+    driver = "docker"
+    shutdown_delay = "10s"
+
+    config {
+      image   = "${ecr_url}/zf/insights:${git_sha}"
+      command = "celery"
+      args = [
+        "--app=superset.tasks.celery_app:app",
+        "beat",
+      ]
+      force_pull = true
+    }
+
+    resources {
+      cpu    = 1024
+      memory = 1024
+
+      network {
+        mbits = 1
+        port "https" {}
+      }
+    }
+
+    service {
+      name = "$${NOMAD_JOB_NAME}-$${NOMAD_GROUP_NAME}"
+      port = "https"
+
+      check {
+        name     = "$${NOMAD_JOB_NAME}-$${NOMAD_GROUP_NAME} up check"
+        type    = "script"
+        command = "celery"
+        args    = [
+          "-A",
+          "superset.tasks.celery_app:app",
+          "inspect",
+          "ping"
+        ]
+        interval = "30s"
+        timeout  = "10s"
+      }
+    }
+template {
+     data = <<EOH
+{
+  "type": "service_account",
+  {{ with secret "secret/superset/gcp_insights_sa" }}
+  "project_id": "{{ .Data.project_id }}",
+  "private_key_id": "{{ .Data.private_key_id }}",
+  "private_key": "{{ .Data.private_key }}",
+  "client_email": "{{ .Data.client_email }}",
+  "client_id": "{{ .Data.client_id }}",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "{{ .Data.client_x509_cert_url }}"{{ end }}
+}
+EOH
+        destination = "secrets/service-acct.json"
+        change_mode="restart"
+      }
+
+    template {
+      data = <<EOH
+GOOGLE_APPLICATION_CREDENTIALS=/secrets/service-acct.json
+
+{{ with secret "database/insights-superset-db/creds/admin" }}
+DATABASE_URL="postgresql://{{ .Data.username }}:{{ .Data.password }}@${db_address}:${db_port}/insights_superset"
+{{ end }}
+
+{{ with secret "secret/superset/insights" }}
+SSO_API_BASE_URL="{{ .Data.SSO_API_BASE_URL }}"
+SSO_CLIENT_ID="{{ .Data.SSO_CLIENT_ID }}"
+SSO_CLIENT_SECRET="{{ .Data.SSO_CLIENT_SECRET }}"
+SUPERSET_SECRET_KEY="{{ .Data.SUPERSET_SECRET_KEY }}"
+{{ end }}
+
+{{ with secret "secret/smtp" }}
+SENDGRID_HOST="{{ .Data.host }}"
+SENDGRID_PORT="{{ .Data.port }}"
+SENDGRID_USERNAME="{{ .Data.username }}"
+SENDGRID_PASSWORD="{{ .Data.password }}"{{ end }}
+
+{{ range ls "${app}/superset/env" }}
+{{ .Key|toUpper }}="{{ .Value }}"
+{{ end }}
+      EOH
+      destination = "secrets/env"
+      change_mode = "restart"
+      env         = true
+    }
+
+    vault {
+      policies    = ["${app}"]
+      change_mode = "restart"
+    }
+  }
+}
diff --git a/.terra/insights/celery_beat/celery_beat.tf b/.terra/insights/celery_beat/celery_beat.tf
@@ -0,0 +1,55 @@
+variable "app" {}
+
+variable "env" {}
+
+variable "aws_region" {
+  default = "us-west-2"
+}
+
+variable "git_sha" {}
+
+variable "cmd" {
+  default = "celery-beat"
+}
+
+variable "ecr_url" {}
+
+variable "db_address" {}
+
+variable "db_port" {}
+
+# ----------------------------------------
+# AWS
+# ----------------------------------------
+provider "aws" {
+  region = "${var.aws_region}"
+}
+
+# ----------------------------------------
+# Nomad Configuration
+# ----------------------------------------
+locals {
+  container_counts = {
+    qa   = 1
+    stag = 1
+    prod = 1
+  }
+}
+
+data "template_file" "nomad_group" {
+  template = "${file("./celery_beat/celery_beat.nomad.hcl")}"
+
+  vars {
+    aws_region = "${var.aws_region}"
+    ecr_url    = "${var.ecr_url}"
+    git_sha    = "${var.git_sha}"
+    app        = "${var.app}"
+    db_address = "${var.db_address}"
+    db_port    = "${var.db_port}"
+    count = "${local.container_counts[var.env]}"
+  }
+}
+
+output "nomad_group" {
+  value = "${data.template_file.nomad_group.rendered}"
+}
diff --git a/.terra/insights/celery_flower/celery_flower.nomad.hcl b/.terra/insights/celery_flower/celery_flower.nomad.hcl
@@ -0,0 +1,109 @@
+group "celery-flower-group" {
+  count = "${count}"
+
+  meta {
+    group = "$${NOMAD_GROUP_NAME}"
+    lang  = "python"
+  }
+
+  task "celery-flower" {
+    driver = "docker"
+    shutdown_delay = "10s"
+
+    config {
+      image   = "${ecr_url}/zf/insights:${git_sha}"
+      command = "celery"
+      args = [
+        "--app=superset.tasks.celery_app:app",
+        "flower",
+      ]
+      force_pull = true
+      port_map = {
+          https = 5555
+        }
+    }
+
+    resources {
+      cpu    = 1024
+      memory = 1024
+
+      network {
+        mbits = 1
+        port "https" {}
+      }
+    }
+
+    service {
+      name = "$${NOMAD_JOB_NAME}-$${NOMAD_GROUP_NAME}"
+      port = "https"
+
+      check {
+        name     = "$${NOMAD_JOB_NAME}-$${NOMAD_GROUP_NAME} up check"
+        type     = "script"
+        command  = "celery"
+        args     = [
+          "-A",
+          "superset.tasks.celery_app:app",
+          "inspect",
+          "ping"
+        ]
+        interval = "30s"
+        timeout  = "10s"
+      }
+    }
+  template {
+     data = <<EOH
+{
+  "type": "service_account",
+  {{ with secret "secret/superset/gcp_insights_sa" }}
+  "project_id": "{{ .Data.project_id }}",
+  "private_key_id": "{{ .Data.private_key_id }}",
+  "private_key": "{{ .Data.private_key }}",
+  "client_email": "{{ .Data.client_email }}",
+  "client_id": "{{ .Data.client_id }}",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "{{ .Data.client_x509_cert_url }}"{{ end }}
+}
+EOH
+        destination = "secrets/service-acct.json"
+        change_mode="restart"
+      }
+
+    template {
+      data = <<EOH
+GOOGLE_APPLICATION_CREDENTIALS=/secrets/service-acct.json
+
+{{ with secret "database/insights-superset-db/creds/admin" }}
+DATABASE_URL="postgresql://{{ .Data.username }}:{{ .Data.password }}@${db_address}:${db_port}/insights_superset"
+{{ end }}
+
+{{ with secret "secret/superset/insights" }}
+SSO_API_BASE_URL="{{ .Data.SSO_API_BASE_URL }}"
+SSO_CLIENT_ID="{{ .Data.SSO_CLIENT_ID }}"
+SSO_CLIENT_SECRET="{{ .Data.SSO_CLIENT_SECRET }}"
+SUPERSET_SECRET_KEY="{{ .Data.SUPERSET_SECRET_KEY }}"
+{{ end }}
+
+{{ with secret "secret/smtp" }}
+SENDGRID_HOST="{{ .Data.host }}"
+SENDGRID_PORT="{{ .Data.port }}"
+SENDGRID_USERNAME="{{ .Data.username }}"
+SENDGRID_PASSWORD="{{ .Data.password }}"{{ end }}
+
+{{ range ls "${app}/superset/env" }}
+{{ .Key|toUpper }}="{{ .Value }}"
+{{ end }}
+      EOH
+      destination = "secrets/env"
+      change_mode = "restart"
+      env         = true
+    }
+
+    vault {
+      policies    = ["${app}"]
+      change_mode = "restart"
+    }
+  }
+}
diff --git a/.terra/insights/celery_flower/celery_flower.tf b/.terra/insights/celery_flower/celery_flower.tf
@@ -0,0 +1,53 @@
+variable "app" {}
+
+variable "env" {}
+
+variable "aws_region" {}
+
+variable "git_sha" {}
+
+variable "cmd" {
+  default = "celery-flower"
+}
+
+variable "ecr_url" {}
+
+variable "db_address" {}
+
+variable "db_port" {}
+
+# ----------------------------------------
+# AWS
+# ----------------------------------------
+provider "aws" {
+  region = "${var.aws_region}"
+}
+
+# ----------------------------------------
+# Nomad Configuration
+# ----------------------------------------
+locals {
+  container_counts = {
+    qa   = 1
+    stag = 1
+    prod = 1
+  }
+}
+
+data "template_file" "nomad_group" {
+  template = "${file("./celery_flower/celery_flower.nomad.hcl")}"
+
+  vars {
+    aws_region = "${var.aws_region}"
+    ecr_url    = "${var.ecr_url}"
+    git_sha    = "${var.git_sha}"
+    app        = "${var.app}"
+    db_address = "${var.db_address}"
+    db_port    = "${var.db_port}"
+    count = "${local.container_counts[var.env]}"
+  }
+}
+
+output "nomad_group" {
+  value = "${data.template_file.nomad_group.rendered}"
+}