diff --git a/gcp/terraform/.terraform.lock.hcl b/gcp/terraform/.terraform.lock.hcl index 06d68e8..d71ff63 100644 --- a/gcp/terraform/.terraform.lock.hcl +++ b/gcp/terraform/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/google" { version = "4.31.0" constraints = "4.31.0" hashes = [ + "h1:+X1KG9mnYGMkYD1c/O+uUi72d4kB3kPSDei4yMEDVAQ=", "h1:p6GUUYG9PQ4XODXBgsHpWTygziRtY61z07slBKYbiCQ=", "zh:02a19ed46c2007f6aadfb6ff90aa6063be063194d1f0dd02dc839adc212f7cae", "zh:1046de7e13e81a8f86461f99e9d5ff25d5dabe8465f51efe72084ded426ba771", @@ -22,48 +23,70 @@ provider "registry.terraform.io/hashicorp/google" { } provider "registry.terraform.io/hashicorp/google-beta" { - version = "4.33.0" + version = "4.46.0" hashes = [ - "h1:EqS+G9gLapyjtlg68x+/6O542/lwO+nbxpDXQkKoatE=", - "zh:1c9f8b86a76cc7495012652acf3e5f865c560fba269ac5b582b3801d765be0fd", - "zh:4af2ba43bd42b6f3dc94b2ab070e5891ae73958e5648f762d529112d5f65faa2", - "zh:4b14fdbaefe1d000cff6bfc86ea3448055903ab667296d44e5436252cc8e36b4", - "zh:4ff9b4e90fc47be68ee04a59ea0deddc49b354a8422620f94f464f1a43a491d6", - "zh:53e9667c4ad88ba807922c20cc6057a4e35d9c3519158ef3a7d75fbf17544e9b", - "zh:9472368264cad36cc6170f045fcc4f8f77ff3937713c9ed0fe0500642218ec0a", - "zh:c7061c192128bbcf420cf9c8c321aac0131ba984510c7fea7ea5670b3b9fe855", - "zh:cf0833a927b9a4a90fa598cb893b2171941f582c9d240d108a3dc4cf899f9a22", - "zh:d2b77452927f02b01568d77dba928f88c3ed466626b5964d1d5da8dfac8dfad1", - "zh:e66188de7c036e3c11f0042965b65c9237822666896b55dbe2382b31fa8e137c", + "h1:GQgp57Wrf3oDVB/nDtBd/VL3/uMTFN5bLPhPLLADGB4=", + "zh:08aa990fd9944061194138ad4f136f5e6b45f331d110d882e4ddb566619eb9d8", + "zh:186b9c7b49ad93a2ab2d8d713429caa8b23dab8d90763c01244205c3455dd813", + "zh:221598948eab9c64e13a778c6be17dd1e9cb2e08a3217072d9759202986c3f09", + "zh:402d386ea907923bbf36568dc481becda2dd0522c5286602dcb716f364f73d91", + "zh:84d70da182503ce312148cc86e110482c88d57041223af00d2ead60fefe851ee", + "zh:92bd8e30f6334988d6e7fedff11b99c68fe0d21bfead6f1cbbfc73acb665c36d", + "zh:a0e5815460c2a1d720955a2abbd6ca6eef450da7a76c52c223203f5d03ca45d7", + "zh:c34c0124f70b86ceb4cf79b93539539286f47175e1a648e37cfb754200cc19b7", + "zh:d00769dbfd1bea46da5ac81a9e3384d0f954ed3f912859b21ed3ea2378c2cb1d", + "zh:dadd937ed05dcc3d521cd6ca04bc9681fd30acbb42265f748e69f5feb0f5b829", + "zh:e4c2d65cf138cc868ae52e3e2cf97e37fbbdbc9c510f1c669fdbcc736256e402", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.6.0" + constraints = "2.6.0" + hashes = [ + "h1:rGVucCeYAqklKupwoLVG5VPQTIkUhO7WGcw3WuHYrm8=", + "zh:0ac248c28acc1a4fd11bd26a85e48ab78dd6abf0f7ac842bf1cd7edd05ac6cf8", + "zh:3d32c8deae3740d8c5310136cc11c8afeffc350fbf88afaca0c34a223a5246f5", + "zh:4055a27489733d19ca7fa2dfce14d323fe99ae9dede7d0fea21ee6db0b9ca74b", + "zh:58a8ed39653fd4c874a2ecb128eccfa24c94266a00e349fd7fb13e22ad81f381", + "zh:6c81508044913f25083de132d0ff81d083732aba07c506cc2db05aa0cefcde2c", + "zh:7db5d18093047bfc4fe597f79610c0a281b21db0d61b0bacb3800585e976f814", + "zh:8269207b7422db99e7be80a5352d111966c3dfc7eb98511f11c8ff7b2e813456", + "zh:b1d7ababfb2374e72532308ff442cc906b79256b66b3fe7a98d42c68c4ddf9c5", + "zh:ca63e226cbdc964a5d63ef21189f059ce45c3fa4a5e972204d6916a9177d2b44", + "zh:d205a72d60e8cc362943d66f5bcdd6b6aaaa9aab2b89fd83bf6f1978ac0b1e4c", + "zh:db47dc579a0e68e5bfe3a61f2e950e6e2af82b1f388d1069de014a937962b56a", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fb9cef7f371ff8e2c9ff9d2690e99958e322d4f22ccec8e900952e08bb434da0", ] } provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.13.0" + version = "2.16.1" constraints = ">= 2.12.1" hashes = [ - "h1:vE0TXrBXKiPbMLC4VqihXCZxWtKyxdmGqlen747/Hhk=", - "zh:0432af00bc631be4019ba7611afae2816c00d4b2ee40469743f64be203f783cb", - "zh:21dabdfaa0f48012ae05a4114f93d0208faa55f98fd260caf8254abc6c1c0682", - "zh:2bea1511150348b8b2d220f83e4f34e24a3b3420608fe1a3763323be12d440d5", - "zh:35a1dba51cb92b5eee966d6c5c99ca8d345db5f082e6c5582cff76ecf3c908c2", - "zh:54539d56202ae457c9a81785295fab87304436a9a91bac6ba46eccb4143556bd", - "zh:62a405082ddc729399bbc7a99a12a59ed76d127330f412947ca6e88f39644c43", - "zh:8505c3e9c5a2f1ffb6ef51bf3ee2abf19d3e6d0a92bf244249483ef5c7e02c68", - "zh:953a964b93e8c50db2da81463fe25e42a04d77abb841d386eda153035a384efe", - "zh:a0ffbad202a4c96c5226a785fab4f9a59d5f9b50c72e27e790ab5ceb7c09e978", - "zh:a9b5ec96e56dbbef4c5b801ba90162c680ad8e1d5096cc92bc83618eccd56cd8", - "zh:c588f471ff111ace9cab27e244eb62de0d828233cbaac6b71bf29d5a7c3c6d43", + "h1:i+DwtJK82sIWmTcQA9lL0mlET+14/QpUqv10fU2o3As=", + "zh:06224975f5910d41e73b35a4d5079861da2c24f9353e3ebb015fbb3b3b996b1c", + "zh:2bc400a8d9fe7755cca27c2551564a9e2609cfadc77f526ef855114ee02d446f", + "zh:3a479014187af1d0aec3a1d3d9c09551b801956fe6dd29af1186dec86712731b", + "zh:73fb0a69f1abdb02858b6589f7fab6d989a0f422f7ad95ed662aaa84872d3473", + "zh:a33852cd382cbc8e06d3f6c018b468ad809d24d912d64722e037aed1f9bf39db", + "zh:b533ff2214dca90296b1d22eace7eaa7e3efe5a7ae9da66a112094abc932db4f", + "zh:ddf74d8bb1aeb01dc2c36ef40e2b283d32b2a96db73f6daaf179fa2f10949c80", + "zh:e720f3a15d34e795fa9ff90bc755e838ebb4aef894aa2a423fb16dfa6d6b0667", + "zh:e789ae70a658800cb0a19ef7e4e9b26b5a38a92b43d1f41d64fc8bb46539cefb", + "zh:e8aed7dc0bd8f843d607dee5f72640dbef6835a8b1c6ea12cea5b4ec53e463f7", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb3ac4f43c8b0dfc0b0103dd0f062ea72b3a34518d4c8808e3a44c9a3dd5f024", ] } provider "registry.terraform.io/hashicorp/local" { - version = "2.2.3" + version = "2.2.3" + constraints = "2.2.3" hashes = [ "h1:FvRIEgCmAezgZUqb2F+PZ9WnSSnR5zbEM2ZI+GLmbMk=", + "h1:aWp5iSUxBGgPv1UnV5yag9Pb0N+U1I0sZb38AXBFO8A=", "zh:04f0978bb3e052707b8e82e46780c371ac1c66b689b4a23bbc2f58865ab7d5c0", "zh:6484f1b3e9e3771eb7cc8e8bab8b35f939a55d550b3f4fb2ab141a24269ee6aa", "zh:78a56d59a013cb0f7eb1c92815d6eb5cf07f8b5f0ae20b96d049e73db915b238", @@ -80,21 +103,21 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/null" { - version = "3.1.1" + version = "3.2.1" hashes = [ - "h1:YvH6gTaQzGdNv+SKTZujU1O0bO+Pw6vJHOPhqgN8XNs=", - "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597", - "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf", - "zh:73ce6dff935150d6ddc6ac4a10071e02647d10175c173cfe5dca81f3d13d8afe", + "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8fdd792a626413502e68c195f2097352bdc6a0df694f7df350ed784741eb587e", - "zh:976bbaf268cb497400fd5b3c774d218f3933271864345f18deebe4dcbfcd6afa", - "zh:b21b78ca581f98f4cdb7a366b03ae9db23a73dfa7df12c533d7c19b68e9e72e5", - "zh:b7fc0c1615dbdb1d6fd4abb9c7dc7da286631f7ca2299fb9cd4664258ccfbff4", - "zh:d1efc942b2c44345e0c29bc976594cb7278c38cfb8897b344669eafbc3cddf46", - "zh:e356c245b3cd9d4789bab010893566acace682d7db877e52d40fc4ca34a50924", - "zh:ea98802ba92fcfa8cf12cbce2e9e7ebe999afbf8ed47fa45fc847a098d89468b", - "zh:eff8872458806499889f6927b5d954560f3d74bf20b6043409edf94d26cd906f", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } @@ -102,6 +125,7 @@ provider "registry.terraform.io/hashicorp/random" { version = "3.3.2" constraints = "3.3.2" hashes = [ + "h1:H5V+7iXol/EHB2+BUMzGlpIiCOdV74H8YjzCxnSAWcg=", "h1:YChjos7Hrvr2KgTc9GzQ+de/QE2VLAeRJgxFemnCltU=", "zh:038293aebfede983e45ee55c328e3fde82ae2e5719c9bd233c324cfacc437f9c", "zh:07eaeab03a723d83ac1cc218f3a59fceb7bbf301b38e89a26807d1c93c81cef8", diff --git a/gcp/terraform/README.md b/gcp/terraform/README.md index 9239bc8..932f1c1 100644 --- a/gcp/terraform/README.md +++ b/gcp/terraform/README.md @@ -50,6 +50,35 @@ Then, apply the `services` module (deploys Metaflow services to GKE) The step above will output next steps for Metaflow end users. +## Metaflow job orchestration options +The recommended way to orchestrate Metaflow workloads on Kubernetes is via [Argo Workflows](https://docs.metaflow.org/going-to-production-with-metaflow/scheduling-metaflow-flows/scheduling-with-argo-workflows). However, Airflow is also supported as an alternative. + +The template also provides the `deploy_airflow` and `deploy_argo` flags as variables. These are booleans that specify if [Airflow](https://airflow.apache.org/) or [Argo Workflows](https://argoproj.github.io/argo-workflows/) will be deployed in the Kubernetes cluster along with Metaflow related services. By default `deploy_argo` is set to __true__ and `deploy_airflow` is set to __false__. +To change these, set them in your `FILE.tfvars` file (or else, via other [terraform variable](https://www.terraform.io/language/values/variables) passing mechanisms) + +### Argo Workflows +Argo Workflows is installed by default on the AKS cluster as part of the `services` submodule. Setting the `deploy_argo` [variable](./variables.tf) will deploy Argo in the GKE cluster. No additional configuration is done in the `infra` module to support `argo`. + +After you have changed the value of `deploy_argo`, re-apply terraform for both [infra and services](#usage). + +### Airflow + +**This is quickstart template only, not recommended for real production deployments** + +If `deploy_airflow` is set to true, then the `services` module will deploy Airflow via a [helm chart](https://airflow.apache.org/docs/helm-chart/stable/index.html) into the kubernetes cluster (the one deployed by the `infra` module). + +The terraform template deploys Airflow configured with a `LocalExecutor`. Metaflow can work with any Airflow executor. This template deploys the `LocalExecutor` for simplicity. + +After you have changed the value of `deploy_airflow`, reapply terraform for both [infra and services](#usage). + +#### Shipping Metaflow compiled DAGs to Airflow +Airflow expects Python files with Airflow DAGS present in the [dags_folder](https://airflow.apache.org/docs/apache-airflow/2.2.0/configurations-ref.html#dags-folder). By default this terraform template uses the [defaults](https://airflow.apache.org/docs/helm-chart/stable/parameters-ref.html#airflow) set in the Airflow helm chart which is `{AIRFLOW_HOME}/dags` (`/opt/airflow/dags`). + +The metaflow-tools repository also ships a [airflow_dag_upload.py](../../scripts/airflow_dag_upload.py) file that can help sync Airflow dag file generated by Metaflow to the Airflow scheduler _deployed by this template_. Under the hood [airflow_dag_upload.py](../../scripts/airflow_dag_upload.py) uses the `kubectl cp` command to copy files from local to the Airflow scheduler's container. Example of how to use the file: +``` +python airflow_dag_upload.py my-dag.py /opt/airflow/dags/my-dag.py +``` + ## (Advanced) Terraform state management Terraform manages the state of GCP resources in [tfstate](https://www.terraform.io/language/state) files locally by default. diff --git a/gcp/terraform/main.tf b/gcp/terraform/main.tf index cf5f353..8d79039 100644 --- a/gcp/terraform/main.tf +++ b/gcp/terraform/main.tf @@ -16,6 +16,10 @@ terraform { source = "hashicorp/local" version = "2.2.3" } + helm = { + source = "hashicorp/helm" + version = "2.6.0" + } } } @@ -45,12 +49,22 @@ data "google_sql_database_instance" "default" { } provider "kubernetes" { - host = "https://${data.google_container_cluster.default.endpoint}" - token = data.google_client_config.default.access_token + host = "https://${data.google_container_cluster.default.endpoint}" + token = data.google_client_config.default.access_token cluster_ca_certificate = base64decode( data.google_container_cluster.default.master_auth[0].cluster_ca_certificate, ) } +provider "helm" { + kubernetes { + host = "https://${data.google_container_cluster.default.endpoint}" + cluster_ca_certificate = base64decode(data.google_container_cluster.default.master_auth[0].cluster_ca_certificate) + token = data.google_client_config.default.access_token + # token is required here and we remove `client_certificate` / `client_key` because it results in this error like : + # `Error: unable to build kubernetes objects from release manifest: unknown` + # More notes on this issue can be found here : https://github.com/hashicorp/terraform-provider-helm/issues/513 + } +} # This will be used for invoking kubectl re: Argo installation resource "local_file" "kubeconfig" { @@ -81,6 +95,7 @@ module "services" { metaflow_ui_static_service_image = local.metaflow_ui_static_service_image metaflow_ui_backend_service_image = local.metaflow_ui_backend_service_image metaflow_datastore_sysroot_gs = local.metaflow_datastore_sysroot_gs + airflow_logs_bucket_path = local.airflow_logs_bucket_path metaflow_db_host = "localhost" metaflow_db_name = "metaflow" metaflow_db_user = "metaflow" @@ -93,4 +108,8 @@ module "services" { metaflow_workload_identity_ksa_name = local.metaflow_workload_identity_ksa_name metadata_service_image = local.metadata_service_image kubeconfig_path = local_file.kubeconfig.filename + deploy_airflow = var.deploy_airflow + deploy_argo = var.deploy_argo + airflow_version = local.airflow_version + airflow_frenet_secret = local.airflow_frenet_secret } \ No newline at end of file diff --git a/gcp/terraform/output.tf b/gcp/terraform/output.tf index 7bc1895..6934c6e 100644 --- a/gcp/terraform/output.tf +++ b/gcp/terraform/output.tf @@ -58,6 +58,8 @@ METAFLOW_SERVICE_URL=http://127.0.0.1:8080/ METAFLOW_SERVICE_INTERNAL_URL=http://metadata-service.default:8080/ [For Argo only] METAFLOW_KUBERNETES_NAMESPACE=argo [For Argo only] METAFLOW_KUBERNETES_SERVICE_ACCOUNT=argo +[For Airflow only] METAFLOW_KUBERNETES_NAMESPACE=airflow +[For Airflow only] METAFLOW_KUBERNETES_SERVICE_ACCOUNT=airflow-deployment-scheduler [For non-Argo only] METAFLOW_KUBERNETES_SERVICE_ACCOUNT=${local.metaflow_workload_identity_ksa_name} Note: you can skip these: @@ -76,7 +78,7 @@ $ kubectl port-forward -n argo deployment/argo-server 2746:2746 option 2 - this script manages the same port-forwards for you (and prevents timeouts) -$ python metaflow-tools/scripts/forward_metaflow_ports.py [--include-argo] +$ python metaflow-tools/scripts/forward_metaflow_ports.py [--include-argo] [--include-airflow] STEP 4: Install GCP Python SDK $ pip install google-cloud-storage google-auth diff --git a/gcp/terraform/services/airflow.tf b/gcp/terraform/services/airflow.tf new file mode 100644 index 0000000..9b806b5 --- /dev/null +++ b/gcp/terraform/services/airflow.tf @@ -0,0 +1,51 @@ +resource "kubernetes_namespace" "airflow" { + count = var.deploy_airflow ? 1 : 0 + metadata { + name = "airflow" + } +} + +locals { + airflow_values = { + "executor" = "LocalExecutor" + "defaultAirflowTag" = var.airflow_version + "airflowVersion" = var.airflow_version + "webserverSecretKey" = var.airflow_frenet_secret + } +} + + +resource "helm_release" "airflow" { + count = var.deploy_airflow ? 1 : 0 + name = "airflow-deployment" + + repository = "https://airflow.apache.org" + chart = "airflow" + + namespace = kubernetes_namespace.airflow[0].metadata[0].name + + timeout = 1200 + + wait = false # Why set `wait=false` + #: Read this (https://github.com/hashicorp/terraform-provider-helm/issues/683#issuecomment-830872443) + # Short summary : If this is not set then airflow doesn't end up running migrations on the database. That makes the scheduler and other containers to keep waiting for migrations. + + values = [ + yamlencode(local.airflow_values) + ] +} +# annotation is added to the scheduler's pod so that the pod's service account can +# talk to Google cloud storage. +resource "kubernetes_annotations" "airflow_service_account_annotation" { + count = var.deploy_airflow ? 1 : 0 + depends_on = [helm_release.airflow] + api_version = "v1" + kind = "ServiceAccount" + metadata { + name = "airflow-deployment-scheduler" + namespace = kubernetes_namespace.airflow[0].metadata[0].name + } + annotations = { + "iam.gke.io/gcp-service-account" = "${var.metaflow_workload_identity_gsa_name}@${var.project}.iam.gserviceaccount.com" + } +} \ No newline at end of file diff --git a/gcp/terraform/services/argo.tf b/gcp/terraform/services/argo.tf index 0b15654..e49dbd6 100644 --- a/gcp/terraform/services/argo.tf +++ b/gcp/terraform/services/argo.tf @@ -1,4 +1,5 @@ resource "kubernetes_namespace" "argo" { + count = var.deploy_argo ? 1 : 0 metadata { name = "argo" } @@ -16,22 +17,24 @@ locals { # https://registry.terraform.io/providers/gavinbunney/kubectl/1.14 # The main challenge is that the Argo yaml contains multiple k8s resources, and terraform does not accept that natively. resource "null_resource" "argo-quick-start-installation" { + count = var.deploy_argo ? 1 : 0 triggers = { cmd = local._apply_cmd } provisioner "local-exec" { interpreter = local.is_windows ? ["PowerShell"] : null - command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._apply_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._apply_cmd}" + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._apply_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._apply_cmd}" } } resource "null_resource" "argo-annotate-service-account" { + count = var.deploy_argo ? 1 : 0 depends_on = [null_resource.argo-quick-start-installation] triggers = { cmd = local._annotate_cmd } provisioner "local-exec" { interpreter = local.is_windows ? ["PowerShell"] : null - command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._annotate_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._annotate_cmd}" + command = local.is_windows ? "$env:KUBECONFIG='${var.kubeconfig_path}'; ${local._annotate_cmd}" : "KUBECONFIG=${var.kubeconfig_path} ${local._annotate_cmd}" } } diff --git a/gcp/terraform/services/service_account.tf b/gcp/terraform/services/service_account.tf index 58244c6..143a574 100644 --- a/gcp/terraform/services/service_account.tf +++ b/gcp/terraform/services/service_account.tf @@ -1,7 +1,7 @@ # TODO rename to "_for_default" -resource kubernetes_service_account "metaflow_service_account" { +resource "kubernetes_service_account" "metaflow_service_account" { metadata { - name = var.metaflow_workload_identity_ksa_name + name = var.metaflow_workload_identity_ksa_name namespace = "default" annotations = { "iam.gke.io/gcp-service-account" = "${var.metaflow_workload_identity_gsa_name}@${var.project}.iam.gserviceaccount.com" @@ -13,8 +13,9 @@ resource "google_service_account_iam_binding" "metaflow-service-account-iam" { service_account_id = var.metaflow_workload_identity_gsa_id role = "roles/iam.workloadIdentityUser" - members = [ + members = flatten([ "serviceAccount:${var.project}.svc.id.goog[${kubernetes_service_account.metaflow_service_account.id}]", - "serviceAccount:${var.project}.svc.id.goog[argo/argo]", - ] + var.deploy_airflow ? ["serviceAccount:${var.project}.svc.id.goog[airflow/airflow-deployment-scheduler]"] : [], + var.deploy_argo ? ["serviceAccount:${var.project}.svc.id.goog[argo/argo]"] : [], + ]) } \ No newline at end of file diff --git a/gcp/terraform/services/variables.tf b/gcp/terraform/services/variables.tf index 10565ad..e448d34 100644 --- a/gcp/terraform/services/variables.tf +++ b/gcp/terraform/services/variables.tf @@ -1,40 +1,40 @@ -variable metaflow_ui_static_service_image { +variable "metaflow_ui_static_service_image" { type = string } -variable metaflow_datastore_sysroot_gs { +variable "metaflow_datastore_sysroot_gs" { type = string } -variable metaflow_db_name { +variable "metaflow_db_name" { type = string } -variable metaflow_db_user { +variable "metaflow_db_user" { type = string } -variable metaflow_db_host { +variable "metaflow_db_host" { type = string } -variable metaflow_ui_backend_service_image { +variable "metaflow_ui_backend_service_image" { type = string } -variable metaflow_db_port { +variable "metaflow_db_port" { type = string } -variable metaflow_db_password { +variable "metaflow_db_password" { type = string } -variable project { +variable "project" { type = string } -variable db_connection_name { +variable "db_connection_name" { type = string } @@ -42,18 +42,41 @@ variable "metaflow_workload_identity_gsa_name" { type = string } -variable metaflow_workload_identity_gsa_id { +variable "metaflow_workload_identity_gsa_id" { type = string } -variable metaflow_workload_identity_ksa_name { +variable "metaflow_workload_identity_ksa_name" { type = string } -variable metadata_service_image { +variable "metadata_service_image" { type = string } variable "kubeconfig_path" { type = string } + + +variable "airflow_version" { + type = string +} + +variable "airflow_frenet_secret" { + type = string +} + + +variable "deploy_argo" { + type = bool +} + +variable "deploy_airflow" { + type = bool +} + +variable "airflow_logs_bucket_path" { + type = string +} + diff --git a/gcp/terraform/variables.tf b/gcp/terraform/variables.tf index ae1d06b..dd7e21b 100644 --- a/gcp/terraform/variables.tf +++ b/gcp/terraform/variables.tf @@ -1,4 +1,4 @@ -resource random_id database_server_name_suffix { +resource "random_id" "database_server_name_suffix" { byte_length = 4 keepers = { db_generation_number = var.db_generation_number @@ -8,34 +8,49 @@ resource random_id database_server_name_suffix { locals { database_server_name_prefix = "psql-metaflow-${terraform.workspace}" - database_server_name = "${local.database_server_name_prefix}-${random_id.database_server_name_suffix.hex}" - kubernetes_cluster_name = "gke-metaflow-${terraform.workspace}" - region = "us-west2" - zone = "us-west2-a" + database_server_name = "${local.database_server_name_prefix}-${random_id.database_server_name_suffix.hex}" + kubernetes_cluster_name = "gke-metaflow-${terraform.workspace}" + region = "us-west2" + zone = "us-west2-a" - storage_bucket_name = "storage-${var.org_prefix}-metaflow-${terraform.workspace}" + storage_bucket_name = "storage-${var.org_prefix}-metaflow-${terraform.workspace}" metaflow_datastore_sysroot_gs = "gs://${local.storage_bucket_name}/tf-full-stack-sysroot" + airflow_logs_bucket_path = "gs://${local.storage_bucket_name}/airflow/logs" + metaflow_ui_static_service_image = "public.ecr.aws/outerbounds/metaflow_ui:v1.1.4" # metaflow_ui_backend_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" metaflow_ui_backend_service_image = "jackieob/metadata_service:gcp.rc1" - metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" + metadata_service_image = "public.ecr.aws/outerbounds/metaflow_metadata_service:2.3.3" # TODO gsa-metaflow-workload-id- metaflow_workload_identity_gsa_name = "gsa-metaflow-${terraform.workspace}" metaflow_workload_identity_ksa_name = "ksa-metaflow" service_account_key_file = "${path.root}/metaflow_gsa_key_${terraform.workspace}.json" + + airflow_version = "2.5.0" + airflow_frenet_secret = "myverysecretvalue" } -variable project { +variable "project" { type = string } -variable org_prefix { +variable "org_prefix" { type = string } -variable db_generation_number { - type = number +variable "db_generation_number" { + type = number default = 0 +} + +variable "deploy_argo" { + type = bool + default = true +} + +variable "deploy_airflow" { + type = bool + default = false } \ No newline at end of file