Merge pull request #30 from pangeo-forge/private-gke-nodes

Private GKE nodes

.env

@@ -9,4 +9,5 @@ CLUSTER_SERVICE_ACCOUNT_NAME="<ACCOUNT 2 HERE>"
 PROJECT_NAME="<PROJECT NAME HERE>"
 STORAGE_NAME="<STORAGE NAME HERE>"
 CLUSTER_NAME="<CLUSTER NAME HERE>"
-CLUSTER_REGION="<CLUSTER REGION HERE>"
+CLUSTER_REGION="<CLUSTER REGION HERE>"
+ENABLE_PRIVATE_CLUSTER="<BOOLEAN; Default=false>"

scripts/deploy.sh

@@ -78,6 +78,13 @@ else
   echo "CLUSTER_REGION is set to ${CLUSTER_REGION}"
 fi
 
+if [ -z "${ENABLE_PRIVATE_CLUSTER}" ]; then
+  echo "[X] - ENABLE_PRIVATE_CLUSTER is not set"
+  OK=0
+else
+  echo "ENABLE_PRIVATE_CLUSTER is set to ${ENABLE_PRIVATE_CLUSTER}"
+fi
+
 if [ $OK == 0 ]; then
   exit 1
 fi
@@ -92,6 +99,7 @@ export TF_VAR_storage_name="$STORAGE_NAME"
 export TF_VAR_cluster_name="$CLUSTER_NAME"
 export TF_VAR_cluster_region="$CLUSTER_REGION"
 export TF_VAR_project_name="$PROJECT_NAME"
+export TF_VAR_enable_private_cluster="$ENABLE_PRIVATE_CLUSTER"
 terraform init
 terraform plan
 terraform apply

terraform/cluster.tf

@@ -25,6 +25,37 @@ resource "google_container_cluster" "primary" {
       ip_allocation_policy
     ]
   }
+
+  /**
+  * Following private cluster config adapted from 2i2c:
+  * https://github.com/2i2c-org/infrastructure/pull/538/files
+  */
+
+  // For private clusters, pass the name of the network and subnetwork created
+  // by the VPC
+  network    = var.enable_private_cluster ? data.google_compute_network.default_network.name : null
+  subnetwork = var.enable_private_cluster ? data.google_compute_subnetwork.default_subnetwork.name : null
+
+  // Dynamically provision the private cluster config when deploying a
+  // private cluster
+  dynamic "private_cluster_config" {
+    for_each = var.enable_private_cluster ? [1] : []
+
+    content {
+      // Decide if this CIDR block is sensible or not
+      master_ipv4_cidr_block  = "172.16.0.0/28"
+      enable_private_nodes    = true
+      enable_private_endpoint = false
+    }
+  }
+
+  // Dynamically provision the IP allocation policy when deploying a
+  // private cluster. This allows for IP aliasing and makes the cluster
+  // VPC-native
+  dynamic "ip_allocation_policy" {
+    for_each = var.enable_private_cluster ? [1] : []
+    content {}
+  }
 }
 
 resource "google_container_node_pool" "primary_preemptible_nodes" {

terraform/network.tf

@@ -0,0 +1,57 @@
+/**
+* Networking to support private clusters. This config is only deployed when the
+* enable_private_cluster variable is set to true.
+* Adapted from 2i2c: https://github.com/2i2c-org/infrastructure/pull/538/files
+*/
+
+data "google_compute_network" "default_network" {
+  name    = "default"
+  project = var.project_name
+}
+
+data "google_compute_subnetwork" "default_subnetwork" {
+  name    = "default"
+  project = var.project_name
+  region  = var.cluster_region
+}
+
+resource "google_compute_firewall" "iap_ssh_ingress" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name    = "allow-ssh"
+  project = var.project_name
+  network = data.google_compute_network.default_network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+
+  // This range contains all IP addresses that IAP uses for TCP forwarding.
+  // https://cloud.google.com/iap/docs/using-tcp-forwarding
+  source_ranges = ["35.235.240.0/20"]
+}
+
+resource "google_compute_router" "router" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name    = "${var.project_name}-router"
+  project = var.project_name
+  region  = var.cluster_region
+  network = data.google_compute_network.default_network.id
+}
+
+resource "google_compute_router_nat" "nat" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name                               = "${var.project_name}-router-nat"
+  project                            = var.project_name
+  region                             = var.cluster_region
+  router                             = google_compute_router.router[0].name
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+
+  // Set these values explicitly so they don't "change outside terraform"
+  nat_ips       = []
+  drain_nat_ips = []
+}

terraform/variables.tf

@@ -15,4 +15,23 @@ variable "cluster_name" {
 }
 variable "cluster_region" {
   type = string
+}
+/*
+* "enable_private_cluster" borrowed from 2i2c; Cf.
+* https://github.com/2i2c-org/infrastructure/pull/538/files
+*/
+variable "enable_private_cluster" {
+  type        = bool
+  default     = false
+  description = <<-EOT
+  Deploy the kubernetes cluster into a private subnet
+
+  By default, GKE gives each of your nodes a public IP & puts them in a public
+  subnet. When this variable is set to `true`, the nodes will be in a private subnet
+  and not have public IPs. A cloud NAT will provide outbound internet access from
+  these nodes. The kubernetes API will still be exposed publicly, so we can access
+  it from our laptops & CD.
+  
+  This is often required by institutional controls banning VMs from having public IPs.
+  EOT
 }