storage-gke.yaml

# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
blueprint_name: storage-gke

vars:
  project_id:  ## Set GCP Project ID Here ##
  deployment_name: storage-gke-01
  region: us-central1
  zone: us-central1-c
  # Cidr block containing the IP of the machine calling terraform.
  # The following line must be updated for this example to work.
  authorized_cidr: <your-ip-address>/32
  gcp_public_cidrs_access_enabled: false

deployment_groups:
- group: primary
  modules:
  - id: network1
    source: modules/network/vpc
    settings:
      subnetwork_name: $(vars.deployment_name)-subnet
      secondary_ranges_list:
      - subnetwork_name: $(vars.deployment_name)-subnet
        ranges:
        - range_name: pods
          ip_cidr_range: 10.4.0.0/14
        - range_name: services
          ip_cidr_range: 10.0.32.0/20

  - id: node_pool_service_account
    source: community/modules/project/service-account
    settings:
      name: gke-np-sa
      project_roles:
      - logging.logWriter
      - monitoring.metricWriter
      - monitoring.viewer
      - stackdriver.resourceMetadata.writer
      - storage.objectViewer
      - artifactregistry.reader

  - id: workload_service_account
    source: community/modules/project/service-account
    settings:
      name: gke-wl-sa
      project_roles:
      - logging.logWriter
      - monitoring.metricWriter
      - monitoring.viewer
      - stackdriver.resourceMetadata.writer
      - storage.objectAdmin
      - artifactregistry.reader

  - id: gke_cluster
    source: modules/scheduler/gke-cluster
    use: [network1, workload_service_account]
    settings:
      enable_filestore_csi: true
      enable_gcsfuse_csi: true
      configure_workload_identity_sa: true  # needed when using GCS
      enable_private_endpoint: false  # Allows for access from authorized public IPs
      gcp_public_cidrs_access_enabled: $(vars.gcp_public_cidrs_access_enabled)
      master_authorized_networks:
      - display_name: deployment-machine
        cidr_block: $(vars.authorized_cidr)
    outputs: [instructions]

  - id: debug_pool
    source: modules/compute/gke-node-pool
    use: [gke_cluster, node_pool_service_account]
    settings:
      name: debug
      zones: [$(vars.zone)]
      machine_type: n2d-standard-2

  ### Google Cloud Storage ###

  - id: data-bucket
    source: community/modules/file-system/cloud-storage-bucket
    settings:
      local_mount: /data
      random_suffix: true
      force_destroy: true

  - id: data-bucket-pv
    source: modules/file-system/gke-persistent-volume
    use: [gke_cluster, data-bucket]
    settings: {capacity_gb: 5000}

  ### Filestore ###

  - id: filestore
    source: modules/file-system/filestore
    use: [network1]
    settings: {local_mount: /shared}

  - id: shared-filestore-pv
    source: modules/file-system/gke-persistent-volume
    use: [gke_cluster, filestore]

  ### Shared Storage Job ###

  - id: shared-fs-job
    source: modules/compute/gke-job-template
    use:
    - gke_cluster
    - debug_pool
    - shared-filestore-pv
    - data-bucket-pv
    settings:
      image: bash
      command:
      - bash
      - -c
      - |
        echo "Set up job folders"
        shopt -s extglob; JOB=${HOSTNAME%%-+([[:digit:]])}
        mkdir /data/${JOB}/ -p; mkdir /shared/${JOB}/ -p;

        echo "Writing seed file to GCS"
        dd if=/dev/urandom of=/data/${JOB}/${JOB_COMPLETION_INDEX}.dat bs=1K count=1000

        echo "Copy seed data from GCS to Filestore"
        cp /data/${JOB}/${JOB_COMPLETION_INDEX}.dat /shared/${JOB}/

        echo "Hash file from Filestore and save to GCS"
        md5sum /shared/${JOB}/${JOB_COMPLETION_INDEX}.dat > /data/${JOB}/${JOB_COMPLETION_INDEX}.md5
      node_count: 5
    outputs: [instructions]

  ### Ephemeral Storage ###

  - id: local-ssd-pool
    source: modules/compute/gke-node-pool
    use: [gke_cluster, node_pool_service_account]
    settings:
      name: local-ssd
      machine_type: n2d-standard-2
      local_ssd_count_ephemeral_storage: 1

  - id: ephemeral-storage-job
    source: modules/compute/gke-job-template
    use: [local-ssd-pool]
    settings:
      name: ephemeral-storage-job
      ephemeral_volumes:  # below shows all options, usually only 1 is needed

      - type: memory  # backed by node memory
        mount_path: /scratch-mem
        size_gb: 5

      - type: local-ssd  # node pool must specify local_ssd_count_ephemeral_storage
        mount_path: /scratch-local-ssd
        size_gb: 280  # System holds back some of 375 GiB

      - type: pd-ssd
        mount_path: /pd-ssd
        size_gb: 100

      - type: pd-balanced
        mount_path: /pd-balanced
        size_gb: 100

      image: ljishen/fio
      command:  # https://cloud.google.com/compute/docs/disks/benchmarking-pd-performance
      - fio
      - --name=write_throughput
      - --directory=/scratch-local-ssd
      - --numjobs=16
      - --size=5G
      - --time_based
      - --runtime=30s
      - --ramp_time=2s
      - --ioengine=libaio
      - --direct=1
      - --verify=0
      - --bs=1M
      - --iodepth=64
      - --rw=write
      - --group_reporting=1
      - --iodepth_batch_submit=64
      - --iodepth_batch_complete_max=64
    outputs: [instructions]