Feat/pvc attribute boot disks (#151)

Introduces a new label `disk_type` which can be represented as
- `boot_disk`
- `persistent_volume`

The purpose of this is to help further differentiate the type of disks that are being exported. When comparing exported results, I found ~4000 disks that were not represented by `opencost`. After investigating further, these disks were specifically related to boot disks. 

The neat thing about this is we can start associated the costs of boot volumes to the team that is responsible for provisioning nodes within our k8s environments. 

- relates to #5

docs/metrics/gcp/gke.md

@@ -1,10 +1,10 @@
 # GKE Compute Metrics
 
-| Metric name                                                | Metric type | Description                                                                                 | Labels                                                                                                                                                                                                                                                                                                                                                                                                                           |
-|------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| cloudcost_gcp_gke_instance_cpu_usd_per_core_hour           | Gauge       | The processing cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(core*h) | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `instance`=&lt;name of the compute instance&gt; <br/> `region`=&lt;GCP region code&gt; <br/> `family`=&lt;broader compute family (n1, n2, c3 ...) &gt; <br/> `machine_type`=&lt;specific machine type, e.g.: n2-standard-2&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `price_tier`=&lt;spot\|ondemand&gt; |
-| cloudcost_gcp_gke_compute_instance_memory_usd_per_gib_hour | Gauge       | The memory cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(GiB*h)      | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `instance`=&lt;name of the compute instance&gt; <br/> `region`=&lt;GCP region code&gt; <br/> `family`=&lt;broader compute family (n1, n2, c3 ...) &gt; <br/> `machine_type`=&lt;specific machine type, e.g.: n2-standard-2&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `price_tier`=&lt;spot\|ondemand&gt; |
-| cloudcost_gcp_gke_persistent_volume_usd_per_gib_hour       | Gauge       | The cost of a GKE Persistent Volume in USD/(GiB*h)                                          | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `namespace`=&lt;The namespace the pvc was created for&gt; <br/> `persistentvolume`=&lt;Name of the persistent volume&gt; <br/> `region`=&lt;The region the pvc was created in&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `storage_class`=&lt;pd-standard\|pd-ssd\|pd-balanced\|pd-extreme&gt;             |
+| Metric name                                                | Metric type | Description                                                                                 | Labels                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+|------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| cloudcost_gcp_gke_instance_cpu_usd_per_core_hour           | Gauge       | The processing cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(core*h) | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `instance`=&lt;name of the compute instance&gt; <br/> `region`=&lt;GCP region code&gt; <br/> `family`=&lt;broader compute family (n1, n2, c3 ...) &gt; <br/> `machine_type`=&lt;specific machine type, e.g.: n2-standard-2&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `price_tier`=&lt;spot\|ondemand&gt;                                            |
+| cloudcost_gcp_gke_compute_instance_memory_usd_per_gib_hour | Gauge       | The memory cost of a GCP Compute Instance, associated to a GKE cluster, in USD/(GiB*h)      | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `instance`=&lt;name of the compute instance&gt; <br/> `region`=&lt;GCP region code&gt; <br/> `family`=&lt;broader compute family (n1, n2, c3 ...) &gt; <br/> `machine_type`=&lt;specific machine type, e.g.: n2-standard-2&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `price_tier`=&lt;spot\|ondemand&gt;                                            |
+| cloudcost_gcp_gke_persistent_volume_usd_per_gib_hour       | Gauge       | The cost of a GKE Persistent Volume in USD/(GiB*h)                                          | `cluster_name`=&lt;name of the cluster the instance is associated with&gt; <br/> `namespace`=&lt;The namespace the pvc was created for&gt; <br/> `persistentvolume`=&lt;Name of the persistent volume&gt; <br/> `region`=&lt;The region the pvc was created in&gt; <br/> `project`=&lt;GCP project, where the instance is provisioned&gt; <br/> `storage_class`=&lt;pd-standard\|pd-ssd\|pd-balanced\|pd-extreme&gt; <br/> `disk_type`=&lt;boot_disk\|persistent_volume&gt; |
 
 ## Persistent Volumes
 

pkg/google/gke/disk.go

@@ -0,0 +1,126 @@
+package gke
+
+import (
+	"encoding/json"
+	"log"
+	"strings"
+
+	"google.golang.org/api/compute/v1"
+
+	gcpCompute "github.com/grafana/cloudcost-exporter/pkg/google/compute"
+)
+
+const (
+	BootDiskLabel        = "goog-gke-node"
+	pvcNamespaceKey      = "kubernetes.io/created-for/pvc/namespace"
+	pvcNamespaceShortKey = "kubernetes.io-created-for/pvc-namespace"
+	pvNameKey            = "kubernetes.io/created-for/pv/name"
+	pvNameShortKey       = "kubernetes.io-created-for/pv-name"
+)
+
+type Disk struct {
+	Cluster string
+
+	Project     string
+	name        string // Name of the disk as it appears in the GCP console. Used as a backup if the name can't be extracted from the description
+	zone        string
+	labels      map[string]string
+	description map[string]string
+	diskType    string // type is a reserved word, which is why we're using diskType
+}
+
+func NewDisk(disk *compute.Disk, project string) *Disk {
+	clusterName := disk.Labels[gcpCompute.GkeClusterLabel]
+	d := &Disk{
+		Cluster:     clusterName,
+		Project:     project,
+		name:        disk.Name,
+		zone:        disk.Zone,
+		diskType:    disk.Type,
+		labels:      disk.Labels,
+		description: make(map[string]string),
+	}
+	err := extractLabelsFromDesc(disk.Description, d.description)
+	if err != nil {
+		log.Printf("error extracting labels from disk(%s) description: %v", d.Name(), err)
+	}
+	return d
+}
+
+// Namespace will search through the description fields for the namespace of the disk. If the namespace can't be determined
+// An empty string is return.
+func (d Disk) Namespace() string {
+	return coalesce(d.description, pvcNamespaceKey, pvcNamespaceShortKey)
+}
+
+// Region will return the region of the disk by search through the zone field and returning the region. If the region can't be determined
+// It will return an empty string
+func (d Disk) Region() string {
+	zone := d.labels[gcpCompute.GkeRegionLabel]
+	if zone == "" {
+		// This would be a case where the disk is no longer mounted _or_ the disk is associated with a Compute instance
+		zone = d.zone[strings.LastIndex(d.zone, "/")+1:]
+	}
+	// If zone _still_ is empty we can't determine the region, so we return an empty string
+	// This prevents an index out of bounds error
+	if zone == "" {
+		return ""
+	}
+	if strings.Count(zone, "-") < 2 {
+		return zone
+	}
+	return zone[:strings.LastIndex(zone, "-")]
+}
+
+// Name will return the name of the disk. If the disk has a label "kubernetes.io/created-for/pv/name" it will return the value stored in that key.
+// otherwise it will return the disk name that is directly associated with the disk.
+func (d Disk) Name() string {
+	if d.description == nil {
+		return d.name
+	}
+	// first check that the key exists in the map, if it does return the value
+	name := coalesce(d.description, pvNameKey, pvNameShortKey)
+	if name != "" {
+		return name
+	}
+	return d.name
+}
+
+// coalesce will take a map and a list of keys and return the first value that is found in the map. If no value is found it will return an empty string
+func coalesce(desc map[string]string, keys ...string) string {
+	for _, key := range keys {
+		if val, ok := desc[key]; ok {
+			return val
+		}
+	}
+	return ""
+}
+
+// extractLabelsFromDesc will take a description string and extract the labels from it. GKE disks store their description as
+// a json blob in the description field. This function will extract the labels from that json blob and return them as a map
+// Some useful information about the json blob are name, cluster, namespace, and pvc's that the disk is associated with
+func extractLabelsFromDesc(description string, labels map[string]string) error {
+	if description == "" {
+		return nil
+	}
+	if err := json.Unmarshal([]byte(description), &labels); err != nil {
+		return err
+	}
+	return nil
+}
+
+// StorageClass will return the storage class of the disk by looking at the type. Type in GCP is represented as a URL and as such
+// we're looking for the last part of the URL to determine the storage class
+func (d Disk) StorageClass() string {
+	diskType := strings.Split(d.diskType, "/")
+	return diskType[len(diskType)-1]
+}
+
+// DiskType will search through the labels to determine the type of disk. If the disk has a label "goog-gke-node" it will return "boot_disk"
+// Otherwise it returns persistent_volume
+func (d Disk) DiskType() string {
+	if _, ok := d.labels[BootDiskLabel]; ok {
+		return "boot_disk"
+	}
+	return "persistent_volume"
+}

pkg/google/gke/disk_test.go

@@ -0,0 +1,244 @@
+package gke
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	computev1 "google.golang.org/api/compute/v1"
+
+	"github.com/grafana/cloudcost-exporter/pkg/google/compute"
+)
+
+func Test_extractLabelsFromDesc(t *testing.T) {
+	tests := map[string]struct {
+		description    string
+		labels         map[string]string
+		expectedLabels map[string]string
+		wantErr        bool
+	}{
+		"Empty description should return an empty map": {
+			description: "",
+			// Label needs to be initialized to an empty map, otherwise the underlying method to write data to it will fail
+			labels:         map[string]string{},
+			expectedLabels: map[string]string{},
+			wantErr:        false,
+		},
+		"Description not formatted as json should return an error": {
+			description: "test",
+			// Label needs to be initialized to an empty map, otherwise the underlying method to write data to it will fail
+			labels:         map[string]string{},
+			expectedLabels: map[string]string{},
+			wantErr:        true,
+		},
+		"Description formatted as json should return a map": {
+			description: `{"test": "test"}`,
+			// Label needs to be initialized to an empty map, otherwise the underlying method to write data to it will fail
+			labels:         map[string]string{},
+			expectedLabels: map[string]string{"test": "test"},
+			wantErr:        false,
+		},
+		"Description formatted as json with multiple keys should return a map": {
+			description: `{"kubernetes.io/created-for/pv/name":"pvc-32613356-4cee-481d-902f-daa7223d14ab","kubernetes.io/created-for/pvc/name":"prometheus-server-data-prometheus-0","kubernetes.io/created-for/pvc/namespace":"prometheus"}`,
+			// Label needs to be initialized to an empty map, otherwise the underlying method to write data to it will fail
+			labels: map[string]string{},
+			expectedLabels: map[string]string{
+				"kubernetes.io/created-for/pv/name":       "pvc-32613356-4cee-481d-902f-daa7223d14ab",
+				"kubernetes.io/created-for/pvc/name":      "prometheus-server-data-prometheus-0",
+				"kubernetes.io/created-for/pvc/namespace": "prometheus",
+			},
+			wantErr: false,
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if err := extractLabelsFromDesc(tt.description, tt.labels); (err != nil) != tt.wantErr {
+				t.Errorf("extractLabelsFromDesc() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			require.Equal(t, tt.expectedLabels, tt.labels)
+		})
+	}
+}
+
+func Test_getNamespaceFromDisk(t *testing.T) {
+	tests := map[string]struct {
+		disk *Disk
+		want string
+	}{
+		"Empty description should return an empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Description: "",
+			}, ""),
+			want: "",
+		},
+		"Description not formatted as json should return an empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Description: "test",
+			}, ""),
+			want: "",
+		},
+		"Description formatted as json with multiple keys should return a namespace": {
+			disk: NewDisk(&computev1.Disk{
+				Description: `{"kubernetes.io/created-for/pv/name":"pvc-32613356-4cee-481d-902f-daa7223d14ab","kubernetes.io/created-for/pvc/name":"prometheus","kubernetes.io/created-for/pvc/namespace":"prometheus"}`,
+			}, ""),
+			want: "prometheus",
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := tt.disk.Namespace(); got != tt.want {
+				t.Errorf("getNamespaceFromDisk() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getRegionFromDisk(t *testing.T) {
+	tests := map[string]struct {
+		disk *Disk
+		want string
+	}{
+		"Empty zone should return an empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Zone: "",
+			}, ""),
+			want: "",
+		},
+		"Zone formatted as a path should return the region": {
+			disk: NewDisk(&computev1.Disk{
+				Zone: "projects/123/zones/us-central1-a",
+			}, ""),
+			want: "us-central1",
+		},
+		"Disk with zone as label should return the region parsed properly": {
+			disk: NewDisk(&computev1.Disk{
+				Labels: map[string]string{
+					compute.GkeRegionLabel: "us-central1-f",
+				},
+			}, ""),
+			want: "us-central1",
+		},
+		"Disk with a label doesn't belong to a specific zone should return the full label": {
+			disk: NewDisk(&computev1.Disk{
+				Labels: map[string]string{
+					compute.GkeRegionLabel: "us-central1",
+				},
+			}, ""),
+			want: "us-central1",
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := tt.disk.Region(); got != tt.want {
+				t.Errorf("getRegionFromDisk() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getNameFromDisk(t *testing.T) {
+	tests := map[string]struct {
+		disk *Disk
+		want string
+	}{
+		"Empty description should return an empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Description: "",
+			}, ""),
+			want: "",
+		},
+		"Description not formatted as json should return an empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Description: "test",
+			}, ""),
+			want: "",
+		},
+		"Description not formatted as json should return the disks name": {
+			disk: NewDisk(&computev1.Disk{
+				Description: "test",
+				Name:        "testing123",
+			}, ""),
+			want: "testing123",
+		},
+		"Description formatted as json with multiple keys should return the name": {
+			disk: NewDisk(&computev1.Disk{
+				Description: `{"kubernetes.io/created-for/pv/name":"pvc-32613356-4cee-481d-902f-daa7223d14ab","kubernetes.io/created-for/pvc/name":"prometheus","kubernetes.io/created-for/pvc/namespace":"prometheus"}`,
+			}, ""),
+			want: "pvc-32613356-4cee-481d-902f-daa7223d14ab",
+		},
+		"Description formatted as json with one key should return the name": {
+			disk: NewDisk(&computev1.Disk{
+				Description: `{"kubernetes.io-created-for/pv-name":"pvc-32613356-4cee-481d-902f-daa7223d14ab"}`,
+			}, ""),
+			want: "pvc-32613356-4cee-481d-902f-daa7223d14ab",
+		},
+		"Description formatted as json with multiple wrong keys should return empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Description: `{"kubernetes.io/created-for/pvc/name":"prometheus","kubernetes.io/created-for/pvc/namespace":"prometheus"}`,
+			}, ""),
+			want: "",
+		},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := tt.disk.Name(); got != tt.want {
+				t.Errorf("getNameFromDisk() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getStorageClassFromDisk(t *testing.T) {
+	tests := map[string]struct {
+		disk *Disk
+		want string
+	}{
+		"Empty Type should return an empty string": {
+			disk: NewDisk(&computev1.Disk{
+				Type: "",
+			}, ""),
+			want: "",
+		},
+		"Type formatted as a path should return the storage class": {
+			disk: NewDisk(&computev1.Disk{
+				Type: "projects/123/zones/us-central1-a/diskTypes/pd-standard",
+			}, ""),
+			want: "pd-standard",
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := test.disk.StorageClass(); got != test.want {
+				t.Errorf("getStorageClassFromDisk() = %v, want %v", got, test.want)
+			}
+		})
+	}
+}
+
+func Test_DiskType(t *testing.T) {
+	tests := map[string]struct {
+		disk *Disk
+		want string
+	}{
+		"Disk with no disk type returns default value": {
+			disk: NewDisk(&computev1.Disk{}, ""),
+			want: "persistent_volume",
+		},
+		"Disk with a boot disk label returns boot_disk": {
+			disk: NewDisk(&computev1.Disk{
+				Labels: map[string]string{
+					BootDiskLabel: "true",
+				},
+			}, ""),
+			want: "boot_disk",
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			if got := test.disk.DiskType(); got != test.want {
+				t.Errorf("DiskType() = %v, want %v", got, test.want)
+			}
+		})
+	}
+}