From f0b03157dc8300c753a227a76c68a55bd03ac637 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Thu, 23 Apr 2020 14:01:39 -0700 Subject: [PATCH 1/8] Add persistent volume claim to raft database --- pkg/apis/storage/v1beta1/storageclass.go | 3 ++ pkg/controller/reconciler.go | 62 ++++++++++++++++-------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/pkg/apis/storage/v1beta1/storageclass.go b/pkg/apis/storage/v1beta1/storageclass.go index bcd2755..c27d449 100644 --- a/pkg/apis/storage/v1beta1/storageclass.go +++ b/pkg/apis/storage/v1beta1/storageclass.go @@ -41,6 +41,9 @@ type RaftStorageClassSpec struct { // ImagePullPolicy is the pull policy to apply ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + + // VolumeClaim + VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaim,omitempty"` } // +genclient diff --git a/pkg/controller/reconciler.go b/pkg/controller/reconciler.go index 8afb95a..518db40 100644 --- a/pkg/controller/reconciler.go +++ b/pkg/controller/reconciler.go @@ -262,6 +262,34 @@ func (r *Reconciler) reconcileStatefulSet(database *v1beta3.Database, storage *v return err } +func newDataVolume(database *v1beta3.Database, storage *v1beta1.RaftStorageClass, cluster int) corev1.Volume { + dataVolume := corev1.Volume{ + Name: dataVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + return dataVolume + +} + +func newConfigVolume(database *v1beta3.Database, storage *v1beta1.RaftStorageClass, cluster int) corev1.Volume { + configVolume := corev1.Volume{ + + Name: configVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: getClusterName(database, cluster), + }, + }, + }, + } + return configVolume + +} + func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1.RaftStorageClass, cluster int) error { log.Info("Creating raft replicas", "Name", database.Name, "Namespace", database.Namespace) @@ -274,6 +302,18 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 pullPolicy = corev1.PullIfNotPresent } + volumes := []corev1.Volume{ + newConfigVolume(database, storage, cluster), + } + + log.Info("volume Claim:", "template", storage.Spec) + log.Info("image", "name", storage.Spec.Image) + + if storage.Spec.VolumeClaimTemplates == nil { + log.Info("here") + volumes = append(volumes, newDataVolume(database, storage, cluster)) + } + set := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: getClusterName(database, cluster), @@ -289,7 +329,8 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ Type: appsv1.RollingUpdateStatefulSetStrategyType, }, - PodManagementPolicy: appsv1.ParallelPodManagement, + PodManagementPolicy: appsv1.ParallelPodManagement, + VolumeClaimTemplates: storage.Spec.VolumeClaimTemplates, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: newClusterLabels(database, cluster), @@ -356,24 +397,7 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 }, }, }, - Volumes: []corev1.Volume{ - { - Name: configVolume, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: getClusterName(database, cluster), - }, - }, - }, - }, - { - Name: dataVolume, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, + Volumes: volumes, }, }, }, From 5b6ed641c67f5746b919abf1c4f195a28bcea891 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Thu, 23 Apr 2020 19:52:02 -0700 Subject: [PATCH 2/8] remove debug statements --- pkg/apis/storage/v1beta1/storageclass.go | 2 +- pkg/controller/reconciler.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/apis/storage/v1beta1/storageclass.go b/pkg/apis/storage/v1beta1/storageclass.go index c27d449..d498da6 100644 --- a/pkg/apis/storage/v1beta1/storageclass.go +++ b/pkg/apis/storage/v1beta1/storageclass.go @@ -43,7 +43,7 @@ type RaftStorageClassSpec struct { ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` // VolumeClaim - VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaim,omitempty"` + VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` } // +genclient diff --git a/pkg/controller/reconciler.go b/pkg/controller/reconciler.go index 518db40..cbda50e 100644 --- a/pkg/controller/reconciler.go +++ b/pkg/controller/reconciler.go @@ -306,11 +306,8 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 newConfigVolume(database, storage, cluster), } - log.Info("volume Claim:", "template", storage.Spec) - log.Info("image", "name", storage.Spec.Image) - if storage.Spec.VolumeClaimTemplates == nil { - log.Info("here") + volumes = append(volumes, newDataVolume(database, storage, cluster)) } From d0ad7e0a7fb88f77b5b6a27178d9287593456359 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Thu, 23 Apr 2020 21:24:46 -0700 Subject: [PATCH 3/8] Update raft storage controller --- deploy/raft-storage-controller.yaml | 86 ++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/deploy/raft-storage-controller.yaml b/deploy/raft-storage-controller.yaml index ada2682..b8b1b78 100644 --- a/deploy/raft-storage-controller.yaml +++ b/deploy/raft-storage-controller.yaml @@ -31,14 +31,98 @@ spec: type: integer min: 1 max: 9 + default: 1 partitionsPerCluster: type: integer min: 1 max: 1024 default: 1 + volumeClaimTemplates: + type: array + "items": { + "type": object, + "required": [ + "metadata", + "spec" + ], + "properties": { + "metadata": { + "$id": "#/items/properties/metadata", + "type": "object", + "default": {}, + "required": [ + "name" + ], + "properties": { + "name": { + "$id": "#/items/properties/metadata/properties/name", + "type": "string", + "default": "", + + } + } + }, + "spec": { + "$id": "#/items/properties/spec", + "type": "object", + "default": {}, + "required": [ + "accessModes", + "storageClassName", + "resources" + ], + "properties": { + "accessModes": { + "$id": "#/items/properties/spec/properties/accessModes", + "type": "array", + "default": [], + + "items": { + "$id": "#/items/properties/spec/properties/accessModes/items", + "type": "string", + "default": "", + + } + }, + "storageClassName": { + "$id": "#/items/properties/spec/properties/storageClassName", + "type": "string", + "default": "", + + }, + "resources": { + "$id": "#/items/properties/spec/properties/resources", + "type": "object", + "default": {}, + "required": [ + "requests" + ], + "properties": { + "requests": { + "$id": "#/items/properties/spec/properties/resources/properties/requests", + "type": "object", + "default": {}, + "required": [ + "storage" + ], + "properties": { + "storage": { + "$id": "#/items/properties/spec/properties/resources/properties/requests/properties/storage", + "type": "string", + "default": "", + + } + } + } + } + } + } + } + } + } --- -apiVersion: cloud.atomix.io/v1beta2 +apiVersion: cloud.atomix.io/v1beta3 kind: StorageController metadata: name: raft From 7a60b9aadb812f685f5587cd853502aa20040610 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Sat, 25 Apr 2020 18:31:31 -0700 Subject: [PATCH 4/8] Update raft storage controller manifest --- deploy/raft-storage-controller.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deploy/raft-storage-controller.yaml b/deploy/raft-storage-controller.yaml index b8b1b78..5143ecf 100644 --- a/deploy/raft-storage-controller.yaml +++ b/deploy/raft-storage-controller.yaml @@ -29,13 +29,9 @@ spec: type: string replicas: type: integer - min: 1 - max: 9 default: 1 partitionsPerCluster: type: integer - min: 1 - max: 1024 default: 1 volumeClaimTemplates: type: array From efaaa2c54e0317abbc7751a671cf226ce0db04f9 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Sat, 25 Apr 2020 19:41:04 -0700 Subject: [PATCH 5/8] Update examples --- deploy/examples/raft-storage.yaml | 39 +++++++++++++++++------------ deploy/raft-storage-controller.yaml | 15 +++-------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/deploy/examples/raft-storage.yaml b/deploy/examples/raft-storage.yaml index d25e372..8b7c5e8 100644 --- a/deploy/examples/raft-storage.yaml +++ b/deploy/examples/raft-storage.yaml @@ -1,26 +1,33 @@ apiVersion: storage.cloud.atomix.io/v1beta1 kind: RaftStorageClass metadata: - name: raft - labels: - app: raft + name: raft-database + labels: + app: raft-database spec: - image: atomix/raft-replica:latest + image: atomix/raft-storage-node:v0.1.0 imagePullPolicy: IfNotPresent replicas: 1 + partitionsPerCluster: 3 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "standard" + resources: + requests: + storage: 1Gi --- -apiVersion: cloud.atomix.io/v1beta2 +apiVersion: cloud.atomix.io/v1beta3 kind: Database metadata: - name: raft + name: raft-database spec: - clusters: 3 - template: - spec: - partitions: 10 - storage: - group: storage.cloud.atomix.io - version: v1beta1 - kind: RaftStorageClass - name: raft - namespace: kube-system + partitions: 1 + storageClass: + group: storage.cloud.atomix.io + version: v1beta1 + kind: RaftStorageClass + name: raft-database + namespace: default diff --git a/deploy/raft-storage-controller.yaml b/deploy/raft-storage-controller.yaml index 5143ecf..011fee7 100644 --- a/deploy/raft-storage-controller.yaml +++ b/deploy/raft-storage-controller.yaml @@ -118,15 +118,6 @@ spec: } --- -apiVersion: cloud.atomix.io/v1beta3 -kind: StorageController -metadata: - name: raft -spec: - group: storage.cloud.atomix.io - version: v1beta1 - kind: RaftStorageClass ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -177,7 +168,7 @@ metadata: subjects: - kind: ServiceAccount name: raft-storage-controller - namespace: kube-system + namespace: default roleRef: kind: ClusterRole name: raft-storage-controller @@ -187,13 +178,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: raft-storage-controller - namespace: kube-system + namespace: default --- apiVersion: apps/v1 kind: Deployment metadata: name: raft-storage-controller - namespace: kube-system + namespace: default spec: replicas: 1 selector: From 9ad46d31920f41a23c2982b1a993c1a1b5bfbb89 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Sat, 25 Apr 2020 22:09:16 -0700 Subject: [PATCH 6/8] Change namespace to kube-system --- deploy/examples/raft-storage.yaml | 2 +- deploy/raft-storage-controller.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/examples/raft-storage.yaml b/deploy/examples/raft-storage.yaml index 8b7c5e8..92b3798 100644 --- a/deploy/examples/raft-storage.yaml +++ b/deploy/examples/raft-storage.yaml @@ -30,4 +30,4 @@ spec: version: v1beta1 kind: RaftStorageClass name: raft-database - namespace: default + namespace: kube-system diff --git a/deploy/raft-storage-controller.yaml b/deploy/raft-storage-controller.yaml index 011fee7..e223a02 100644 --- a/deploy/raft-storage-controller.yaml +++ b/deploy/raft-storage-controller.yaml @@ -168,7 +168,7 @@ metadata: subjects: - kind: ServiceAccount name: raft-storage-controller - namespace: default + namespace: kube-system roleRef: kind: ClusterRole name: raft-storage-controller @@ -178,13 +178,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: raft-storage-controller - namespace: default + namespace: kube-system --- apiVersion: apps/v1 kind: Deployment metadata: name: raft-storage-controller - namespace: default + namespace: kube-system spec: replicas: 1 selector: From 7c97b027e0dd6d97e18625ecc8fe3405db2be8f7 Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Mon, 27 Apr 2020 00:23:56 -0700 Subject: [PATCH 7/8] New design of api --- deploy/examples/raft-storage.yaml | 8 +- deploy/raft-storage-controller.yaml | 139 ++++++++++------------- pkg/apis/storage/v1beta1/storageclass.go | 4 +- pkg/apis/storage/v1beta1/volumeclaim.go | 24 ++++ pkg/controller/reconciler.go | 41 +++++-- 5 files changed, 116 insertions(+), 100 deletions(-) create mode 100644 pkg/apis/storage/v1beta1/volumeclaim.go diff --git a/deploy/examples/raft-storage.yaml b/deploy/examples/raft-storage.yaml index 92b3798..b1b6d78 100644 --- a/deploy/examples/raft-storage.yaml +++ b/deploy/examples/raft-storage.yaml @@ -6,18 +6,18 @@ metadata: app: raft-database spec: image: atomix/raft-storage-node:v0.1.0 - imagePullPolicy: IfNotPresent replicas: 1 partitionsPerCluster: 3 volumeClaimTemplates: - - metadata: - name: data + data: + metadata: + name: "test" spec: accessModes: [ "ReadWriteOnce" ] storageClassName: "standard" resources: requests: - storage: 1Gi + storage: "1Gi" --- apiVersion: cloud.atomix.io/v1beta3 kind: Database diff --git a/deploy/raft-storage-controller.yaml b/deploy/raft-storage-controller.yaml index e223a02..ac52a69 100644 --- a/deploy/raft-storage-controller.yaml +++ b/deploy/raft-storage-controller.yaml @@ -29,93 +29,68 @@ spec: type: string replicas: type: integer + min: 1 + max: 9 default: 1 partitionsPerCluster: type: integer + min: 1 + max: 1024 default: 1 volumeClaimTemplates: - type: array - "items": { - "type": object, - "required": [ - "metadata", - "spec" - ], - "properties": { - "metadata": { - "$id": "#/items/properties/metadata", - "type": "object", - "default": {}, - "required": [ - "name" - ], - "properties": { - "name": { - "$id": "#/items/properties/metadata/properties/name", - "type": "string", - "default": "", - - } - } - }, - "spec": { - "$id": "#/items/properties/spec", - "type": "object", - "default": {}, - "required": [ - "accessModes", - "storageClassName", - "resources" - ], - "properties": { - "accessModes": { - "$id": "#/items/properties/spec/properties/accessModes", - "type": "array", - "default": [], - - "items": { - "$id": "#/items/properties/spec/properties/accessModes/items", - "type": "string", - "default": "", - - } - }, - "storageClassName": { - "$id": "#/items/properties/spec/properties/storageClassName", - "type": "string", - "default": "", - - }, - "resources": { - "$id": "#/items/properties/spec/properties/resources", - "type": "object", - "default": {}, - "required": [ - "requests" - ], - "properties": { - "requests": { - "$id": "#/items/properties/spec/properties/resources/properties/requests", - "type": "object", - "default": {}, - "required": [ - "storage" - ], - "properties": { - "storage": { - "$id": "#/items/properties/spec/properties/resources/properties/requests/properties/storage", - "type": "string", - "default": "", - - } - } - } - } - } - } - } - } - } + type: object + default: {} + required: + - data + properties: + data: + type: object + default: {} + required: + - metadata + - spec + properties: + metadata: + type: object + default: {} + required: + - name + properties: + name: + type: string + default: '' + spec: + type: object + default: {} + required: + - accessModes + - storageClassName + - resources + properties: + accessModes: + type: array + default: [] + items: + type: string + default: '' + storageClassName: + type: string + default: '' + resources: + type: object + default: {} + required: + - requests + properties: + requests: + type: object + default: {} + required: + - storage + properties: + storage: + type: string + default: '' --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/pkg/apis/storage/v1beta1/storageclass.go b/pkg/apis/storage/v1beta1/storageclass.go index d498da6..3dd546c 100644 --- a/pkg/apis/storage/v1beta1/storageclass.go +++ b/pkg/apis/storage/v1beta1/storageclass.go @@ -42,8 +42,8 @@ type RaftStorageClassSpec struct { // ImagePullPolicy is the pull policy to apply ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - // VolumeClaim - VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` + // VolumeClaimTemplates are volume claim templates to define persistent volumes + VolumeClaimTemplates VolumeClaimTemplates `json:"volumeClaimTemplates,omitempty"` } // +genclient diff --git a/pkg/apis/storage/v1beta1/volumeclaim.go b/pkg/apis/storage/v1beta1/volumeclaim.go new file mode 100644 index 0000000..a17317e --- /dev/null +++ b/pkg/apis/storage/v1beta1/volumeclaim.go @@ -0,0 +1,24 @@ +// Copyright 2020-present Open Networking Foundation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1beta1 + +import ( + corev1 "k8s.io/api/core/v1" +) + +// VolumeClaimTemplates k8s persistent volume claims +type VolumeClaimTemplates struct { + Data *corev1.PersistentVolumeClaim `json:"data,omitempty"` +} diff --git a/pkg/controller/reconciler.go b/pkg/controller/reconciler.go index cbda50e..0d0337d 100644 --- a/pkg/controller/reconciler.go +++ b/pkg/controller/reconciler.go @@ -306,9 +306,35 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 newConfigVolume(database, storage, cluster), } - if storage.Spec.VolumeClaimTemplates == nil { + var volumeClaimTemplates []corev1.PersistentVolumeClaim + var volumeMounts []corev1.VolumeMount + configVolumeMount := corev1.VolumeMount{ + Name: configVolume, + MountPath: configPath, + } + + volumeMounts = append(volumeMounts, configVolumeMount) + if storage.Spec.VolumeClaimTemplates.Data.Name == "" { + dataVolumeMount := corev1.VolumeMount{ + Name: dataVolume, + MountPath: dataPath, + } volumes = append(volumes, newDataVolume(database, storage, cluster)) + volumeMounts = append(volumeMounts, dataVolumeMount) + } else { + dataVolumeMount := corev1.VolumeMount{ + Name: storage.Spec.VolumeClaimTemplates.Data.Name, + MountPath: dataPath, + } + volumeMounts = append(volumeMounts, dataVolumeMount) + persistentVolumeClaim := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: storage.Spec.VolumeClaimTemplates.Data.Name, + }, + Spec: storage.Spec.VolumeClaimTemplates.Data.Spec, + } + volumeClaimTemplates = append(volumeClaimTemplates, persistentVolumeClaim) } set := &appsv1.StatefulSet{ @@ -327,7 +353,7 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 Type: appsv1.RollingUpdateStatefulSetStrategyType, }, PodManagementPolicy: appsv1.ParallelPodManagement, - VolumeClaimTemplates: storage.Spec.VolumeClaimTemplates, + VolumeClaimTemplates: volumeClaimTemplates, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: newClusterLabels(database, cluster), @@ -382,16 +408,7 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 InitialDelaySeconds: 60, TimeoutSeconds: 10, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: dataVolume, - MountPath: dataPath, - }, - { - Name: configVolume, - MountPath: configPath, - }, - }, + VolumeMounts: volumeMounts, }, }, Volumes: volumes, From f6203b2951f3e4f1ef6d7998f003afa95f60919a Mon Sep 17 00:00:00 2001 From: adibrastegarnia Date: Mon, 27 Apr 2020 11:32:04 -0700 Subject: [PATCH 8/8] Update manifest --- deploy/raft-storage-controller.yaml | 26 ++++---------------------- pkg/controller/reconciler.go | 2 +- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/deploy/raft-storage-controller.yaml b/deploy/raft-storage-controller.yaml index ac52a69..941637d 100644 --- a/deploy/raft-storage-controller.yaml +++ b/deploy/raft-storage-controller.yaml @@ -29,47 +29,35 @@ spec: type: string replicas: type: integer - min: 1 - max: 9 + minimum: 1 + maximum: 9 default: 1 partitionsPerCluster: type: integer - min: 1 - max: 1024 + minimum: 1 + maximum: 1024 default: 1 volumeClaimTemplates: type: object - default: {} required: - data properties: data: type: object - default: {} required: - - metadata - spec properties: metadata: type: object - default: {} - required: - - name properties: name: type: string default: '' spec: type: object - default: {} - required: - - accessModes - - storageClassName - - resources properties: accessModes: type: array - default: [] items: type: string default: '' @@ -78,15 +66,9 @@ spec: default: '' resources: type: object - default: {} - required: - - requests properties: requests: type: object - default: {} - required: - - storage properties: storage: type: string diff --git a/pkg/controller/reconciler.go b/pkg/controller/reconciler.go index 0d0337d..5d0e739 100644 --- a/pkg/controller/reconciler.go +++ b/pkg/controller/reconciler.go @@ -315,7 +315,7 @@ func (r *Reconciler) addStatefulSet(database *v1beta3.Database, storage *v1beta1 } volumeMounts = append(volumeMounts, configVolumeMount) - if storage.Spec.VolumeClaimTemplates.Data.Name == "" { + if storage.Spec.VolumeClaimTemplates.Data == nil { dataVolumeMount := corev1.VolumeMount{ Name: dataVolume, MountPath: dataPath,