diff --git a/Makefile b/Makefile index 9e68ee19..934242a4 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false,maxDescLen=0" +CRD_OPTIONS ?= "crd:generateEmbeddedObjectMeta=false,maxDescLen=0" # Set build symbols LDFLAGS = $(if $(DEBUGGER),,-s -w) $(shell ./hack/version.sh) -DOCKER_REGISTRY ?= localhost:5000 +DOCKER_REGISTRY ?= docker.io DOCKER_REPO ?= ${DOCKER_REGISTRY}/vesoft -IMAGE_TAG ?= latest +IMAGE_TAG ?= v1.4.0 + +CHARTS_VERSION ?= 1.4.0 export GO111MODULE := on GOOS := $(if $(GOOS),$(GOOS),linux) @@ -81,7 +83,7 @@ test: manifests generate check ## Run tests. source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./pkg/... ./apis/... -coverprofile cover.out ##@ e2e -e2e: $(GOBIN)/ginkgo $(GOBIN)/kind helm +e2e: $(GOBIN)/ginkgo $(GOBIN)/kind PATH="${GOBIN}:${PATH}" ./hack/e2e.sh ##@ Build @@ -90,12 +92,12 @@ build: generate check ## Build binary. $(GO_BUILD) -ldflags '$(LDFLAGS)' -o images/nebula-operator/bin/controller-manager cmd/controller-manager/main.go $(GO_BUILD) -ldflags '$(LDFLAGS)' -o images/nebula-operator/bin/scheduler cmd/scheduler/main.go -build-helm: helm - helm repo index charts --url https://vesoft-inc.github.io/nebula-operator/charts - helm package charts/nebula-operator - helm package charts/nebula-cluster +helm-charts: + cp config/crd/bases/*.yaml charts/nebula-operator/crds/ + helm package charts/nebula-operator --version $(CHARTS_VERSION) + helm package charts/nebula-cluster --version $(CHARTS_VERSION) mv nebula-operator-*.tgz nebula-cluster-*.tgz charts/ - cp config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml charts/nebula-operator/crds/nebulacluster.yaml + helm repo index charts/ --url https://github.com/vesoft-inc/nebula-operator/releases/download/v$(CHARTS_VERSION) run: run-controller-manager @@ -108,7 +110,7 @@ run-scheduler: manifests generate check docker-build: build ## Build docker images. docker build -t "${DOCKER_REPO}/nebula-operator:${IMAGE_TAG}" images/nebula-operator/ -docker-push: docker-build ## Push docker images. +docker-push: ## Push docker images. docker push "${DOCKER_REPO}/nebula-operator:${IMAGE_TAG}" docker-manifest: ## Build all docker images and push it to registry. @@ -138,8 +140,7 @@ tools: $(GOBIN)/goimports \ $(GOBIN)/controller-gen \ $(GOBIN)/kustomize \ $(GOBIN)/ginkgo \ - $(GOBIN)/kind \ - helm + $(GOBIN)/kind $(GOBIN)/goimports: $(call go-get-tool,$(GOBIN)/goimports,golang.org/x/tools/cmd/goimports) @@ -156,26 +157,20 @@ $(GOBIN)/gofumpt: $(GOBIN)/golangci-lint: @[ -f $(GOBIN)/golangci-lint ] || { \ set -e ;\ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOBIN) v1.40.1 ;\ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOBIN) v1.51.1 ;\ } $(GOBIN)/controller-gen: - $(call go-get-tool,$(GOBIN)/controller-gen,sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) + $(call go-get-tool,$(GOBIN)/controller-gen,sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3) $(GOBIN)/kustomize: - $(call go-get-tool,$(GOBIN)/kustomize,sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) + $(call go-get-tool,$(GOBIN)/kustomize,sigs.k8s.io/kustomize/kustomize/v4@v4.5.7) $(GOBIN)/ginkgo: - $(call go-get-tool,$(GOBIN)/kustomize,github.com/onsi/ginkgo/ginkgo@v1.16.2) + $(call go-get-tool,$(GOBIN)/ginkgo,github.com/onsi/ginkgo/ginkgo@v1.16.5) $(GOBIN)/kind: - $(call go-get-tool,$(GOBIN)/kustomize,sigs.k8s.io/kind@v0.10.0) - -helm: - @[ -f /usr/local/bin/helm ] || { \ - set -e ;\ - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash ;\ - } + $(call go-get-tool,$(GOBIN)/kind,sigs.k8s.io/kind@v0.17.0) # go-get-tool will 'go get' any package $2 and install it to $1. define go-get-tool @@ -186,6 +181,7 @@ cd $$TMP_DIR ;\ go mod init tmp ;\ echo "Downloading $(2)" ;\ go get $(2) ;\ +go install $(2) ;\ rm -rf $$TMP_DIR ;\ } endef diff --git a/README.md b/README.md index d2c9d029..a34992f0 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,31 @@ # Nebula Operator -Nebula Operator manages [NebulaGraph](https://github.com/vesoft-inc/nebula) clusters on [Kubernetes](https://kubernetes.io) and automates tasks related to operating a NebulaGraph cluster. -It evolved from [NebulaGraph Cloud Service](https://www.nebula-cloud.io/), makes NebulaGraph a truly cloud-native database. +Nebula Operator manages [NebulaGraph](https://github.com/vesoft-inc/nebula) clusters +on [Kubernetes](https://kubernetes.io) and automates tasks related to operating a NebulaGraph cluster. +It evolved from [NebulaGraph Cloud Service](https://www.nebula-cloud.io/), makes NebulaGraph a truly cloud-native +database. ## Quick Start + - [Install Nebula Operator](#install-nebula-operator) - [Create and Destroy](#create-and-destroy-a-nebula-cluster) - [Resize](#resize-a-nebula-cluster) - [Rolling Upgrade](#upgrade-a-nebula-cluster) +- [Backup and Restore](#backup-and-restore-a-nebula-cluster) - [Failover](#failover) ### install nebula operator + See [install/uninstall nebula operator](doc/user/install_guide.md) . ### Create and destroy a nebula cluster + ```bash $ kubectl create -f config/samples/apps_v1alpha1_nebulacluster.yaml ``` + A none ha-mode nebula cluster will be created. + ```bash $ kubectl get pods -l app.kubernetes.io/cluster=nebula NAME READY STATUS RESTARTS AGE @@ -29,7 +37,8 @@ nebula-storaged-2 1/1 Running 0 1m ``` See [client service](doc/user/client_service.md) for how to access nebula clusters created by the operator. -If you are working with [kubeadm locally](https://kubernetes.io/docs/reference/setup-tools/kubeadm/), create a nodePort service and test that nebula is responding: +If you are working with [kubeadm locally](https://kubernetes.io/docs/reference/setup-tools/kubeadm/), create a nodePort +service and test that nebula is responding: ```bash $ kubectl create -f config/samples/graphd-nodeport-service.yaml @@ -48,6 +57,7 @@ $ kubectl delete -f config/samples/apps_v1alpha1_nebulacluster.yaml ``` ### Resize a nebula cluster + Create a nebula cluster: ```bash @@ -68,7 +78,7 @@ Modify the file and change `replicas` from 3 to 5. memory: "1Gi" replicas: 5 image: vesoft/nebula-storaged - version: v3.3.0 + version: v3.4.0 storageClaim: resources: requests: @@ -96,7 +106,8 @@ nebula-storaged-3 1/1 Running 0 5m nebula-storaged-4 1/1 Running 0 5m ``` -Similarly we can decrease the size of the cluster from 5 back to 3 by changing the replicas field again and reapplying the change. +Similarly, we can decrease the size of the cluster from 5 back to 3 by changing the replicas field again and reapplying +the change. ```yaml storaged: @@ -109,7 +120,7 @@ Similarly we can decrease the size of the cluster from 5 back to 3 by changing t memory: "1Gi" replicas: 3 image: vesoft/nebula-storaged - version: v3.3.0 + version: v3.4.0 storageClaim: resources: requests: @@ -132,6 +143,7 @@ nebula-storaged-2 1/1 Running 0 10m In addition, you can [Install Nebula Cluster with helm](doc/user/nebula_cluster_helm_guide.md). ### Upgrade a nebula cluster + Create a nebula cluster with the version specified (v3.0.0): ```bash @@ -154,7 +166,7 @@ $ kubectl get pods -l app.kubernetes.io/cluster=nebula -o jsonpath="{.items[*]. 3 vesoft/nebula-storaged:v3.0.0 ``` -Now modify the file `apps_v1alpha1_nebulacluster.yaml` and change the `version` from v3.0.0 to v3.2.0: +Now modify the file `apps_v1alpha1_nebulacluster.yaml` and change the `version` from v3.0.0 to v3.4.0: Apply the version change to the cluster CR: @@ -162,17 +174,27 @@ Apply the version change to the cluster CR: $ kubectl apply -f config/samples/apps_v1alpha1_nebulacluster.yaml ``` -Wait 2 minutes. The container image version should be updated to v3.2.0: +Wait few minutes. The container image version should be updated to v3.4.0: ```bash $ kubectl get pods -l app.kubernetes.io/cluster=nebula -o jsonpath="{.items[*].spec.containers[*].image}" |tr -s '[[:space:]]' '\n' |sort |uniq -c - 1 vesoft/nebula-graphd:v3.3.0 - 1 vesoft/nebula-metad:v3.3.0 - 3 vesoft/nebula-storaged:v3.3.0 + 1 vesoft/nebula-graphd:v3.4.0 + 1 vesoft/nebula-metad:v3.4.0 + 3 vesoft/nebula-storaged:v3.4.0 ``` +**Warning:** + +Enterprise version doesn't support upgrade from v3.1.x or v3.2.x to v3.4.0 + +### Backup and Restore a nebula cluster + +See [backup/restore nebula cluster](doc/user/br_guide.md) . + ### Failover -If the minority of nebula components crash, the nebula operator will automatically recover the failure. Let's walk through this in the following steps. + +If the minority of nebula components crash, the nebula operator will automatically recover the failure. Let's walk +through this in the following steps. Create a nebula cluster: @@ -202,20 +224,22 @@ nebula-storaged-2 1/1 Running 0 19s Nebula Operator <-> NebulaGraph -| | NebulaGraph v2.5 | NebulaGraph v2.6 | NebulaGraph v3.0 | NebulaGraph v3.1 | NebulaGraph v3.2 | NebulaGraph v3.3 | -|-----------|------------------|------------------|------------------|------------------|------------------|------------------| -| `v0.8.0` | ✓ | - | - | - | - | - | -| `v0.9.0`* | ✓ | ✓ | - | - | - | - | -| `v1.0.0`* | - | - | ✓ | ✓ | ✓ | ✓ | -| `v1.1.0` | - | - | ✓ | ✓ | ✓ | ✓ | -| `v1.2.0` | - | - | ✓ | ✓ | ✓ | ✓ | -| `v1.3.0` | - | - | ✓ | ✓ | ✓ | ✓ | +| | NebulaGraph v2.5 | NebulaGraph v2.6 | NebulaGraph v3.0 | NebulaGraph v3.1 | NebulaGraph v3.2 | NebulaGraph v3.3 | NebulaGraph v3.4 | +|:----------|:-----------------|:-----------------|:-----------------|:-----------------|:-----------------|:-----------------|:-----------------| +| `v0.8.0` | ✓ | - | - | - | - | - | - | +| `v0.9.0`* | ✓ | ✓ | - | - | - | - | - | +| `v1.0.0`* | - | - | ✓ | ✓ | ✓ | ✓ | ✓ | +| `v1.1.0` | - | - | ✓ | ✓ | ✓ | ✓ | ✓ | +| `v1.2.0` | - | - | ✓ | ✓ | ✓ | ✓ | ✓ | +| `v1.3.0` | - | - | ✓ | ✓ | ✓ | ✓ | ✓ | +| `1.4.0` | - | - | ✓ | ✓ | ✓ | ✓ | ✓ | -Key: +**Note:** * `✓` Compatible. * `-` Not Compatible. -* `*` Please notice that the StorageClaim is split into LogVolumeClaim and DataVolumeClaim in crd. v0.9.0 can't forward compatible. +* `*` Please notice that the StorageClaim is split into LogVolumeClaim and DataVolumeClaim in crd. v0.9.0 can't forward + compatible. * v1.0.0 does not support storage auto scaling. * v1.x does not support upgrading from v2.x to v3.x. @@ -224,19 +248,24 @@ Key: Please refer to [FAQ.md](FAQ.md) ## Community + Feel free to reach out if you have any questions. The maintainers of this project are reachable via: - [Filing an issue](https://github.com/vesoft-inc/nebula-operator/issues) against this repo ## Contributing -Contributions are welcome and greatly appreciated. +Contributions are welcome and greatly appreciated. + - Start by some issues -- Submit Pull Requests to us. Please refer to [how-to-contribute](https://docs.nebula-graph.io/manual-EN/4.contributions/how-to-contribute/). +- Submit Pull Requests to us. Please refer + to [how-to-contribute](https://docs.nebula-graph.io/manual-EN/4.contributions/how-to-contribute/). ## Acknowledgements -nebula-operator refers to [tidb-operator](https://github.com/pingcap/tidb-operator). They have made a very good product. We have a similar architecture, although the product pattern is different from the application scenario, we would like to express our gratitude here. +nebula-operator refers to [tidb-operator](https://github.com/pingcap/tidb-operator). They have made a very good product. +We have a similar architecture, although the product pattern is different from the application scenario, we would like +to express our gratitude here. ## License diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 093cd904..00000000 --- a/ROADMAP.md +++ /dev/null @@ -1,33 +0,0 @@ -# Roadmap - -This document defines a high level roadmap for the nebula-operator development. - -**Note:** -The dates below should not be considered authoritative. - - -### 2021 - -#### Features - -- Integration with graph computing engine - -- Upgrade Cluster - - Backup before upgrading - - Rolling update Cluster to higher version - - Rollback if upgrade failed - -- Cluster AutoScaling - -#### Stability/Reliability - -- More testing - - 60%+ unit tests coverage - - e2e testing - -- Storage snapshot - - Before upgrading cluster - -- Failover in local storage scenario - - \ No newline at end of file diff --git a/apis/apps/v1alpha1/nebulacluster.go b/apis/apps/v1alpha1/nebulacluster.go index c5ebb027..459d6f62 100644 --- a/apis/apps/v1alpha1/nebulacluster.go +++ b/apis/apps/v1alpha1/nebulacluster.go @@ -56,6 +56,10 @@ func (nc *NebulaCluster) GetMetadEndpoints() []string { return nc.MetadComponent().GetHeadlessConnAddresses(MetadPortNameThrift) } +func (nc *NebulaCluster) GetStoragedEndpoints() []string { + return nc.StoragedComponent().GetHeadlessConnAddresses(StoragedPortNameThrift) +} + func (nc *NebulaCluster) GetClusterName() string { return nc.Name } @@ -88,3 +92,19 @@ func (nc *NebulaCluster) IsAutoBalanceEnabled() bool { } return *enabled } + +func (nc *NebulaCluster) IsForceUpdateEnabled() bool { + enabled := nc.Spec.Storaged.EnableForceUpdate + if enabled == nil { + return false + } + return *enabled +} + +func (nc *NebulaCluster) IsBREnabled() bool { + enabled := nc.Spec.EnableBR + if enabled == nil { + return false + } + return *enabled +} diff --git a/apis/apps/v1alpha1/nebulacluster_common.go b/apis/apps/v1alpha1/nebulacluster_common.go index 3e4e225c..73b5d15a 100644 --- a/apis/apps/v1alpha1/nebulacluster_common.go +++ b/apis/apps/v1alpha1/nebulacluster_common.go @@ -31,10 +31,19 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "github.com/vesoft-inc/nebula-operator/pkg/label" ) +const ( + AgentSidecarContainerName = "br-agent" + AgentInitContainerName = "br-init-agent" + DefaultAgentPortGRPC = 8888 + agentPortNameGRPC = "grpc" + defaultAgentImage = "vesoft/nebula-agent:latest" +) + func getComponentName(clusterName string, typ ComponentType) string { return fmt.Sprintf("%s-%s", clusterName, typ) } @@ -84,10 +93,6 @@ func getConnAddress(serviceFQDN string, port int32) string { return joinHostPort(serviceFQDN, port) } -func getPodConnAddress(podFQDN string, port int32) string { - return joinHostPort(podFQDN, port) -} - func getHeadlessConnAddresses(connAddress, componentName string, replicas int32, isHeadless bool) []string { if isHeadless { addresses := make([]string, 0, replicas) @@ -184,6 +189,54 @@ func parseStorageRequest(res corev1.ResourceList) (corev1.ResourceRequirements, }, nil } +func GenerateInitAgentContainer(c NebulaClusterComponentter) corev1.Container { + container := generateAgentContainer(c, true) + container.Name = AgentInitContainerName + + return container +} + +func generateAgentContainer(c NebulaClusterComponentter, init bool) corev1.Container { + componentType := c.Type().String() + metadAddr := c.GetNebulaCluster().GetMetadThriftConnAddress() + agentCmd := []string{"/bin/sh", "-ecx"} + if init { + agentCmd = append(agentCmd, "exec /usr/local/bin/agent"+ + fmt.Sprintf(" --agent=$(hostname).%s:%d", c.GetServiceFQDN(), DefaultAgentPortGRPC)+ + " --ratelimit=1073741824 --debug") + } else { + agentCmd = append(agentCmd, "sleep 30; exec /usr/local/bin/agent"+ + fmt.Sprintf(" --agent=$(hostname).%s:%d", c.GetServiceFQDN(), DefaultAgentPortGRPC)+ + " --meta="+metadAddr+ + " --ratelimit=1073741824 --debug") + } + + container := corev1.Container{ + Name: AgentSidecarContainerName, + Image: defaultAgentImage, + ImagePullPolicy: corev1.PullAlways, + Command: agentCmd, + Ports: []corev1.ContainerPort{ + { + Name: agentPortNameGRPC, + ContainerPort: int32(DefaultAgentPortGRPC), + }, + }, + } + + if c.Type() != GraphdComponentType { + container.VolumeMounts = []corev1.VolumeMount{ + { + Name: dataVolume(componentType), + MountPath: "/usr/local/nebula/data", + SubPath: "data", + }, + } + } + + return container +} + func generateContainers(c NebulaClusterComponentter, cm *corev1.ConfigMap) []corev1.Container { componentType := c.Type().String() nc := c.GetNebulaCluster() @@ -219,7 +272,7 @@ func generateContainers(c NebulaClusterComponentter, cm *corev1.ConfigMap) []cor ports := c.GenerateContainerPorts() - container := corev1.Container{ + baseContainer := corev1.Container{ Name: componentType, Image: c.GetImage(), Command: cmd, @@ -229,9 +282,9 @@ func generateContainers(c NebulaClusterComponentter, cm *corev1.ConfigMap) []cor } if c.ReadinessProbe() != nil { - container.ReadinessProbe = c.ReadinessProbe() + baseContainer.ReadinessProbe = c.ReadinessProbe() } else { - container.ReadinessProbe = &corev1.Probe{ + baseContainer.ReadinessProbe = &corev1.Probe{ Handler: corev1.Handler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/status", @@ -247,16 +300,22 @@ func generateContainers(c NebulaClusterComponentter, cm *corev1.ConfigMap) []cor resources := c.GetResources() if resources != nil { - container.Resources = *resources + baseContainer.Resources = *resources } imagePullPolicy := nc.Spec.ImagePullPolicy if imagePullPolicy != nil { - container.ImagePullPolicy = *imagePullPolicy + baseContainer.ImagePullPolicy = *imagePullPolicy } - containers = append(containers, container) - containers = append(containers, c.SidecarContainers()...) + containers = append(containers, baseContainer) + + if nc.IsBREnabled() { + agentContainer := generateAgentContainer(c, false) + containers = append(containers, agentContainer) + } + + containers = mergeSidecarContainers(containers, c.SidecarContainers()) return containers } @@ -288,11 +347,12 @@ func generateStatefulSet(c NebulaClusterComponentter, cm *corev1.ConfigMap, enab }, }) } - volumes = append(volumes, c.SidecarVolumes()...) + volumes = mergeVolumes(volumes, c.SidecarVolumes()) podSpec := corev1.PodSpec{ SchedulerName: nc.Spec.SchedulerName, NodeSelector: c.NodeSelector(), + InitContainers: c.InitContainers(), Containers: containers, Volumes: volumes, ImagePullSecrets: nc.Spec.ImagePullSecrets, @@ -491,3 +551,39 @@ func isStringMapExist(m map[string]string, key string) bool { _, exist := m[key] return exist } + +func mergeVolumes(original []corev1.Volume, additional []corev1.Volume) []corev1.Volume { + exists := sets.NewString() + for _, volume := range original { + exists.Insert(volume.Name) + } + + for _, volume := range additional { + if exists.Has(volume.Name) { + continue + } + original = append(original, volume) + exists.Insert(volume.Name) + } + + return original +} + +func mergeSidecarContainers(origins, injected []corev1.Container) []corev1.Container { + containersInPod := make(map[string]int) + for index, container := range origins { + containersInPod[container.Name] = index + } + + var appContainers []corev1.Container + for _, sidecar := range injected { + if index, ok := containersInPod[sidecar.Name]; ok { + origins[index] = sidecar + continue + } + appContainers = append(appContainers, sidecar) + } + + origins = append(origins, appContainers...) + return origins +} diff --git a/apis/apps/v1alpha1/nebulacluster_componentter.go b/apis/apps/v1alpha1/nebulacluster_componentter.go index b670e9a3..d1bb344c 100644 --- a/apis/apps/v1alpha1/nebulacluster_componentter.go +++ b/apis/apps/v1alpha1/nebulacluster_componentter.go @@ -17,8 +17,6 @@ limitations under the License. package v1alpha1 import ( - "fmt" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -44,6 +42,7 @@ type NebulaClusterComponentter interface { NodeSelector() map[string]string Affinity() *corev1.Affinity Tolerations() []corev1.Toleration + InitContainers() []corev1.Container SidecarContainers() []corev1.Container SidecarVolumes() []corev1.Volume ReadinessProbe() *corev1.Probe @@ -54,7 +53,6 @@ type NebulaClusterComponentter interface { GetPodFQDN(int32) string GetPort(string) int32 GetConnAddress(string) string - GetPodConnAddresses(portName string, ordinal int32) string GetHeadlessConnAddresses(string) []string IsReady() bool @@ -73,7 +71,6 @@ type NebulaClusterComponentter interface { // +k8s:deepcopy-gen=false type baseComponentter interface { - fmt.Stringer GraphdComponent() NebulaClusterComponentter MetadComponent() NebulaClusterComponentter StoragedComponent() NebulaClusterComponentter @@ -102,10 +99,6 @@ type baseComponent struct { typ ComponentType } -func (c *baseComponent) String() string { - return fmt.Sprintf("%s component for [%s/%s] NebulaCluster", c.Type(), c.GetNamespace(), c.GetClusterName()) -} - func (c *baseComponent) GraphdComponent() NebulaClusterComponentter { return newGraphdComponent(c.nc) } diff --git a/apis/apps/v1alpha1/nebulacluster_graphd.go b/apis/apps/v1alpha1/nebulacluster_graphd.go index 17eae4dd..18e32a2a 100644 --- a/apis/apps/v1alpha1/nebulacluster_graphd.go +++ b/apis/apps/v1alpha1/nebulacluster_graphd.go @@ -136,6 +136,10 @@ func (c *graphdComponent) Tolerations() []corev1.Toleration { return tolerations } +func (c *graphdComponent) InitContainers() []corev1.Container { + return c.nc.Spec.Graphd.PodSpec.InitContainers +} + func (c *graphdComponent) SidecarContainers() []corev1.Container { return c.nc.Spec.Graphd.PodSpec.SidecarContainers } @@ -179,10 +183,6 @@ func (c *graphdComponent) GetConnAddress(portName string) string { return getConnAddress(c.GetServiceFQDN(), c.GetPort(portName)) } -func (c *graphdComponent) GetPodConnAddresses(portName string, ordinal int32) string { - return getPodConnAddress(c.GetPodFQDN(ordinal), c.GetPort(portName)) -} - func (c *graphdComponent) GetHeadlessConnAddresses(portName string) []string { return getHeadlessConnAddresses( c.GetConnAddress(portName), diff --git a/apis/apps/v1alpha1/nebulacluster_metad.go b/apis/apps/v1alpha1/nebulacluster_metad.go index fe2c0af5..6ff3a844 100644 --- a/apis/apps/v1alpha1/nebulacluster_metad.go +++ b/apis/apps/v1alpha1/nebulacluster_metad.go @@ -147,6 +147,10 @@ func (c *metadComponent) Tolerations() []corev1.Toleration { return tolerations } +func (c *metadComponent) InitContainers() []corev1.Container { + return c.nc.Spec.Metad.PodSpec.InitContainers +} + func (c *metadComponent) SidecarContainers() []corev1.Container { return c.nc.Spec.Metad.PodSpec.SidecarContainers } @@ -190,10 +194,6 @@ func (c *metadComponent) GetConnAddress(portName string) string { return getConnAddress(c.GetServiceFQDN(), c.GetPort(portName)) } -func (c *metadComponent) GetPodConnAddresses(portName string, ordinal int32) string { - return getPodConnAddress(c.GetPodFQDN(ordinal), c.GetPort(portName)) -} - func (c *metadComponent) GetHeadlessConnAddresses(portName string) []string { return getHeadlessConnAddresses( c.GetConnAddress(portName), @@ -234,7 +234,8 @@ func (c *metadComponent) GenerateVolumeMounts() []corev1.VolumeMount { Name: logVolume(componentType), MountPath: "/usr/local/nebula/logs", SubPath: "logs", - }, { + }, + { Name: dataVolume(componentType), MountPath: "/usr/local/nebula/data", SubPath: "data", @@ -245,7 +246,7 @@ func (c *metadComponent) GenerateVolumeMounts() []corev1.VolumeMount { mounts = append(mounts, corev1.VolumeMount{ Name: "nebula-license", ReadOnly: true, - MountPath: "/usr/local/nebula/share/nebula.license", + MountPath: "/usr/local/nebula/nebula.license", SubPath: "nebula.license", }) } diff --git a/apis/apps/v1alpha1/nebulacluster_storaged.go b/apis/apps/v1alpha1/nebulacluster_storaged.go index 70efbfc5..72e2064a 100644 --- a/apis/apps/v1alpha1/nebulacluster_storaged.go +++ b/apis/apps/v1alpha1/nebulacluster_storaged.go @@ -155,6 +155,10 @@ func (c *storagedComponent) Tolerations() []corev1.Toleration { return tolerations } +func (c *storagedComponent) InitContainers() []corev1.Container { + return c.nc.Spec.Storaged.PodSpec.InitContainers +} + func (c *storagedComponent) SidecarContainers() []corev1.Container { return c.nc.Spec.Storaged.PodSpec.SidecarContainers } @@ -172,7 +176,7 @@ func (c *storagedComponent) IsHeadlessService() bool { } func (c *storagedComponent) GetServiceSpec() *ServiceSpec { - if c.nc.Spec.Storaged.Service != nil { + if c.nc.Spec.Storaged.Service == nil { return nil } return c.nc.Spec.Storaged.Service.DeepCopy() @@ -198,10 +202,6 @@ func (c *storagedComponent) GetConnAddress(portName string) string { return getConnAddress(c.GetServiceFQDN(), c.GetPort(portName)) } -func (c *storagedComponent) GetPodConnAddresses(portName string, ordinal int32) string { - return getPodConnAddress(c.GetPodFQDN(ordinal), c.GetPort(portName)) -} - func (c *storagedComponent) GetHeadlessConnAddresses(portName string) []string { return getHeadlessConnAddresses( c.GetConnAddress(portName), diff --git a/apis/apps/v1alpha1/nebulacluster_types.go b/apis/apps/v1alpha1/nebulacluster_types.go index 09a83b70..179e73b9 100644 --- a/apis/apps/v1alpha1/nebulacluster_types.go +++ b/apis/apps/v1alpha1/nebulacluster_types.go @@ -92,6 +92,13 @@ type NebulaClusterSpec struct { // UpdatePolicy indicates how pods should be updated // +optional UpdatePolicy string `json:"strategy,omitempty"` + + // Flag to enable/disable sidecar container nebula-agent injection, default false. + // +optional + EnableBR *bool `json:"enableBR,omitempty"` + + // optional + //BrAgentVersion *string `json:"brAgentVersion,omitempty"` } // NebulaClusterStatus defines the observed state of NebulaCluster @@ -170,7 +177,7 @@ type NebulaClusterCondition struct { // The reason for the condition's last transition. // +optional Reason string `json:"reason,omitempty"` - // A human readable message indicating details about the transition. + // A human-readable message indicating details about the transition. // +optional Message string `json:"message,omitempty"` } @@ -254,6 +261,10 @@ type StoragedSpec struct { // Flag to enable/disable auto balance data and leader while the nebula storaged scale out, default false // +optional EnableAutoBalance *bool `json:"enableAutoBalance,omitempty"` + + // Flag to enable/disable rolling update without leader state transition + // +optional + EnableForceUpdate *bool `json:"enableForceUpdate,omitempty"` } // PodSpec is a common set of k8s resource configs for nebula components. @@ -295,6 +306,9 @@ type PodSpec struct { // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // +optional + InitContainers []corev1.Container `json:"initContainers,omitempty"` + // +optional SidecarContainers []corev1.Container `json:"sidecarContainers,omitempty"` diff --git a/apis/apps/v1alpha1/nebularestore_types.go b/apis/apps/v1alpha1/nebularestore_types.go new file mode 100644 index 00000000..5df71d4e --- /dev/null +++ b/apis/apps/v1alpha1/nebularestore_types.go @@ -0,0 +1,129 @@ +package v1alpha1 + +import ( + "github.com/vesoft-inc/nebula-go/v3/nebula" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName="rt" +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase`,description="The current status of the restore" +// +kubebuilder:printcolumn:name="Started",type=date,JSONPath=`.status.timeStarted`,description="The time at which the restore was started" +// +kubebuilder:printcolumn:name="Completed",type=date,JSONPath=`.status.timeCompleted`,description="The time at which the restore was completed" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +type NebulaRestore struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RestoreSpec `json:"spec,omitempty"` + Status RestoreStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +// NebulaRestoreList contains a list of NebulaRestore. +type NebulaRestoreList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []NebulaRestore `json:"items"` +} + +// RestoreConditionType represents a valid condition of a Restore. +type RestoreConditionType string + +const ( + // RestoreComplete means the restore has successfully executed and the + // backup data has been loaded into nebula cluster. + RestoreComplete RestoreConditionType = "Complete" + // RestoreMetadComplete means metad pods have been rebuilded from the backup data + RestoreMetadComplete RestoreConditionType = "MetadComplete" + // RestoreStoragedCompleted means storaged pods have been rebuilded from the backup data + RestoreStoragedCompleted RestoreConditionType = "StoragedComplete" + // RestoreFailed means the restore has failed. + RestoreFailed RestoreConditionType = "Failed" + // RestoreInvalid means invalid restore CR. + RestoreInvalid RestoreConditionType = "Invalid" +) + +// RestoreCondition describes the observed state of a Restore at a certain point. +type RestoreCondition struct { + // Type of the condition. + Type RestoreConditionType `json:"type"` + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status"` + // The last time this condition was updated. + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + // Last time the condition transitioned from one status to another. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // The reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty"` + // A human-readable message indicating details about the transition. + // +optional + Message string `json:"message,omitempty"` +} + +type BRConfig struct { + // ClusterName of restore cluster + ClusterName string `json:"clusterName"` + // ClusterNamespace of restore cluster + ClusterNamespace *string `json:"clusterNamespace,omitempty"` + // The name of the backup file. + BackupName string `json:"backupName"` + // Concurrency is used to control the number of concurrent file downloads during data restoration. + Concurrency int32 `json:"concurrency,omitempty"` + // StorageProvider configures where and how backups should be stored. + StorageProvider `json:",inline"` +} + +type StorageProvider struct { + S3 *S3StorageProvider `json:"s3,omitempty"` +} + +// S3StorageProvider represents a S3 compliant storage for storing backups. +type S3StorageProvider struct { + // Region in which the S3 compatible bucket is located. + Region string `json:"region,omitempty"` + // Bucket in which to store the backup data. + Bucket string `json:"bucket,omitempty"` + // Endpoint of S3 compatible storage service + Endpoint string `json:"endpoint,omitempty"` + // SecretName is the name of secret which stores access key and secret key. + SecretName string `json:"secretName,omitempty"` +} + +// RestoreSpec contains the specification for a restore of a nebula cluster backup. +type RestoreSpec struct { + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + BR *BRConfig `json:"br,omitempty"` +} + +// RestoreStatus represents the current status of a nebula cluster restore. +type RestoreStatus struct { + // TimeStarted is the time at which the restore was started. + // +nullable + TimeStarted metav1.Time `json:"timeStarted,omitempty"` + // TimeCompleted is the time at which the restore was completed. + // +nullable + TimeCompleted metav1.Time `json:"timeCompleted,omitempty"` + // ClusterName is the name of restored nebula cluster. + ClusterName string `json:"clusterName,omitempty"` + // Phase is a user readable state inferred from the underlying Restore conditions + Phase RestoreConditionType `json:"phase,omitempty"` + // +nullable + Conditions []RestoreCondition `json:"conditions,omitempty"` + // +nullable + Partitions map[string][]*nebula.HostAddr `json:"partitions,omitempty"` + // +nullable + Checkpoints map[string]map[string]string `json:"checkpoints,omitempty"` +} + +func init() { + SchemeBuilder.Register(&NebulaRestore{}, &NebulaRestoreList{}) +} diff --git a/apis/apps/v1alpha1/template.go b/apis/apps/v1alpha1/template.go index 99e734dd..8cd1c0a3 100644 --- a/apis/apps/v1alpha1/template.go +++ b/apis/apps/v1alpha1/template.go @@ -26,8 +26,10 @@ const ( --pid_file=pids/nebula-graphd.pid # Whether to enable optimizer --enable_optimizer=true -# Heartbeat interval of communication between meta client and graphd service ---heartbeat_interval_secs=10 +# The default charset when a space is created +--default_charset=utf8 +# The default collate when a space is created +--default_collate=utf8_bin # Whether to use the configuration obtained from the configuration file --local_config=true @@ -46,8 +48,8 @@ const ( --stdout_log_file=graphd-stdout.log --stderr_log_file=graphd-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 -# wether logging files' name contain timestamp +--stderrthreshold=3 +# wether logging files' name contain timestamp. --timestamp_in_logfile_name=true ########## query ########## @@ -89,6 +91,12 @@ const ( --ws_http_port=19669 # storage client timeout --storage_client_timeout_ms=60000 +# Enable slow query records +--enable_record_slow_query=true +# The number of slow query records +--slow_query_limit=100 +# slow query threshold in us +--slow_query_threshold_us=200000 # Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file --ws_meta_http_port=19559 @@ -100,7 +108,7 @@ const ( ########## memory ########## # System memory high watermark ratio, cancel the memory checking when the ratio greater than 1.0 ---system_memory_high_watermark_ratio=1.0 +--system_memory_high_watermark_ratio=0.8 ########## audit ########## # This variable is used to enable audit. The value can be 'true' or 'false'. @@ -127,7 +135,7 @@ const ( # This variable is used to specify the audit log format. Supports three log formats [ xml | json | csv ] # This variable has effect only when audit_log_handler is set to 'file'. --audit_log_format=xml -# This variable can be used to specify the comma-separated list of Elasticsearch addresses, +# This variable can be used to specify the comma-seperated list of Elasticsearch addresses, # eg, '192.168.0.1:7001, 192.168.0.2:7001'. # This variable has effect only when audit_log_handler is set to 'es'. --audit_log_es_address= @@ -144,7 +152,7 @@ const ( # The value can be comma separated list of spaces, ie, 'nba, basketball'. --audit_log_exclude_spaces= # This variable is used to specify the list of log categories for tracking, eg, 'login, ddl'. -# There are eight categories for tracking. There are: [ login | exit | ddl | dql | dml | dcl | util | unknown ]. +# There are eight categories for tracking. There are: [ login ｜ exit | ddl | dql | dml | dcl | util | unknown ]. --audit_log_categories=login,exit ########## metrics ########## @@ -154,9 +162,35 @@ const ( # if use experimental features --enable_experimental_feature=false +########## Black box ######## +# Enable black box +--ng_black_box_switch=true +# Black box log folder +--ng_black_box_home=black_box +# Black box dump metrics log period +--ng_black_box_dump_period_seconds=5 +# Black box log files expire time +--ng_black_box_file_lifetime_seconds=1800 + ########## session ########## # Maximum number of sessions that can be created per IP and per user --max_sessions_per_ip_per_user=300 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=false +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 ` // nolint: revive MetadhConfigTemplate = ` @@ -165,7 +199,7 @@ const ( --daemonize=true # The file to host the process id --pid_file=pids/nebula-metad.pid ---license-path=share/nebula.license +--license_path=nebula.license ########## logging ########## # The directory to host logging files @@ -182,8 +216,8 @@ const ( --stdout_log_file=metad-stdout.log --stderr_log_file=metad-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 -# wether logging files' name contain time stamp. +--stderrthreshold=3 +# wether logging files' name contain time stamp, If Using logrotate to rotate logging files, than should set it to true. --timestamp_in_logfile_name=true ########## networking ########## @@ -208,17 +242,22 @@ const ( ########## Misc ######### # The default number of parts when a space is created ---default_parts_num=100 +--default_parts_num=10 # The default replica factor when a space is created --default_replica_factor=1 --heartbeat_interval_secs=10 --agent_heartbeat_interval_secs=60 -############## rocksdb Options ############## ---rocksdb_wal_sync=true - ---minimum_reserved_bytes=104857600 +########## Black box ######## +# Enable black box +--ng_black_box_switch=true +# Black box log folder +--ng_black_box_home=black_box +# Black box dump metrics log period +--ng_black_box_dump_period_seconds=5 +# Black box log files expire time +--ng_black_box_file_lifetime_seconds=1800 ` // nolint: revive StoragedConfigTemplate = ` @@ -245,8 +284,8 @@ const ( --stdout_log_file=storaged-stdout.log --stderr_log_file=storaged-stderr.log # Copy log messages at or above this level to stderr in addition to logfiles. The numbers of severity levels INFO, WARNING, ERROR, and FATAL are 0, 1, 2, and 3, respectively. ---stderrthreshold=2 -# Wether logging files' name contain timestamp. +--stderrthreshold=3 +# Wether logging files' name contain time stamp. --timestamp_in_logfile_name=true ########## networking ########## @@ -270,11 +309,13 @@ const ( --raft_heartbeat_interval_secs=30 # RPC timeout for raft client (ms) --raft_rpc_timeout_ms=500 -## recycle Raft WAL +# recycle Raft WAL --wal_ttl=14400 +# whether send raft snapshot by files via http +--snapshot_send_files=true ########## Disk ########## -# Root data path. split by comma. e.g. --data_path=/disk1/path1/,/disk2/path2/ +# Root data path. Split by comma. e.g. --data_path=/disk1/path1/,/disk2/path2/ # One path per Rocksdb instance. --data_path=data/storage @@ -289,6 +330,8 @@ const ( # Disable page cache to better control memory used by rocksdb. # Caution: Make sure to allocate enough block cache if disabling page cache! --disable_page_cache=false +# The type of storage engine, rocksdb, memory, etc. +--engine_type=rocksdb # Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd # For the sake of binary compatibility, the default value is snappy. @@ -304,14 +347,6 @@ const ( # In order to disable compression for level 0/1, set it to "no:no" --rocksdb_compression_per_level= -############## rocksdb Options ############## -# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma ---rocksdb_db_options={"max_subcompactions":"4","max_background_jobs":"4"} -# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma ---rocksdb_column_family_options={"disable_auto_compactions":"false","write_buffer_size":"67108864","max_write_buffer_number":"4","max_bytes_for_level_base":"268435456"} -# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma ---rocksdb_block_based_table_options={"block_size":"8192"} - # Whether or not to enable rocksdb's statistics, disabled by default --enable_rocksdb_statistics=false @@ -328,27 +363,22 @@ const ( # Whether or not to enable rocksdb's whole key bloom filter, disabled by default. --enable_rocksdb_whole_key_filtering=false -############## Key-Value separation ############## -# Whether or not to enable BlobDB (RocksDB key-value separation support) ---rocksdb_enable_kv_separation=false -# RocksDB key value separation threshold in bytes. Values at or above this threshold will be written to blob files during flush or compaction. ---rocksdb_kv_separation_threshold=100 -# Compression algorithm for blobs, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd ---rocksdb_blob_compression=lz4 -# Whether to garbage collect blobs during compaction ---rocksdb_enable_blob_garbage_collection=true +############## rocksdb Options ############## +# rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma +--rocksdb_db_options={} +# rocksdb ColumnFamilyOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_column_family_options={"write_buffer_size":"67108864","max_write_buffer_number":"4","max_bytes_for_level_base":"268435456"} +# rocksdb BlockBasedTableOptions in json, each name and value of option is string, given as "option_name":"option_value" separated by comma +--rocksdb_block_based_table_options={"block_size":"8192"} ############## storage cache ############## # Whether to enable storage cache --enable_storage_cache=false # Total capacity reserved for storage in memory cache in MB --storage_cache_capacity=0 -# Number of buckets in base 2 logarithm. E.g., in case of 20, the total number of buckets will be 2^20. -# A good estimate can be ceil(log2(cache_entries * 1.6)). The maximum allowed is 32. ---storage_cache_buckets_power=20 -# Number of locks in base 2 logarithm. E.g., in case of 10, the total number of locks will be 2^10. -# A good estimate can be max(1, buckets_power - 10). The maximum allowed is 32. ---storage_cache_locks_power=10 +# Estimated number of cache entries on this storage node in base 2 logarithm. E.g., in case of 20, the estimated number of entries will be 2^20. +# A good estimate can be log2(#vertices on this storage node). The maximum allowed is 31. +--storage_cache_entries_power=20 # Whether to add vertex pool in cache. Only valid when storage cache is enabled. --enable_vertex_pool=false @@ -357,17 +387,69 @@ const ( # TTL in seconds for vertex items in the cache --vertex_item_ttl=300 -# Whether to add empty key pool in cache. Only valid when storage cache is enabled. ---enable_empty_key_pool=false -# Empty key pool size in MB ---empty_key_pool_capacity=50 -# TTL in seconds for empty key items in the cache ---empty_key_item_ttl=300 +# Whether to add negative pool in cache. Only valid when storage cache is enabled. +--enable_negative_pool=false +# Negative pool size in MB +--negative_pool_capacity=50 +# TTL in seconds for negative items in the cache +--negative_item_ttl=300 ############### misc #################### +# Whether turn on query in multiple thread +--query_concurrently=true +# Whether remove outdated space data +--auto_remove_invalid_space=true +# Network IO threads number +--num_io_threads=16 +# Worker threads number to handle request +--num_worker_threads=32 +# Maximum subtasks to run admin jobs concurrently +--max_concurrent_subtasks=10 +# The rate limit in bytes when leader synchronizes snapshot data --snapshot_part_rate_limit=10485760 +# The amount of data sent in each batch when leader synchronizes snapshot data --snapshot_batch_size=1048576 +# The rate limit in bytes when leader synchronizes rebuilding index --rebuild_index_part_rate_limit=4194304 +# The amount of data sent in each batch when leader synchronizes rebuilding index --rebuild_index_batch_size=1048576 + +############## non-volatile cache ############## +# Cache file location +--nv_cache_path=/tmp/cache +# Cache file size in MB +--nv_cache_size=0 +# DRAM part size of non-volatile cache in MB +--nv_dram_size=50 +# DRAM part bucket power. The value is a logarithm with a base of 2. Optional values are 0-32. +--nv_bucket_power=20 +# DRAM part lock power. The value is a logarithm with a base of 2. The recommended value is max(1, nv_bucket_power - 10). +--nv_lock_power=10 + +########## Black box ######## +# Enable black box +--ng_black_box_switch=true +# Black box log folder +--ng_black_box_home=black_box +# Black box dump metrics log period +--ng_black_box_dump_period_seconds=5 +# Black box log files expire time +--ng_black_box_file_lifetime_seconds=1800 + +########## memory tracker ########## +# trackable memory ratio (trackable_memory / (total_memory - untracked_reserved_memory) ) +--memory_tracker_limit_ratio=0.8 +# untracked reserved memory in Mib +--memory_tracker_untracked_reserved_memory_mb=50 + +# enable log memory tracker stats periodically +--memory_tracker_detail_log=false +# log memory tacker stats interval in milliseconds +--memory_tracker_detail_log_interval_ms=60000 + +# enable memory background purge (if jemalloc is used) +--memory_purge_enabled=true +# memory background purge interval in seconds +--memory_purge_interval_seconds=10 ` ) diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 72e278cf..d7f29d17 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -21,10 +22,32 @@ limitations under the License. package v1alpha1 import ( + "github.com/vesoft-inc/nebula-go/v3/nebula" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BRConfig) DeepCopyInto(out *BRConfig) { + *out = *in + if in.ClusterNamespace != nil { + in, out := &in.ClusterNamespace, &out.ClusterNamespace + *out = new(string) + **out = **in + } + in.StorageProvider.DeepCopyInto(&out.StorageProvider) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BRConfig. +func (in *BRConfig) DeepCopy() *BRConfig { + if in == nil { + return nil + } + out := new(BRConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BalanceJob) DeepCopyInto(out *BalanceJob) { *out = *in @@ -302,6 +325,11 @@ func (in *NebulaClusterSpec) DeepCopyInto(out *NebulaClusterSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.EnableBR != nil { + in, out := &in.EnableBR, &out.EnableBR + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NebulaClusterSpec. @@ -339,6 +367,65 @@ func (in *NebulaClusterStatus) DeepCopy() *NebulaClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NebulaRestore) DeepCopyInto(out *NebulaRestore) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NebulaRestore. +func (in *NebulaRestore) DeepCopy() *NebulaRestore { + if in == nil { + return nil + } + out := new(NebulaRestore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NebulaRestore) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NebulaRestoreList) DeepCopyInto(out *NebulaRestoreList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NebulaRestore, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NebulaRestoreList. +func (in *NebulaRestoreList) DeepCopy() *NebulaRestoreList { + if in == nil { + return nil + } + out := new(NebulaRestoreList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NebulaRestoreList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodSpec) DeepCopyInto(out *PodSpec) { *out = *in @@ -392,6 +479,13 @@ func (in *PodSpec) DeepCopyInto(out *PodSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.InitContainers != nil { + in, out := &in.InitContainers, &out.InitContainers + *out = make([]v1.Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.SidecarContainers != nil { in, out := &in.SidecarContainers, &out.SidecarContainers *out = make([]v1.Container, len(*in)) @@ -423,6 +517,127 @@ func (in *PodSpec) DeepCopy() *PodSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestoreCondition) DeepCopyInto(out *RestoreCondition) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreCondition. +func (in *RestoreCondition) DeepCopy() *RestoreCondition { + if in == nil { + return nil + } + out := new(RestoreCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.BR != nil { + in, out := &in.BR, &out.BR + *out = new(BRConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreSpec. +func (in *RestoreSpec) DeepCopy() *RestoreSpec { + if in == nil { + return nil + } + out := new(RestoreSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestoreStatus) DeepCopyInto(out *RestoreStatus) { + *out = *in + in.TimeStarted.DeepCopyInto(&out.TimeStarted) + in.TimeCompleted.DeepCopyInto(&out.TimeCompleted) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]RestoreCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Partitions != nil { + in, out := &in.Partitions, &out.Partitions + *out = make(map[string][]*nebula.HostAddr, len(*in)) + for key, val := range *in { + var outVal []*nebula.HostAddr + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]*nebula.HostAddr, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(nebula.HostAddr) + **out = **in + } + } + } + (*out)[key] = outVal + } + } + if in.Checkpoints != nil { + in, out := &in.Checkpoints, &out.Checkpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreStatus. +func (in *RestoreStatus) DeepCopy() *RestoreStatus { + if in == nil { + return nil + } + out := new(RestoreStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3StorageProvider) DeepCopyInto(out *S3StorageProvider) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3StorageProvider. +func (in *S3StorageProvider) DeepCopy() *S3StorageProvider { + if in == nil { + return nil + } + out := new(S3StorageProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = *in @@ -478,6 +693,26 @@ func (in *StorageClaim) DeepCopy() *StorageClaim { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageProvider) DeepCopyInto(out *StorageProvider) { + *out = *in + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(S3StorageProvider) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageProvider. +func (in *StorageProvider) DeepCopy() *StorageProvider { + if in == nil { + return nil + } + out := new(StorageProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StoragedSpec) DeepCopyInto(out *StoragedSpec) { *out = *in @@ -511,6 +746,11 @@ func (in *StoragedSpec) DeepCopyInto(out *StoragedSpec) { *out = new(bool) **out = **in } + if in.EnableForceUpdate != nil { + in, out := &in.EnableForceUpdate, &out.EnableForceUpdate + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoragedSpec. diff --git a/charts/index.yaml b/charts/index.yaml index 9d6c7193..6db13f28 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -1,6 +1,30 @@ apiVersion: v1 entries: nebula-cluster: + - apiVersion: v2 + appVersion: 1.0.0 + created: "2023-02-13T17:20:16.058861988+08:00" + description: Nebula Cluster Helm chart for Kubernetes + digest: 6281e3356786d0f45b491fe83c8ef9f0fa7e27d785eafad9e566bb3d7994f20b + home: https://nebula-graph.io + keywords: + - kubernetes + - operator + - nebula-graph + - database + - graph database + maintainers: + - email: vee.zhang@vesoft.com + name: Vee Zhang + - email: kevin.qiao@vesoft.com + name: Kevin Qiao + name: nebula-cluster + sources: + - https://github.com/vesoft-inc/nebula + - https://github.com/vesoft-inc/nebula-operator + urls: + - https://github.com/vesoft-inc/nebula-operator/releases/download/v1.4.0/nebula-cluster-1.4.0.tgz + version: 1.4.0 - apiVersion: v2 appVersion: 1.3.0 created: "2022-11-01T11:44:39.833325528+08:00" @@ -121,31 +145,30 @@ entries: urls: - https://github.com/vesoft-inc/nebula-operator/releases/download/v0.9.0/nebula-cluster-0.9.0.tgz version: 0.9.0 + nebula-operator: - apiVersion: v2 - appVersion: 0.8.0 - created: "2021-11-15T18:00:37.434965894+08:00" - description: Nebula Cluster Helm chart for Kubernetes - digest: a39fca70e19f228334ff038edce782ba199a97bb511d843894c89cbbb22a1260 + appVersion: 1.0.0 + created: "2023-02-13T17:20:16.061651741+08:00" + description: Nebula Operator Helm chart for Kubernetes + digest: 59b39cdc70617747ada0cd40d3edd3ffc1834aa3bb09ce49e8ff8949fbc4f097 home: https://nebula-graph.io keywords: - - kubernetes - - operator - - nebula-graph - - database - - graph database + - kubernetes + - operator + - nebula-graph + - database + - graph database maintainers: - - email: vee.zhang@vesoft.com - name: Vee Zhang - - email: kevin.qiao@vesoft.com - name: Kevin Qiao - name: nebula-cluster + - email: vee.zhang@vesoft.com + name: Vee Zhang + - email: kevin.qiao@vesoft.com + name: Kevin Qiao + name: nebula-operator sources: - - https://github.com/vesoft-inc/nebula - - https://github.com/vesoft-inc/nebula-operator + - https://github.com/vesoft-inc/nebula-operator urls: - - https://vesoft-inc.github.io/nebula-operator/charts/nebula-cluster-0.8.0.tgz - version: 0.8.0 - nebula-operator: + - https://github.com/vesoft-inc/nebula-operator/releases/download/v1.4.0/nebula-operator-1.4.0.tgz + version: 1.4.0 - apiVersion: v2 appVersion: 1.3.0 created: "2022-11-01T11:44:39.83600451+08:00" @@ -261,27 +284,4 @@ entries: urls: - https://github.com/vesoft-inc/nebula-operator/releases/download/v0.9.0/nebula-operator-0.9.0.tgz version: 0.9.0 - - apiVersion: v2 - appVersion: 0.8.0 - created: "2021-12-02T18:40:22.614490813+08:00" - description: Nebula Operator Helm chart for Kubernetes - digest: 9fb689a95d626b6cc421c35c2991f695c799922cc30e36c2155e51d73d76d490 - home: https://nebula-graph.io - keywords: - - kubernetes - - operator - - nebula-graph - - database - - graph database - maintainers: - - email: vee.zhang@vesoft.com - name: Vee Zhang - - email: kevin.qiao@vesoft.com - name: Kevin Qiao - name: nebula-operator - sources: - - https://github.com/vesoft-inc/nebula-operator - urls: - - https://vesoft-inc.github.io/nebula-operator/charts/nebula-operator-0.8.0.tgz - version: 0.8.0 generated: "2022-03-31T15:31:42.140130104+08:00" diff --git a/charts/nebula-cluster-0.8.0.tgz b/charts/nebula-cluster-0.8.0.tgz deleted file mode 100644 index e4d0faff..00000000 Binary files a/charts/nebula-cluster-0.8.0.tgz and /dev/null differ diff --git a/charts/nebula-cluster/templates/nebula-cluster.yaml b/charts/nebula-cluster/templates/nebula-cluster.yaml index 7cc14c70..6e695a68 100644 --- a/charts/nebula-cluster/templates/nebula-cluster.yaml +++ b/charts/nebula-cluster/templates/nebula-cluster.yaml @@ -41,6 +41,10 @@ spec: readinessProbe: {{- toYaml . | nindent 6 }} {{- end }} + {{- with .Values.nebula.graphd.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values.nebula.graphd.sidecarContainers }} sidecarContainers: {{- toYaml . | nindent 6 }} @@ -92,6 +96,10 @@ spec: readinessProbe: {{- toYaml . | nindent 6 }} {{- end }} + {{- with .Values.nebula.metad.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values.nebula.metad.sidecarContainers }} sidecarContainers: {{- toYaml . | nindent 6 }} @@ -106,6 +114,7 @@ spec: image: {{ .Values.nebula.storaged.image }} version: {{ .Values.nebula.version }} enableAutoBalance: {{ .Values.nebula.storaged.enableAutoBalance }} + enableForceUpdate: {{ .Values.nebula.enableForceUpdate }} env: {{ toYaml .Values.nebula.storaged.env | nindent 6 }} config: {{ toYaml .Values.nebula.storaged.config | nindent 6 }} logVolumeClaim: @@ -140,6 +149,10 @@ spec: readinessProbe: {{- toYaml . | nindent 6 }} {{- end }} + {{- with .Values.nebula.storaged.initContainers }} + initContainers: + {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values.nebula.storaged.sidecarContainers }} sidecarContainers: {{- toYaml . | nindent 6 }} @@ -156,3 +169,5 @@ spec: schedulerName: {{ .Values.nebula.schedulerName }} unsatisfiableAction: {{ .Values.nebula.unsatisfiableAction }} enablePVReclaim: {{ .Values.nebula.enablePVReclaim }} + enableBR: {{ .Values.nebula.enableBR }} + diff --git a/charts/nebula-cluster/values.yaml b/charts/nebula-cluster/values.yaml index 23f52fd3..3c8dfe7a 100644 --- a/charts/nebula-cluster/values.yaml +++ b/charts/nebula-cluster/values.yaml @@ -1,8 +1,10 @@ nebula: - version: v3.3.0 + version: v3.4.0 imagePullPolicy: Always storageClassName: "" enablePVReclaim: false + enableBR: false + enableForceUpdate: false schedulerName: default-scheduler # nebula-scheduler unsatisfiableAction: ScheduleAnyway reference: @@ -28,6 +30,7 @@ nebula: tolerations: [] affinity: {} readinessProbe: {} + initContainers: [] sidecarContainers: [] sidecarVolumes: [] @@ -44,7 +47,7 @@ nebula: cpu: "1" memory: "1Gi" logStorage: "1Gi" - dataStorage: "5Gi" + dataStorage: "10Gi" license: {} podLabels: {} podAnnotations: {} @@ -52,6 +55,7 @@ nebula: tolerations: [] affinity: {} readinessProbe: {} + initContainers: [] sidecarContainers: [] sidecarVolumes: [] @@ -76,6 +80,7 @@ nebula: tolerations: [] affinity: {} readinessProbe: {} + initContainers: [] sidecarContainers: [] sidecarVolumes: [] diff --git a/charts/nebula-operator-0.8.0.tgz b/charts/nebula-operator-0.8.0.tgz deleted file mode 100644 index 0a9a4906..00000000 Binary files a/charts/nebula-operator-0.8.0.tgz and /dev/null differ diff --git a/charts/nebula-operator/crds/nebulacluster.yaml b/charts/nebula-operator/crds/nebulacluster.yaml index 2e842ce2..2d0ee154 100644 --- a/charts/nebula-operator/crds/nebulacluster.yaml +++ b/charts/nebula-operator/crds/nebulacluster.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.3 creationTimestamp: null name: nebulaclusters.apps.nebula-graph.io spec: @@ -102,6 +101,7 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic weight: format: int32 type: integer @@ -148,10 +148,12 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic type: array required: - nodeSelectorTerms type: object + x-kubernetes-map-type: atomic type: object podAffinity: properties: @@ -183,6 +185,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -206,6 +209,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -249,6 +253,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -272,6 +277,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -313,6 +319,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -336,6 +343,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -379,6 +387,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -402,6 +411,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -414,6 +424,8 @@ spec: type: array type: object type: object + enableBR: + type: boolean enablePVReclaim: type: boolean graphd: @@ -460,6 +472,7 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic weight: format: int32 type: integer @@ -506,10 +519,12 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic type: array required: - nodeSelectorTerms type: object + x-kubernetes-map-type: atomic type: object podAffinity: properties: @@ -541,6 +556,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -564,6 +580,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -607,6 +624,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -630,6 +648,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -671,6 +690,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -694,6 +714,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -737,6 +758,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -760,6 +782,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -800,6 +823,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -809,6 +833,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -824,6 +849,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -835,6 +861,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -843,151 +870,7 @@ spec: image: default: vesoft/graphd type: string - labels: - additionalProperties: - type: string - type: object - logVolumeClaim: - properties: - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - storageClassName: - type: string - type: object - nodeSelector: - additionalProperties: - type: string - type: object - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - replicas: - format: int32 - minimum: 0 - type: integer - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - service: - properties: - annotations: - additionalProperties: - type: string - type: object - clusterIP: - type: string - externalTrafficPolicy: - type: string - loadBalancerIP: - type: string - publishNotReadyAddresses: - type: boolean - selector: - additionalProperties: - type: string - type: object - type: - type: string - type: object - sidecarContainers: + initContainers: items: properties: args: @@ -1018,6 +901,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -1027,6 +911,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -1042,6 +927,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -1053,6 +939,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -1068,6 +955,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic prefix: type: string secretRef: @@ -1077,6 +965,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic type: object type: array image: @@ -1544,144 +1433,182 @@ spec: - name type: object type: array - sidecarVolumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: + labels: + additionalProperties: + type: string + type: object + logVolumeClaim: + properties: + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + storageClassName: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: type: string - secretRef: + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: properties: name: type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: + value: type: string + required: + - name + - value type: object - readOnly: - type: boolean - volumeAttributes: - additionalProperties: + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + replicas: + format: int32 + minimum: 0 + type: integer + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + service: + properties: + annotations: + additionalProperties: + type: string + type: object + clusterIP: + type: string + externalTrafficPolicy: + type: string + loadBalancerIP: + type: string + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + type: + type: string + type: object + sidecarContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: + value: + type: string + valueFrom: properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -1691,11 +1618,7 @@ spec: required: - fieldPath type: object - mode: - format: int32 - type: integer - path: - type: string + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -1711,564 +1634,1578 @@ spec: required: - resource type: object - required: - - path + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: properties: - volumeClaimTemplate: + postStart: properties: - metadata: - type: object - spec: + exec: properties: - accessModes: + command: items: type: string type: array - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: type: string - type: object - type: object - storageClassName: + value: + type: string + required: + - name + - value + type: object + type: array + path: type: string - volumeMode: + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string - volumeName: + required: + - port + type: object + tcpSocket: + properties: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - required: - - spec type: object type: object - fc: + livenessProbe: properties: - fsType: - type: string - lun: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: format: int32 type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: + httpGet: properties: - name: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string + required: + - port type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: + initialDelaySeconds: format: int32 type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: + periodSeconds: format: int32 type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - name: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object name: type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: properties: - defaultMode: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: format: int32 type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: + httpGet: + properties: + host: + type: string + httpHeaders: + items: properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array name: type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: + value: type: string required: - - path + - name + - value type: object - type: object - type: array - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - name: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - user: - type: string - required: - - image - - monitors + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object - scaleIO: + resources: properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system type: object - secret: + securityContext: properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: type: string - mode: - format: int32 - type: integer - path: + type: array + drop: + items: type: string - required: - - key - - path - type: object - type: array - optional: + type: array + type: object + privileged: type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: + procMount: type: string - readOnly: + readOnlyRootFilesystem: type: boolean - secretRef: + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: properties: - name: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: type: string type: object - volumeName: - type: string - volumeNamespace: - type: string type: object - vsphereVolume: + startupProbe: properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - version: - type: string - type: object - imagePullPolicy: - default: Always - type: string - imagePullSecrets: - items: - properties: - name: - type: string - type: object - type: array - metad: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + exec: properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string + required: + - name + type: object + type: array + sidecarVolumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + version: + type: string + type: object + imagePullPolicy: + default: Always + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metad: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string operator: type: string values: @@ -2280,430 +3217,686 @@ spec: - operator type: object type: array - matchFields: + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + config: + additionalProperties: + type: string + type: object + dataVolumeClaim: + properties: + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + storageClassName: + type: string + type: object + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + default: vesoft/graphd + type: string + initContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - weight: - format: int32 - type: integer - required: - - preference - - weight type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: + preStop: + properties: + exec: properties: - matchExpressions: + command: items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object + type: string type: array - matchFields: + type: object + httpGet: + properties: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: + tcpSocket: properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - topologyKey + - port type: object - weight: - format: int32 - type: integer + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - podAffinityTerm - - weight + - port type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: + command: items: type: string type: array - topologyKey: + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string required: - - topologyKey + - port type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - podAffinityTerm - - weight + - port type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: + add: items: type: string type: array - topologyKey: + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: type: string - required: - - topologyKey type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - config: - additionalProperties: - type: string - type: object - dataVolumeClaim: - properties: - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - storageClassName: - type: string - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: + seccompProfile: properties: - key: + localhostProfile: type: string - name: + type: type: string - optional: - type: boolean required: - - key + - type type: object - fieldRef: + windowsOptions: properties: - apiVersion: + gmsaCredentialSpec: type: string - fieldPath: + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: type: string - required: - - fieldPath type: object - resourceFieldRef: + type: object + startupProbe: + properties: + exec: properties: - containerName: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: type: string - divisor: + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: anyOf: - type: integer - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - resource: + scheme: type: string required: - - resource + - port type: object - secretKeyRef: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - key: - type: string - name: + host: type: string - optional: - type: boolean + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - key + - port type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string required: - name type: object type: array - image: - default: vesoft/graphd - type: string labels: additionalProperties: type: string @@ -2882,6 +4075,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -2891,6 +4085,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -2906,6 +4101,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -2917,6 +4113,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -2932,6 +4129,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic prefix: type: string secretRef: @@ -2941,6 +4139,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic type: object type: array image: @@ -3472,6 +4671,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -3488,6 +4688,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeID: type: string required: @@ -3518,6 +4719,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic csi: properties: driver: @@ -3529,6 +4731,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic readOnly: type: boolean volumeAttributes: @@ -3555,6 +4758,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -3575,6 +4779,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -3615,6 +4820,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic dataSourceRef: properties: apiGroup: @@ -3627,6 +4833,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic resources: properties: limits: @@ -3669,6 +4876,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic storageClassName: type: string volumeMode: @@ -3715,6 +4923,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic required: - driver type: object @@ -3799,6 +5008,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic targetPortal: type: string required: @@ -3879,6 +5089,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic downwardAPI: properties: items: @@ -3893,6 +5104,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -3913,6 +5125,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -3940,6 +5153,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic serviceAccountToken: properties: audience: @@ -3994,6 +5208,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -4015,6 +5230,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic sslEnabled: type: boolean storageMode: @@ -4066,6 +5282,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeName: type: string volumeNamespace: @@ -4167,6 +5384,7 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic weight: format: int32 type: integer @@ -4213,10 +5431,12 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic type: array required: - nodeSelectorTerms type: object + x-kubernetes-map-type: atomic type: object podAffinity: properties: @@ -4248,6 +5468,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -4271,6 +5492,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -4314,6 +5536,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -4337,6 +5560,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -4378,6 +5602,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -4401,95 +5626,559 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string type: array - topologyKey: - type: string - required: - - topologyKey + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + config: + additionalProperties: + type: string + type: object + dataVolumeClaims: + items: + properties: + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + storageClassName: + type: string + type: object + type: array + enableAutoBalance: + type: boolean + enableForceUpdate: + type: boolean + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + default: vesoft/graphd + type: string + initContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: + httpGet: properties: - matchExpressions: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array - matchLabels: - additionalProperties: + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: type: string - type: object + type: array type: object - namespaceSelector: + httpGet: properties: - matchExpressions: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array - matchLabels: - additionalProperties: - type: string - type: object + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port type: object - namespaces: + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: items: - type: string + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object type: array - topologyKey: + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string required: - - topologyKey + - port type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - config: - additionalProperties: - type: string - type: object - dataVolumeClaims: - items: - properties: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object resources: properties: limits: @@ -4509,75 +6198,183 @@ spec: x-kubernetes-int-or-string: true type: object type: object - storageClassName: - type: string - type: object - type: array - enableAutoBalance: - type: boolean - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: + securityContext: properties: - configMapKeyRef: + allowPrivilegeEscalation: + type: boolean + capabilities: properties: - key: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: type: string - name: + role: + type: string + type: + type: string + user: type: string - optional: - type: boolean - required: - - key type: object - fieldRef: + seccompProfile: properties: - apiVersion: + localhostProfile: type: string - fieldPath: + type: type: string required: - - fieldPath + - type type: object - resourceFieldRef: + windowsOptions: properties: - containerName: + gmsaCredentialSpec: type: string - divisor: + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: anyOf: - type: integer - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - resource: + scheme: type: string required: - - resource + - port type: object - secretKeyRef: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - key: - type: string - name: + host: type: string - optional: - type: boolean + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - key + - port type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string required: - name type: object type: array - image: - default: vesoft/graphd - type: string labels: additionalProperties: type: string @@ -4749,6 +6546,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -4758,6 +6556,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -4773,6 +6572,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -4784,6 +6584,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -4799,6 +6600,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic prefix: type: string secretRef: @@ -4808,6 +6610,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic type: object type: array image: @@ -5339,6 +7142,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -5355,6 +7159,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeID: type: string required: @@ -5385,6 +7190,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic csi: properties: driver: @@ -5396,6 +7202,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic readOnly: type: boolean volumeAttributes: @@ -5422,6 +7229,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -5442,6 +7250,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -5482,6 +7291,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic dataSourceRef: properties: apiGroup: @@ -5494,6 +7304,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic resources: properties: limits: @@ -5536,6 +7347,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic storageClassName: type: string volumeMode: @@ -5582,6 +7394,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic required: - driver type: object @@ -5666,6 +7479,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic targetPortal: type: string required: @@ -5746,6 +7560,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic downwardAPI: properties: items: @@ -5760,6 +7575,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -5780,6 +7596,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -5807,6 +7624,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic serviceAccountToken: properties: audience: @@ -5861,6 +7679,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -5882,6 +7701,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic sslEnabled: type: boolean storageMode: @@ -5933,6 +7753,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeName: type: string volumeNamespace: @@ -6156,9 +7977,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/charts/nebula-operator/crds/nebularestores.yaml b/charts/nebula-operator/crds/nebularestores.yaml new file mode 100644 index 00000000..5ab741b5 --- /dev/null +++ b/charts/nebula-operator/crds/nebularestores.yaml @@ -0,0 +1,139 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: nebularestores.apps.nebula-graph.io +spec: + group: apps.nebula-graph.io + names: + kind: NebulaRestore + listKind: NebulaRestoreList + plural: nebularestores + shortNames: + - rt + singular: nebularestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current status of the restore + jsonPath: .status.phase + name: Status + type: string + - description: The time at which the restore was started + jsonPath: .status.timeStarted + name: Started + type: date + - description: The time at which the restore was completed + jsonPath: .status.timeCompleted + name: Completed + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + br: + properties: + backupName: + type: string + clusterName: + type: string + clusterNamespace: + type: string + concurrency: + format: int32 + type: integer + s3: + properties: + bucket: + type: string + endpoint: + type: string + region: + type: string + secretName: + type: string + type: object + required: + - backupName + - clusterName + type: object + nodeSelector: + additionalProperties: + type: string + type: object + type: object + status: + properties: + checkpoints: + additionalProperties: + additionalProperties: + type: string + type: object + type: object + clusterName: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + partitions: + additionalProperties: + items: + properties: + host: + type: string + port: + format: int32 + type: integer + required: + - host + - port + type: object + type: array + type: object + phase: + type: string + timeCompleted: + format: date-time + type: string + timeStarted: + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/nebula-operator/templates/controller-manager-rbac.yaml b/charts/nebula-operator/templates/controller-manager-rbac.yaml index 27fa71ff..fb4df70c 100644 --- a/charts/nebula-operator/templates/controller-manager-rbac.yaml +++ b/charts/nebula-operator/templates/controller-manager-rbac.yaml @@ -62,6 +62,13 @@ metadata: labels: {{- include "controller-manager.labels" . | nindent 4 }} rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list - apiGroups: - "" resources: @@ -192,6 +199,33 @@ rules: - get - patch - update + - apiGroups: + - apps.nebula-graph.io + resources: + - nebularestores + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apps.nebula-graph.io + resources: + - nebularestores/finalizers + verbs: + - update + - apiGroups: + - apps.nebula-graph.io + resources: + - nebularestores/status + verbs: + - get + - patch + - update + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/charts/nebula-operator/values.yaml b/charts/nebula-operator/values.yaml index c088845a..78005651 100644 --- a/charts/nebula-operator/values.yaml +++ b/charts/nebula-operator/values.yaml @@ -43,8 +43,8 @@ scheduler: # Enable openkruise scheme for controller manager. (default false) enableKruise: false -# Period at which the controller forces the repopulation of its local object stores. (default 1h0m0s) -syncPeriod: +# Period at which the controller forces the repopulation of its local object stores. (default 0h30m0s) +syncPeriod: 0h30m0s # Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. watchNamespace: diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index ec70a876..5c7782c2 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -23,42 +23,42 @@ import ( kruise "github.com/openkruise/kruise-api/apps/v1alpha1" "github.com/spf13/pflag" - "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/klog/v2/klogr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" "github.com/vesoft-inc/nebula-operator/pkg/controller/nebulacluster" - "github.com/vesoft-inc/nebula-operator/pkg/logging" + "github.com/vesoft-inc/nebula-operator/pkg/controller/nebularestore" "github.com/vesoft-inc/nebula-operator/pkg/version" "github.com/vesoft-inc/nebula-operator/pkg/webhook" ) const ( - defaultWatchNamespace = corev1.NamespaceAll - defaultLeaderElectionNamespace = "" - defaultLeaderElectionID = "nebula-controller-manager-leader" - defaultMetricsBindAddr = ":8080" - defaultHealthProbeBindAddr = ":8081" - defaultSyncPeriod = 30 * time.Minute - defaultWebhookBindPort = 9443 - defaultMaxIngressConcurrentReconciles = 3 - defaultPrintVersion = false - defaultEnableLeaderElection = false - defaultEnableAdmissionWebhook = false - defaultEnableKruise = false + defaultWatchNamespace = corev1.NamespaceAll + defaultLeaderElectionNamespace = "" + defaultLeaderElectionID = "nebula-controller-manager-leader" + defaultMetricsBindAddr = ":8080" + defaultHealthProbeBindAddr = ":8081" + defaultSyncPeriod = 30 * time.Minute + defaultWebhookBindPort = 9443 + defaultMaxConcurrentReconciles = 3 + defaultPrintVersion = false + defaultEnableLeaderElection = false + defaultEnableAdmissionWebhook = false + defaultEnableKruise = false ) var ( scheme = runtime.NewScheme() - log = logging.Log.WithName("setup") + log = ctrl.Log.WithName("setup") ) func init() { @@ -99,24 +99,16 @@ func main() { pflag.BoolVar(&enableAdmissionWebhook, "admission-webhook", defaultEnableAdmissionWebhook, "Enable admission webhook for controller manager. ") // nolint: revive pflag.IntVar(&webhookBindPort, "webhook-bind-port", defaultWebhookBindPort, "The TCP port the Webhook server binds to.") - pflag.IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", defaultMaxIngressConcurrentReconciles, "The max concurrent reconciles.") // nolint: revive + pflag.IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", defaultMaxConcurrentReconciles, "The max concurrent reconciles.") // nolint: revive pflag.StringVar(&watchNamespace, "watch-namespace", defaultWatchNamespace, "Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched.") pflag.DurationVar(&syncPeriod, "sync-period", defaultSyncPeriod, "Period at which the controller forces the repopulation of its local object stores.") pflag.BoolVar(&enableKruise, "enable-kruise", defaultEnableKruise, "Enable openkruise scheme for controller manager.") - opts := logging.Options{ - Development: true, - StacktraceLevel: zap.NewAtomicLevelAt(zap.FatalLevel), - ZapOpts: []zap.Option{ - zap.AddCaller(), - zap.AddCallerSkip(-1), - }, - } - opts.BindFlags(flag.CommandLine) pflag.Parse() - logging.SetLogger(logging.New(logging.UseFlagOptions(&opts))) + + ctrl.SetLogger(klogr.New()) if printVersion { log.Info("Nebula Operator Version", "version", version.Version()) @@ -160,6 +152,18 @@ func main() { os.Exit(1) } + nebulaRestoreReconciler, err := nebularestore.NewRestoreReconciler(mgr) + if err != nil { + log.Error(err, "unable to create nebula restore reconciler", "controller", "NebulaRestore") + os.Exit(1) + } + + if err := nebulaRestoreReconciler.SetupWithManager(mgr, + controller.Options{MaxConcurrentReconciles: maxConcurrentReconciles}); err != nil { + log.Error(err, "unable to create controller", "controller", "NebulaRestore") + os.Exit(1) + } + if enableAdmissionWebhook { log.Info("setup webhook") if err = webhook.SetupWithManager(mgr); err != nil { diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index dae014f5..14127bf2 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -22,15 +22,15 @@ import ( "github.com/gin-gonic/gin" "github.com/spf13/pflag" - "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" - "github.com/vesoft-inc/nebula-operator/pkg/logging" "github.com/vesoft-inc/nebula-operator/pkg/scheduler/extender" "github.com/vesoft-inc/nebula-operator/pkg/scheduler/extender/predicates" "github.com/vesoft-inc/nebula-operator/pkg/scheduler/extender/server" @@ -39,7 +39,7 @@ import ( var ( scheme = runtime.NewScheme() - log = logging.Log.WithName("setup") + log = ctrl.Log.WithName("setup") ) func init() { @@ -53,26 +53,17 @@ func main() { flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.StringVar(&httpPort, "port", "12021", "schedule extender listen address") - opts := logging.Options{ - Development: true, - StacktraceLevel: zap.NewAtomicLevelAt(zap.FatalLevel), - ZapOpts: []zap.Option{ - zap.AddCaller(), - zap.AddCallerSkip(-1), - }, - } - opts.BindFlags(flag.CommandLine) pflag.Parse() - logging.SetLogger(logging.New(logging.UseFlagOptions(&opts))) + ctrl.SetLogger(klogr.New()) if printVersion { log.Info("Nebula Operator Version", "version", version.Version()) os.Exit(0) } - log.Info("Welcome to Nebula Operator.") - log.Info("Nebula Operator Version", "version", version.Version()) + klog.Info("Welcome to Nebula Operator.") + klog.Info("Nebula Operator Version", "version", version.Version()) restConfig := ctrl.GetConfigOrDie() diff --git a/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml b/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml index e2b29f0f..1a9193cb 100644 --- a/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml +++ b/config/crd/bases/apps.nebula-graph.io_nebulaclusters.yaml @@ -1,10 +1,9 @@ - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.7.0 + controller-gen.kubebuilder.io/version: v0.11.3 creationTimestamp: null name: nebulaclusters.apps.nebula-graph.io spec: @@ -102,6 +101,7 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic weight: format: int32 type: integer @@ -148,10 +148,12 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic type: array required: - nodeSelectorTerms type: object + x-kubernetes-map-type: atomic type: object podAffinity: properties: @@ -183,6 +185,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -206,6 +209,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -249,6 +253,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -272,6 +277,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -313,6 +319,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -336,6 +343,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -379,6 +387,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -402,6 +411,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -414,6 +424,8 @@ spec: type: array type: object type: object + enableBR: + type: boolean enablePVReclaim: type: boolean graphd: @@ -460,6 +472,7 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic weight: format: int32 type: integer @@ -506,10 +519,12 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic type: array required: - nodeSelectorTerms type: object + x-kubernetes-map-type: atomic type: object podAffinity: properties: @@ -541,6 +556,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -564,6 +580,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -607,6 +624,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -630,6 +648,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -671,6 +690,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -694,6 +714,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -737,6 +758,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -760,6 +782,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -800,6 +823,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -809,6 +833,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -824,6 +849,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -835,6 +861,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -843,151 +870,7 @@ spec: image: default: vesoft/graphd type: string - labels: - additionalProperties: - type: string - type: object - logVolumeClaim: - properties: - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - storageClassName: - type: string - type: object - nodeSelector: - additionalProperties: - type: string - type: object - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - terminationGracePeriodSeconds: - format: int64 - type: integer - timeoutSeconds: - format: int32 - type: integer - type: object - replicas: - format: int32 - minimum: 0 - type: integer - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - service: - properties: - annotations: - additionalProperties: - type: string - type: object - clusterIP: - type: string - externalTrafficPolicy: - type: string - loadBalancerIP: - type: string - publishNotReadyAddresses: - type: boolean - selector: - additionalProperties: - type: string - type: object - type: - type: string - type: object - sidecarContainers: + initContainers: items: properties: args: @@ -1018,6 +901,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -1027,6 +911,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -1042,6 +927,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -1053,6 +939,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -1068,6 +955,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic prefix: type: string secretRef: @@ -1077,6 +965,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic type: object type: array image: @@ -1544,144 +1433,182 @@ spec: - name type: object type: array - sidecarVolumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: + labels: + additionalProperties: + type: string + type: object + logVolumeClaim: + properties: + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + storageClassName: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: type: string - secretRef: + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: properties: name: type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - csi: - properties: - driver: - type: string - fsType: - type: string - nodePublishSecretRef: - properties: - name: + value: type: string + required: + - name + - value type: object - readOnly: - type: boolean - volumeAttributes: - additionalProperties: + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + replicas: + format: int32 + minimum: 0 + type: integer + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + service: + properties: + annotations: + additionalProperties: + type: string + type: object + clusterIP: + type: string + externalTrafficPolicy: + type: string + loadBalancerIP: + type: string + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + type: + type: string + type: object + sidecarContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: type: string - type: object - required: - - driver - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: + value: + type: string + valueFrom: properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -1691,11 +1618,7 @@ spec: required: - fieldPath type: object - mode: - format: int32 - type: integer - path: - type: string + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -1711,564 +1634,1578 @@ spec: required: - resource type: object - required: - - path + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: properties: - volumeClaimTemplate: + postStart: properties: - metadata: - type: object - spec: + exec: properties: - accessModes: + command: items: type: string type: array - dataSource: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - dataSourceRef: - properties: - apiGroup: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: type: string - type: object - type: object - storageClassName: + value: + type: string + required: + - name + - value + type: object + type: array + path: type: string - volumeMode: + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string - volumeName: + required: + - port + type: object + tcpSocket: + properties: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - required: - - spec type: object type: object - fc: + livenessProbe: properties: - fsType: - type: string - lun: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: format: int32 type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: + httpGet: properties: - name: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string + required: + - port type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: + initialDelaySeconds: format: int32 type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: + periodSeconds: format: int32 type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - name: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object name: type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: properties: - defaultMode: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: format: int32 type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: + httpGet: + properties: + host: + type: string + httpHeaders: + items: properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array name: type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: + value: type: string required: - - path + - name + - value type: object - type: object - type: array - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - tenant: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - name: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - user: - type: string - required: - - image - - monitors + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object - scaleIO: + resources: properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system type: object - secret: + securityContext: properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: type: string - mode: - format: int32 - type: integer - path: + type: array + drop: + items: type: string - required: - - key - - path - type: object - type: array - optional: + type: array + type: object + privileged: type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: + procMount: type: string - readOnly: + readOnlyRootFilesystem: type: boolean - secretRef: + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: properties: - name: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: type: string type: object - volumeName: - type: string - volumeNamespace: - type: string type: object - vsphereVolume: + startupProbe: properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - version: - type: string - type: object - imagePullPolicy: - default: Always - type: string - imagePullSecrets: - items: - properties: - name: - type: string - type: object - type: array - metad: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + exec: properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string + required: + - name + type: object + type: array + sidecarVolumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + version: + type: string + type: object + imagePullPolicy: + default: Always + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + metad: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string operator: type: string values: @@ -2280,430 +3217,686 @@ spec: - operator type: object type: array - matchFields: + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + config: + additionalProperties: + type: string + type: object + dataVolumeClaim: + properties: + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + storageClassName: + type: string + type: object + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + default: vesoft/graphd + type: string + initContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + httpGet: + properties: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port type: object - weight: - format: int32 - type: integer - required: - - preference - - weight type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: + preStop: + properties: + exec: properties: - matchExpressions: + command: items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object + type: string type: array - matchFields: + type: object + httpGet: + properties: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: + tcpSocket: properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: + host: type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - topologyKey + - port type: object - weight: - format: int32 - type: integer + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - podAffinityTerm - - weight + - port type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: + command: items: type: string type: array - topologyKey: + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string required: - - topologyKey + - port type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - podAffinityTerm - - weight + - port type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaceSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: + add: items: type: string type: array - topologyKey: + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: type: string - required: - - topologyKey type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - config: - additionalProperties: - type: string - type: object - dataVolumeClaim: - properties: - resources: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - storageClassName: - type: string - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: + seccompProfile: properties: - key: + localhostProfile: type: string - name: + type: type: string - optional: - type: boolean required: - - key + - type type: object - fieldRef: + windowsOptions: properties: - apiVersion: + gmsaCredentialSpec: type: string - fieldPath: + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: type: string - required: - - fieldPath type: object - resourceFieldRef: + type: object + startupProbe: + properties: + exec: properties: - containerName: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: type: string - divisor: + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: anyOf: - type: integer - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - resource: + scheme: type: string required: - - resource + - port type: object - secretKeyRef: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - key: - type: string - name: + host: type: string - optional: - type: boolean + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - key + - port type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string required: - name type: object type: array - image: - default: vesoft/graphd - type: string labels: additionalProperties: type: string @@ -2882,6 +4075,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -2891,6 +4085,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -2906,6 +4101,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -2917,6 +4113,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -2932,6 +4129,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic prefix: type: string secretRef: @@ -2941,6 +4139,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic type: object type: array image: @@ -3472,6 +4671,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -3488,6 +4688,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeID: type: string required: @@ -3518,6 +4719,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic csi: properties: driver: @@ -3529,6 +4731,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic readOnly: type: boolean volumeAttributes: @@ -3555,6 +4758,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -3575,6 +4779,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -3615,6 +4820,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic dataSourceRef: properties: apiGroup: @@ -3627,6 +4833,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic resources: properties: limits: @@ -3669,6 +4876,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic storageClassName: type: string volumeMode: @@ -3715,6 +4923,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic required: - driver type: object @@ -3799,6 +5008,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic targetPortal: type: string required: @@ -3879,6 +5089,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic downwardAPI: properties: items: @@ -3893,6 +5104,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -3913,6 +5125,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -3940,6 +5153,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic serviceAccountToken: properties: audience: @@ -3994,6 +5208,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -4015,6 +5230,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic sslEnabled: type: boolean storageMode: @@ -4066,6 +5282,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeName: type: string volumeNamespace: @@ -4167,6 +5384,7 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic weight: format: int32 type: integer @@ -4213,10 +5431,12 @@ spec: type: object type: array type: object + x-kubernetes-map-type: atomic type: array required: - nodeSelectorTerms type: object + x-kubernetes-map-type: atomic type: object podAffinity: properties: @@ -4248,6 +5468,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -4271,6 +5492,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -4314,6 +5536,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -4337,6 +5560,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string @@ -4378,6 +5602,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaceSelector: properties: matchExpressions: @@ -4401,95 +5626,559 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic namespaces: items: type: string type: array - topologyKey: - type: string - required: - - topologyKey + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + config: + additionalProperties: + type: string + type: object + dataVolumeClaims: + items: + properties: + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + storageClassName: + type: string + type: object + type: array + enableAutoBalance: + type: boolean + enableForceUpdate: + type: boolean + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + default: vesoft/graphd + type: string + initContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: + httpGet: properties: - matchExpressions: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array - matchLabels: - additionalProperties: + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: type: string - type: object + type: array type: object - namespaceSelector: + httpGet: properties: - matchExpressions: + host: + type: string + httpHeaders: items: properties: - key: + name: type: string - operator: + value: type: string - values: - items: - type: string - type: array required: - - key - - operator + - name + - value type: object type: array - matchLabels: - additionalProperties: - type: string - type: object + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port type: object - namespaces: + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: items: - type: string + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object type: array - topologyKey: + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: type: string required: - - topologyKey + - port type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - config: - additionalProperties: - type: string - type: object - dataVolumeClaims: - items: - properties: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object resources: properties: limits: @@ -4509,75 +6198,183 @@ spec: x-kubernetes-int-or-string: true type: object type: object - storageClassName: - type: string - type: object - type: array - enableAutoBalance: - type: boolean - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: + securityContext: properties: - configMapKeyRef: + allowPrivilegeEscalation: + type: boolean + capabilities: properties: - key: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: type: string - name: + role: + type: string + type: + type: string + user: type: string - optional: - type: boolean - required: - - key type: object - fieldRef: + seccompProfile: properties: - apiVersion: + localhostProfile: type: string - fieldPath: + type: type: string required: - - fieldPath + - type type: object - resourceFieldRef: + windowsOptions: properties: - containerName: + gmsaCredentialSpec: type: string - divisor: + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: anyOf: - type: integer - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - resource: + scheme: type: string required: - - resource + - port type: object - secretKeyRef: + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: properties: - key: - type: string - name: + host: type: string - optional: - type: boolean + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true required: - - key + - port type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + type: string required: - name type: object type: array - image: - default: vesoft/graphd - type: string labels: additionalProperties: type: string @@ -4749,6 +6546,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic fieldRef: properties: apiVersion: @@ -4758,6 +6556,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic resourceFieldRef: properties: containerName: @@ -4773,6 +6572,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic secretKeyRef: properties: key: @@ -4784,6 +6584,7 @@ spec: required: - key type: object + x-kubernetes-map-type: atomic type: object required: - name @@ -4799,6 +6600,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic prefix: type: string secretRef: @@ -4808,6 +6610,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic type: object type: array image: @@ -5339,6 +7142,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -5355,6 +7159,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeID: type: string required: @@ -5385,6 +7190,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic csi: properties: driver: @@ -5396,6 +7202,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic readOnly: type: boolean volumeAttributes: @@ -5422,6 +7229,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -5442,6 +7250,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -5482,6 +7291,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic dataSourceRef: properties: apiGroup: @@ -5494,6 +7304,7 @@ spec: - kind - name type: object + x-kubernetes-map-type: atomic resources: properties: limits: @@ -5536,6 +7347,7 @@ spec: type: string type: object type: object + x-kubernetes-map-type: atomic storageClassName: type: string volumeMode: @@ -5582,6 +7394,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic required: - driver type: object @@ -5666,6 +7479,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic targetPortal: type: string required: @@ -5746,6 +7560,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic downwardAPI: properties: items: @@ -5760,6 +7575,7 @@ spec: required: - fieldPath type: object + x-kubernetes-map-type: atomic mode: format: int32 type: integer @@ -5780,6 +7596,7 @@ spec: required: - resource type: object + x-kubernetes-map-type: atomic required: - path type: object @@ -5807,6 +7624,7 @@ spec: optional: type: boolean type: object + x-kubernetes-map-type: atomic serviceAccountToken: properties: audience: @@ -5861,6 +7679,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic user: type: string required: @@ -5882,6 +7701,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic sslEnabled: type: boolean storageMode: @@ -5933,6 +7753,7 @@ spec: name: type: string type: object + x-kubernetes-map-type: atomic volumeName: type: string volumeNamespace: @@ -6156,9 +7977,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/config/crd/bases/apps.nebula-graph.io_nebularestores.yaml b/config/crd/bases/apps.nebula-graph.io_nebularestores.yaml new file mode 100644 index 00000000..5ab741b5 --- /dev/null +++ b/config/crd/bases/apps.nebula-graph.io_nebularestores.yaml @@ -0,0 +1,139 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: nebularestores.apps.nebula-graph.io +spec: + group: apps.nebula-graph.io + names: + kind: NebulaRestore + listKind: NebulaRestoreList + plural: nebularestores + shortNames: + - rt + singular: nebularestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current status of the restore + jsonPath: .status.phase + name: Status + type: string + - description: The time at which the restore was started + jsonPath: .status.timeStarted + name: Started + type: date + - description: The time at which the restore was completed + jsonPath: .status.timeCompleted + name: Completed + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + br: + properties: + backupName: + type: string + clusterName: + type: string + clusterNamespace: + type: string + concurrency: + format: int32 + type: integer + s3: + properties: + bucket: + type: string + endpoint: + type: string + region: + type: string + secretName: + type: string + type: object + required: + - backupName + - clusterName + type: object + nodeSelector: + additionalProperties: + type: string + type: object + type: object + status: + properties: + checkpoints: + additionalProperties: + additionalProperties: + type: string + type: object + type: object + clusterName: + type: string + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + partitions: + additionalProperties: + items: + properties: + host: + type: string + port: + format: int32 + type: integer + required: + - host + - port + type: object + type: array + type: object + phase: + type: string + timeCompleted: + format: date-time + type: string + timeStarted: + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 670f05b5..3de033c4 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -6,6 +6,13 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list - apiGroups: - "" resources: @@ -136,3 +143,29 @@ rules: - get - patch - update +- apiGroups: + - apps.nebula-graph.io + resources: + - nebularestores + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.nebula-graph.io + resources: + - nebularestores/finalizers + verbs: + - update +- apiGroups: + - apps.nebula-graph.io + resources: + - nebularestores/status + verbs: + - get + - patch + - update diff --git a/config/samples/apps_v1alpha1_nebulacluster.yaml b/config/samples/apps_v1alpha1_nebulacluster.yaml index 1e266790..e4db5c99 100644 --- a/config/samples/apps_v1alpha1_nebulacluster.yaml +++ b/config/samples/apps_v1alpha1_nebulacluster.yaml @@ -13,7 +13,7 @@ spec: memory: "1Gi" replicas: 1 image: vesoft/nebula-graphd - version: v3.3.0 + version: v3.4.0 service: type: NodePort externalTrafficPolicy: Local @@ -21,8 +21,11 @@ spec: resources: requests: storage: 1Gi - storageClassName: gp2 + storageClassName: ebs-sc metad: +# license: +# secretName: "nebula-license" +# licenseKey: "nebula.license" resources: requests: cpu: "500m" @@ -32,17 +35,17 @@ spec: memory: "1Gi" replicas: 1 image: vesoft/nebula-metad - version: v3.3.0 + version: v3.4.0 dataVolumeClaim: resources: requests: storage: 5Gi - storageClassName: gp2 + storageClassName: ebs-sc logVolumeClaim: resources: requests: storage: 1Gi - storageClassName: gp2 + storageClassName: ebs-sc storaged: resources: requests: @@ -53,19 +56,21 @@ spec: memory: "1Gi" replicas: 3 image: vesoft/nebula-storaged - version: v3.3.0 + version: v3.4.0 dataVolumeClaims: - resources: requests: storage: 10Gi - storageClassName: gp2 + storageClassName: ebs-sc logVolumeClaim: resources: requests: storage: 1Gi - storageClassName: gp2 + storageClassName: ebs-sc reference: name: statefulsets.apps version: v1 schedulerName: default-scheduler +# enablePVReclaim: false +# enableBR: false imagePullPolicy: Always diff --git a/config/samples/apps_v1alpha1_nebularestore.yaml b/config/samples/apps_v1alpha1_nebularestore.yaml new file mode 100644 index 00000000..c38404df --- /dev/null +++ b/config/samples/apps_v1alpha1_nebularestore.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aws-s3-secret +type: Opaque +data: + access-key: QVNJQVE0WFlSWFE1TlhVMlczNlUK + secret-key: ZFJ6OEdNcDdxenMwVGFGdExVM2RpYkk4b2hHRWRSamgvNTdzWkg3Ugo= +--- +apiVersion: apps.nebula-graph.io/v1alpha1 +kind: NebulaRestore +metadata: + name: restore1 +spec: + br: + clusterName: nebula + backupName: "BACKUP_2023_02_12_10_04_16" + concurrency: 3 + s3: + region: "us-west-2" + bucket: "nebula-br-test" + endpoint: "https://s3.us-west-2.amazonaws.com" + secretName: "aws-s3-secret" \ No newline at end of file diff --git a/config/samples/full-backup-job.yaml b/config/samples/full-backup-job.yaml new file mode 100644 index 00000000..0e71c6ba --- /dev/null +++ b/config/samples/full-backup-job.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: nebula-full-backup +spec: + parallelism: 1 + ttlSecondsAfterFinished: 60 + template: + spec: + restartPolicy: OnFailure + containers: + - image: vesoft/br-ent:v3.4.0 + imagePullPolicy: Always + name: backup + command: + - /bin/sh + - -ecx + - exec /usr/local/bin/nebula-br backup full + - --meta $META_ADDRESS:9559 + - --storage s3://$BUCKET + - --s3.access_key $ACCESS_KEY + - --s3.secret_key $SECRET_KEY + - --s3.region $REGION + - --s3.endpoint https://s3.$REGION.amazonaws.com \ No newline at end of file diff --git a/config/samples/incremental-backup-job.yaml b/config/samples/incremental-backup-job.yaml new file mode 100644 index 00000000..ee6bbf82 --- /dev/null +++ b/config/samples/incremental-backup-job.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aws-s3-secret +type: Opaque +data: + access-key: QVNJQVE0WFlSWFE1TlhVMlczNlUK + secret-key: ZFJ6OEdNcDdxenMwVGFGdExVM2RpYkk4b2hHRWRSamgvNTdzWkg3Ugo= +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: nebula-incr-backup +spec: + parallelism: 1 + ttlSecondsAfterFinished: 60 + template: + spec: + restartPolicy: OnFailure + containers: + - image: vesoft/br-ent:v3.4.0 + imagePullPolicy: Always + name: backup + command: + - /bin/sh + - -ecx + - exec /usr/local/bin/nebula-br backup incr + - --meta $META_ADDRESS:9559 + - --base $BACKUP_NAME + - --storage s3://$BUCKET + - --s3.access_key $ACCESS_KEY + - --s3.secret_key $SECRET_KEY + - --s3.region $REGION + - --s3.endpoint https://s3.$REGION.amazonaws.com \ No newline at end of file diff --git a/config/samples/restore-pod.yaml b/config/samples/restore-pod.yaml new file mode 100644 index 00000000..2fe7dfce --- /dev/null +++ b/config/samples/restore-pod.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aws-s3-secret +type: Opaque +data: + access-key: QVNJQVE0WFlSWFE1TlhVMlczNlUK + secret-key: ZFJ6OEdNcDdxenMwVGFGdExVM2RpYkk4b2hHRWRSamgvNTdzWkg3Ugo= +--- +apiVersion: v1 +kind: Pod +metadata: + name: nebula-restore +spec: + containers: + - image: vesoft/br-ent:v3.4.0 + imagePullPolicy: Always + name: restore + command: + - /bin/sh + - -ecx + - exec /usr/local/bin/nebula-br restore full + - --mode=k8s + - --cluster=nebula + - --namespace default + - --secret=aws-s3-secret + - --name BACKUP_2023_02_10_09_57_17 + - --storage s3://$BUCKET/ + - --s3.region=$REGION + - --s3.endpoint https://s3.$REGION.amazonaws.com \ No newline at end of file diff --git a/doc/pictures/restore.png b/doc/pictures/restore.png new file mode 100644 index 00000000..64c1d2fb Binary files /dev/null and b/doc/pictures/restore.png differ diff --git a/doc/pictures/storage.png b/doc/pictures/storage.png new file mode 100644 index 00000000..d52f9817 Binary files /dev/null and b/doc/pictures/storage.png differ diff --git a/doc/user/add-ons.md b/doc/user/add-ons.md index 6acade1d..ad1c54f9 100644 --- a/doc/user/add-ons.md +++ b/doc/user/add-ons.md @@ -1,39 +1,59 @@ # Installing Add-ons **Caution:** -This section links to third party projects that provide functionality required by nebula-operator. The nebula-operator project authors aren't responsible for these projects. +This section links to third party projects that provide functionality required by nebula-operator. The nebula-operator +project authors aren't responsible for these projects. ## coredns -[CoreDNS](https://coredns.io/) is a flexible, extensible DNS server which can be [installed](https://github.com/coredns/deployment/tree/master/kubernetes) as the in-cluster DNS for pods. -NebulaGraph each component can communicate via DNS like _x.default.svc.cluster.local_, coredns is used for address resolution. +[CoreDNS](https://coredns.io/) is a flexible, extensible DNS server which can +be [installed](https://github.com/coredns/deployment/tree/master/kubernetes) as the in-cluster DNS for pods. + +NebulaGraph each component can communicate via DNS like _x.default.svc.cluster.local_, coredns is used for address +resolution. ## cert-manager + **Note:** -If you set helm chart nebula-operator _.Values.admissionWebhook.create_ to false, the cert-manager is not needed. +If you set helm chart nebula-operator _.Values.admissionWebhook.create_ to false, the cert-manager is not needed. -[cert-manager](https://cert-manager.io/) is a tool that automates certificate management. It makes use of extending the Kubernetes API server using a Webhook server to provide dynamic admission control over cert-manager resources. +[cert-manager](https://cert-manager.io/) is a tool that automates certificate management. It makes use of extending the +Kubernetes API server using a Webhook server to provide dynamic admission control over cert-manager resources. -Consult the [cert-manager installation documentation](https://cert-manager.io/docs/installation/kubernetes/) to get started. +Refer to the [cert-manager installation documentation](https://cert-manager.io/docs/installation/kubernetes/) to get +started. -cert-manager is used for validating NebulaGraph each component replicas, if you run it in production environment and care high availability, you should set _.Values.admissionWebhook.create_ to true and install cert-manager. +cert-manager is used for validating NebulaGraph each component replicas, if you run it in production environment and +care high availability, you should set _.Values.admissionWebhook.create_ to true and install cert-manager. ## openkruise + **Note:** -nebula-operator need advanced features for StatefulSet when it starts. nebula-operator will update nebula cluster pod in-place instead of destroying it. +If you set helm chart nebula-operator _.Values.enableKruise.create_ to false, the kruise suites is not needed. -[openkruise](https://openkruise.io/) is a full set of standard extensions for Kubernetes. It works well with original Kubernetes and provides more powerful and efficient features for managing applications Pods, sidecar containers, and even images on Node. +[openkruise](https://openkruise.io/) is a full set of standard extensions for Kubernetes. It works well with original +Kubernetes and provides more powerful and efficient features for managing applications Pods, sidecar containers, and +even images on Node. -Consult the [openkruise installation documentation](https://openkruise.io/docs/installation) to get started. +Refer to the [openkruise installation documentation](https://openkruise.io/docs/installation) to get started. ## sig-storage-local-static-provisioner + **Note:** -It is required in the scenario that you deploy NebulaGraph with local storage. +It is only used in the scenario that you deploy NebulaGraph with local storage, it's not necessary. -[local-static-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) manages the PersistentVolume lifecycle for pre-allocated disks by detecting and creating PVs for each local disk on the host, and cleaning up the disks when released. It does not support dynamic provisioning. +[local-static-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) manages the +PersistentVolume lifecycle for pre-allocated disks by detecting and creating PVs for each local disk on the host, and +cleaning up the disks when released. It does not support dynamic provisioning. -Follow the [getting started guide](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/docs/getting-started.md) to deploy local-volume-provisioner to provision local volumes. +Follow +the [getting started guide](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/docs/getting-started.md) +to deploy local-volume-provisioner to provision local volumes. -Follow the [best practices](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/docs/best-practices.md) for more information on local PV in Kubernetes. +Follow +the [best practices](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/docs/best-practices.md) +for more information on local PV in Kubernetes. -Follow the [mount disks](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/docs/operations.md#sharing-a-disk-filesystem-by-multiple-filesystem-pvs) to mount the disk. +Follow +the [mount disks](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/docs/operations.md#sharing-a-disk-filesystem-by-multiple-filesystem-pvs) +to mount the disk. diff --git a/doc/user/balance.md b/doc/user/balance.md index 64b3f11e..e8d6b287 100644 --- a/doc/user/balance.md +++ b/doc/user/balance.md @@ -6,7 +6,7 @@ Scaling out Storage is divided into two stages. * In the second stage, the BALANCE DATA and BALANCE LEADER command is executed. -We provide a parameter `enableAutoBalance` in crd to control whether to automatically balance data and leader. +We provide a parameter `enableAutoBalance` in CRD to control whether to automatically balance data and leader. Through both stages, the scaling process of the controller replicas is decoupled from the balancing data process and user executing it at low traffic. diff --git a/doc/user/br_guide.md b/doc/user/br_guide.md new file mode 100644 index 00000000..cd626c41 --- /dev/null +++ b/doc/user/br_guide.md @@ -0,0 +1,103 @@ +# Backup & Restore + +### Requirements + +* The feature is used for enterprise version +* The NebulaGraph cluster is running +* Operator version >= 1.4.0 +* Set the field `enableBR` ot __true__ +* Sufficient computational resources can be scheduled to restore NebulaGraph cluster(only restore scenario needed) +* S3 protocol compatible storage (AWS S3，Minio, etc.) + +## Backup NebulaGraph cluster + +#### Full backup + +Update the [full-backup-job.yaml](../../config/samples/full-backup-job.yaml) fields: + +* META_ENDPOINT +* AWS_S3_REGION +* AWS_S3_BUCKET +* AWS_S3_ENDPOINT +* secret aws-s3-secret data access-key and secret-key + +```shell +$ kubectl apply -f full-backup-job.yaml +$ kubectl describe job nebula-full-backup +# Pod name is shown under "Events" +$ kubectl logs logs $POD -f +``` + +#### Incremental backup + +Update the [full-backup-job.yaml](../../config/samples/incremental-backup-job.yaml) fields: + +* META_ENDPOINT +* AWS_S3_REGION +* AWS_S3_BUCKET +* AWS_S3_ENDPOINT +* secret data access-key and secret-key + +```shell +$ kubectl apply -f incremental-backup-job.yaml +$ kubectl describe job nebula-incr-backup +# Pod name is shown under "Events" +$ kubectl logs logs $POD -f +``` + +## Restore NebulaGraph cluster + +The restore flow: + +![avatar](../pictures/restore.png) + +Update the [apps_v1alpha1_nebularestore.yaml](../../config/samples/apps_v1alpha1_nebularestore.yaml) fields: + +* clusterName +* backupName +* concurrency +* S3 storage sections +* secret aws-s3-secret data access-key and secret-key + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: aws-s3-secret +type: Opaque +data: + access-key: QVNJQVE0WFlSXFE1TlhVMlczNlUK + secret-key: ZFJ6OEdNcDdxenMwVGFGdFxVM2RpYkk4b2hHRWRSamgvNTdzWkg3Ugo= +--- +apiVersion: apps.nebula-graph.io/v1alpha1 +kind: NebulaRestore +metadata: + name: restore1 +spec: + br: + # The name of the restore nebula cluster + clusterName: nebula + # The name of the backup file. + backupName: "BACKUP_2023_02_05_04_36_41" + # Used to control the number of concurrent file downloads during data restoration. + # The default value is 5. + concurrency: 5 + s3: + # Region in which the S3 compatible bucket is located. + region: "us-east-1" + # Bucket in which to store the backup data. + bucket: "nebula-br-test" + # Endpoint of S3 compatible storage service + endpoint: "https://s3.us-east-1.amazonaws.com" + # SecretName is the name of secret which stores access key and secret key. + secretName: "aws-s3-secret" +``` + +```shell +$ kubectl apply -f apps_v1alpha1_nebularestore.yaml +$ kubectl get rt restore1 -w +``` + +**Note:** + +The operator won't remove old nebula cluster after restore successfully, you can do it manually. diff --git a/doc/user/client_service.md b/doc/user/client_service.md index 5efd6907..ae4e4c0b 100644 --- a/doc/user/client_service.md +++ b/doc/user/client_service.md @@ -15,7 +15,7 @@ The client service is of type `ClusterIP` and accessible only from within the Ku For example, access the service from a pod in the cluster: ```shell script -$ kubectl run --rm -ti --image vesoft/nebula-console:v2.5.0 --restart=Never -- /bin/sh +$ kubectl run --rm -ti --image vesoft/nebula-console:v3.4.0 --restart=Never -- /bin/sh / # nebula-console -u user -p password --address=nebula-graphd-svc --port=9669 2021/04/12 08:16:30 [INFO] connection pool is initialized successfully @@ -88,9 +88,9 @@ Welcome to NebulaGraph! ``` ## Accessing the service via nginx-ingress-controller -Nginx Ingress is an implementation of Kubernetes Ingress. It watch the Ingress resources of the Kubernetes cluster then translate the Ingress rules into Nginx configurations, enabling Nginx to forward layer 7 traffic. +Nginx Ingress is an implementation of Kubernetes Ingress. It watches the Ingress resources of the Kubernetes cluster then translate the Ingress rules into Nginx configurations, enabling Nginx to forward layer 7 traffic. -We provide an scenario to replace `NodePort` service, it runs in hostNetwork + daemonSet mode. +We provide a scenario to replace `NodePort` service, it runs in hostNetwork + daemonSet mode. As hostNetwork is used, the Nginx Ingress pods cannot be scheduled to the same node. In order to avoid listening port conflicts, some nodes can be selected and labeled as edge nodes in advance, which are specially used to deploy Nginx Ingress. Nginx Ingress is then deployed on these nodes in DaemonSet mode. diff --git a/doc/user/custom_config.md b/doc/user/custom_config.md index 5a35fd6c..a47d121c 100644 --- a/doc/user/custom_config.md +++ b/doc/user/custom_config.md @@ -25,7 +25,7 @@ spec: memory: "1Gi" replicas: 1 image: vesoft/nebula-graphd - version: v3.3.0 + version: v3.4.0 storageClaim: resources: requests: diff --git a/doc/user/install_guide.md b/doc/user/install_guide.md index 1cf14719..40b1d78d 100644 --- a/doc/user/install_guide.md +++ b/doc/user/install_guide.md @@ -1,6 +1,7 @@ # Install Guide -Instead of manually installing, scaling, upgrading, and uninstalling NebulaGraph in a production environment, you can instead let the Nebula [operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) manage the installation for you. This relieves you of the burden of managing different versions and managing the NebulaGraph cluster more easier. Simply update the operator [custom resource (CR)](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) and the operator controller will apply the corresponding configuration changes for you. +Instead of manually installing, scaling, upgrading, and uninstalling NebulaGraph in a production environment, you can +use the Nebula [operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) to manage NebulaGraph automatically. Follow this guide to install Nebula Operator using Helm for in-depth evaluation. @@ -13,13 +14,13 @@ Follow this guide to install Nebula Operator using Helm for in-depth evaluation. ## Add-ons -See [add ons](add-ons.md) for how to install the add-ons. +See [add-ons](add-ons.md) for how to install the add-ons. ### Get Repo Info ```shell script -helm repo add nebula-operator https://vesoft-inc.github.io/nebula-operator/charts -helm repo update +$ helm repo add nebula-operator https://vesoft-inc.github.io/nebula-operator/charts +$ helm repo update ``` _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ @@ -32,9 +33,11 @@ $ helm install nebula-operator nebula-operator/nebula-operator --namespace=nebul ``` Note: -If the corresponding nebula-system namespace does not exist, you can create the namespace first by running the _kubectl create namespace nebula-operator-system_ command. +If the corresponding nebula-system namespace does not exist, you can create the namespace first by running the _kubectl +create namespace nebula-operator-system_ command. -${chart_version} represents the chart version of Nebula Operator. For example, v0.1.0. You can view the currently supported versions by running the _helm search repo -l nebula-operator_ command. +${chart_version} represents the chart version of Nebula Operator. For example, v0.1.0. You can view the currently +supported versions by running the _helm search repo -l nebula-operator_ command. _See [configuration](#configure-operator) below._ @@ -42,7 +45,10 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen ### Configure Operator -See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](https://github.com/vesoft-inc/nebula-operator/blob/master/charts/nebula-operator/values.yaml), or run the following commands: +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing) +. To see all configurable options with detailed comments, visit the +chart's [values.yaml](https://github.com/vesoft-inc/nebula-operator/blob/master/charts/nebula-operator/values.yaml), or +run the following commands: ```shell script $ helm show values nebula-operator/nebula-operator @@ -50,7 +56,8 @@ $ helm show values nebula-operator/nebula-operator ### Upgrade Operator -If you need to upgrade the Nebula Operator, modify the ${HOME}/nebula-operator/values.yaml file, and then execute the following command to upgrade: +If you need to upgrade the Nebula Operator, modify the ${HOME}/nebula-operator/values.yaml file, and then execute the +following command to upgrade: ```shell script $ helm upgrade nebula-operator nebula-operator/nebula-operator --namespace=nebula-operator-system -f `${HOME}/nebula-operator/values.yaml` diff --git a/doc/user/nebula_cluster_helm_guide.md b/doc/user/nebula_cluster_helm_guide.md index 1b548338..c5e1bc5e 100644 --- a/doc/user/nebula_cluster_helm_guide.md +++ b/doc/user/nebula_cluster_helm_guide.md @@ -1,6 +1,6 @@ -# Install Nebula Cluster with helm +# Install NebulaGraph cluster with helm -Please install [nebula-operator](install_guide.md) before installing Nebula Cluster. +Please install [nebula-operator](install_guide.md) before installing NebulaGraph cluster. ### Get Repo Info @@ -75,14 +75,17 @@ The following table lists is the configurable parameters of the chart and their | Parameter | Description | Default | |:------------------------------------|:----------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------| | `nameOverride` | Override the name of the chart | `nil` | -| `nebula.version` | Nebula version | `v3.3.0` | +| `nebula.version` | Nebula image tag | `v3.4.0` | | `nebula.imagePullPolicy` | Nebula image pull policy | `Always` | -| `nebula.storageClassName` | PersistentVolume class, default to use the default StorageClass | `nil` | +| `enablePVReclaim` | Flag to enable/disable PV reclaim while the Nebula cluster deleted | `false` | +| `enableBR` | Flag to enable/disable sidecar container nebula-agent injection | `false` | +| `enableForUpdate` | Flag to enable/disable rolling update without leader state transition | `false` | +| `nebula.storageClassName` | StorageClass object name | `nil` | | `nebula.schedulerName` | Scheduler for nebula component | `default-scheduler` | | `nebula.reference` | Reference for nebula component | `{"name": "statefulsets.apps", "version": "v1"}` | | `nebula.graphd.image` | Graphd container image without tag, and use `nebula.version` as tag | `vesoft/nebula-graphd` | | `nebula.graphd.replicas` | Graphd replica number | `2` | -| `nebula.graphd.env` | Graphd env | `[]` | +| `nebula.graphd.env` | Graphd container environment variables | `[]` | | `nebula.graphd.resources` | Graphd resources | `{"resources":{"requests":{"cpu":"500m","memory":"500Mi"},"limits":{"cpu":"1","memory":"1Gi"}}}` | | `nebula.graphd.logStorage` | Graphd log volume size | `1Gi` | | `nebula.graphd.podLabels` | Graphd pod labels | `{}` | @@ -91,11 +94,12 @@ The following table lists is the configurable parameters of the chart and their | `nebula.graphd.tolerations` | Graphd pod tolerations | `{}` | | `nebula.graphd.affinity` | Graphd affinity | `{}` | | `nebula.graphd.readinessProbe` | Graphd pod readinessProbe | `{}` | -| `nebula.graphd.sidecarContainers` | Graphd pod sidecarContainers | `{}` | -| `nebula.graphd.sidecarVolumes` | Graphd pod sidecarVolumes | `{}` | +| `nebula.graphd.initContainers` | Graphd pod init containers | `{}` | +| `nebula.graphd.sidecarContainers` | Graphd pod sidecar containers | `{}` | +| `nebula.graphd.sidecarVolumes` | Graphd pod sidecar volumes | `{}` | | `nebula.metad.image` | Metad container image without tag, and use `nebula.version` as tag | `vesoft/nebula-metad` | | `nebula.metad.replicas` | Metad replica number | `3` | -| `nebula.metad.env` | Metad env | `[]` | +| `nebula.metad.env` | Metad container environment variables | `[]` | | `nebula.metad.resources` | Metad resources | `{"resources":{"requests":{"cpu":"500m","memory":"500Mi"},"limits":{"cpu":"1","memory":"1Gi"}}}` | | `nebula.metad.logStorage` | Metad log volume size | `1Gi` | | `nebula.metad.dataStorage` | Metad data volume size | `5Gi` | @@ -105,11 +109,12 @@ The following table lists is the configurable parameters of the chart and their | `nebula.metad.tolerations` | Metad pod tolerations | `{}` | | `nebula.metad.affinity` | Metad affinity | `{}` | | `nebula.metad.readinessProbe` | Metad pod readinessProbe | `{}` | -| `nebula.metad.sidecarContainers` | Metad pod sidecarContainers | `{}` | -| `nebula.metad.sidecarVolumes` | Metad pod sidecarVolumes | `{}` | +| `nebula.graphd.initContainers` | Graphd pod init containers | `{}` | +| `nebula.metad.sidecarContainers` | Metad pod sidecar containers | `{}` | +| `nebula.metad.sidecarVolumes` | Metad pod sidecar volumes | `{}` | | `nebula.storaged.image` | Storaged container image without tag, and use `nebula.version` as tag | `vesoft/nebula-storaged` | | `nebula.storaged.replicas` | Storaged replica number | `3` | -| `nebula.storaged.env` | Storaged env | `[]` | +| `nebula.storaged.env` | Storaged container environment variables | `[]` | | `nebula.storaged.resources` | Storaged resources | `{"resources":{"requests":{"cpu":"500m","memory":"500Mi"},"limits":{"cpu":"1","memory":"1Gi"}}}` | | `nebula.storaged.logStorage` | Storaged log volume size | `1Gi` | | `nebula.storaged.dataStorage` | Storaged data volume size | `10Gi` | @@ -119,6 +124,7 @@ The following table lists is the configurable parameters of the chart and their | `nebula.storaged.tolerations` | Storaged pod tolerations | `{}` | | `nebula.storaged.affinity` | Storaged affinity | `{}` | | `nebula.storaged.readinessProbe` | Storaged pod readinessProbe | `{}` | -| `nebula.storaged.sidecarContainers` | Storaged pod sidecarContainers | `{}` | -| `nebula.storaged.sidecarVolumes` | Storaged pod sidecarVolumes | `{}` | +| `nebula.graphd.initContainers` | Graphd pod init containers | `{}` | +| `nebula.storaged.sidecarContainers` | Storaged pod sidecar containers | `{}` | +| `nebula.storaged.sidecarVolumes` | Storaged pod sidecar volumes | `{}` | | `imagePullSecrets` | The secret to use for pulling the images | `[]` | diff --git a/doc/user/pv_reclaim.md b/doc/user/pv_reclaim.md index 30d15c77..fbc019a1 100644 --- a/doc/user/pv_reclaim.md +++ b/doc/user/pv_reclaim.md @@ -1,7 +1,30 @@ -# PV reclaim +# PV Reclaim -Nebula Operator uses PV (Persistent Volume) and PVC (Persistent Volume Claim) to store persistent data. If you accidentally delete a nebula cluster, the PV/PVC objects and data are still retained to ensure data safety. +Nebula Operator uses [PV (Persistent Volume)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) to store +persistent data. If you accidentally delete a NebulaGraph cluster, the PV objects and data are still retained to ensure +data safety. -We provide a parameter `enablePVReclaim` in crd to control whether reclaim the pv or not. +We provide a parameter `enablePVReclaim` in CRD to control whether reclaim the PV or not. -If you need release the storage spaces and don't want to retain the data, you can update your nebula instance and set the parameter `enablePVReclaim` to __true__. +If you need release the storage volumes and don't want to retain the data, you can update your nebula instance and set +the parameter `enablePVReclaim` to __true__. + +**The details you should know:** + +PVs are created automatically by the system administrator or volume provisioner. PVs and Pods are bound by +PersistentVolumeClaim (PVC). Users request for using a PV through a PVC instead of creating a PV directly. The +corresponding volume provisioner creates a PV that meets the requirements of PVC and then binds the PV to the PVC. + +![avatar](../pictures/storage.png) + +PersistentVolumes that are dynamically created by a StorageClass will have the reclaim policy specified in the +reclaimPolicy field of the class, which can be either Delete or Retain. If no reclaimPolicy is specified when a +StorageClass object is created, it will default to Delete. +This means that a dynamically provisioned volume is automatically deleted when a user deletes the corresponding +PersistentVolumeClaim. This automatic behavior might be inappropriate if the volume contains precious data. In that +case, it is more appropriate to use the "Retain" policy. With the "Retain" policy, if a user deletes a +PersistentVolumeClaim, the corresponding PersistentVolume will not be deleted. Instead, it is moved to the Released +phase, where all of its data can be manually recovered. + +The reclaim policy of a StorageClass is set at creation time, and it cannot be updated once it is created. If it is not +set when created, you can create another StorageClass of the same provisioner. \ No newline at end of file diff --git a/go.mod b/go.mod index f71978f9..59e79bc3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vesoft-inc/nebula-operator -go 1.17 +go 1.18 require ( github.com/facebook/fbthrift v0.31.1-0.20211129061412-801ed7f9f295 @@ -14,11 +14,10 @@ require ( github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 - github.com/vesoft-inc/nebula-go/v3 v3.3.0 - go.uber.org/zap v1.19.1 + github.com/vesoft-inc/nebula-agent v0.0.0-20230204020341-4e388437cd50 + github.com/vesoft-inc/nebula-go/v3 v3.4.0 k8s.io/api v0.23.5 k8s.io/apimachinery v0.23.5 - k8s.io/apiserver v0.22.13 k8s.io/cli-runtime v0.22.13 k8s.io/client-go v0.23.5 k8s.io/code-generator v0.22.13 @@ -43,12 +42,12 @@ require ( github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/aws/aws-sdk-go v1.42.22 // indirect + github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect @@ -62,7 +61,6 @@ require ( github.com/fvbommel/sortorder v1.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-errors/errors v1.0.1 // indirect - github.com/go-logr/zapr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect @@ -75,15 +73,16 @@ require ( github.com/google/btree v1.0.1 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.1.2 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/juju/ratelimit v1.0.1 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.6 // indirect @@ -105,13 +104,10 @@ require ( github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/russross/blackfriday v1.5.2 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect github.com/ugorji/go/codec v1.1.7 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect - go.etcd.io/etcd/api/v3 v3.5.0 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect - go.etcd.io/etcd/client/v3 v3.5.0 // indirect go.opentelemetry.io/contrib v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect go.opentelemetry.io/otel v0.20.0 // indirect go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect @@ -122,8 +118,6 @@ require ( go.opentelemetry.io/otel/trace v0.20.0 // indirect go.opentelemetry.io/proto/otlp v0.7.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect @@ -137,13 +131,14 @@ require ( gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect - google.golang.org/grpc v1.38.0 // indirect + google.golang.org/grpc v1.41.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiextensions-apiserver v0.23.5 // indirect + k8s.io/apiserver v0.22.13 // indirect k8s.io/component-base v0.23.5 // indirect k8s.io/component-helpers v0.22.13 // indirect k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c // indirect diff --git a/go.sum b/go.sum index 4920614d..6be86cb7 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.42.22 h1:EwcM7/+Ytg6xK+jbeM2+f9OELHqPiEiEKetT/GgAr7I= +github.com/aws/aws-sdk-go v1.42.22/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -110,7 +112,6 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -128,6 +129,7 @@ github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1w github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= @@ -152,13 +154,10 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -183,7 +182,6 @@ github.com/docker/docker v20.10.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05b github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -196,6 +194,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -247,7 +246,6 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= -github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -349,8 +347,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= @@ -362,14 +361,11 @@ github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORR github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= @@ -406,10 +402,11 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -423,6 +420,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -438,6 +437,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -582,6 +582,7 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -604,7 +605,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -638,24 +638,24 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/vesoft-inc/nebula-go/v3 v3.3.0 h1:BikigvOF1G0mveRdJEjAeoL3lPNWrtMJs7FyN0k8Awc= -github.com/vesoft-inc/nebula-go/v3 v3.3.0/go.mod h1:+sXv05jYQBARdTbTcIEsWVXCnF/6ttOlDK35xQ6m54s= +github.com/vesoft-inc/nebula-agent v0.0.0-20230204020341-4e388437cd50 h1:VP7jO6YeW672c4HSBkg3tURiUXTztYJ0dJMFIp4evW4= +github.com/vesoft-inc/nebula-agent v0.0.0-20230204020341-4e388437cd50/go.mod h1:4t/Oj5Iu8e9ctwq1kRBsokJdJYduq+JkrINJv62NMtM= +github.com/vesoft-inc/nebula-go/v3 v3.0.0/go.mod h1:+sXv05jYQBARdTbTcIEsWVXCnF/6ttOlDK35xQ6m54s= +github.com/vesoft-inc/nebula-go/v3 v3.4.0 h1:7q2DSW4QABwI2oGPSVuC+Ql7kGwj26G/YVPGD7gETys= +github.com/vesoft-inc/nebula-go/v3 v3.4.0/go.mod h1:+sXv05jYQBARdTbTcIEsWVXCnF/6ttOlDK35xQ6m54s= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -666,21 +666,13 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0 h1:ftQ0nOOHMcbMS3KIaDQ0g5Qcd6bhaBrQT6b89DfwLTs= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0 h1:ntrg6vvKRW26JRmHTE0iNlDgYK6JX3hg/4cD62X0ixk= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -691,7 +683,6 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -719,17 +710,13 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -835,6 +822,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -932,9 +921,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 h1:M69LAlWZCshgp0QSzyDcSsSIejIEeuaCVpmwcKwyLMk= golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1132,8 +1121,9 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1160,7 +1150,6 @@ gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1248,7 +1237,6 @@ k8s.io/sample-apiserver v0.22.13/go.mod h1:VqpaEk9pZepQQXdek5GUX9smI0n5CugwWzeHQ k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= @@ -1264,7 +1252,6 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 h1:dUk62HQ3ZFhD4 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= diff --git a/pkg/annotation/annotation.go b/pkg/annotation/annotation.go index a6b4e1c3..7d9fd379 100644 --- a/pkg/annotation/annotation.go +++ b/pkg/annotation/annotation.go @@ -34,10 +34,67 @@ const ( // AnnPvReclaimKey is annotation key that indicate whether reclaim persistent volume AnnPvReclaimKey = "nebula-graph.io/enable-pv-reclaim" + // AnnRestoreNameKey is restore name annotation key used for creating new nebula cluster with backup data + AnnRestoreNameKey = "nebula-graph.io/restore-name" + // AnnRestoreMetadStepKey is the annotation key to control Metad reconcile process + AnnRestoreMetadStepKey = "nebula-graph.io/restore-metad-done" + // AnnRestoreStoragedStepKey is the annotation key to control Storaged reconcile process + AnnRestoreStoragedStepKey = "nebula-graph.io/restore-storaged-done" + // AnnRestoreStageKey is the annotation key to indicate what is the current stage + AnnRestoreStageKey = "restore-stage" + // AnnHaModeVal is annotation value to indicate whether in ha mode AnnHaModeVal = "true" + + // AnnRestoreMetadStepVal is annotation value to indicate whether Metad restore step is completed in stage 1 + AnnRestoreMetadStepVal = "true" + // AnnRestoreStoragedStepVal is annotation value to indicate whether Storaged restore step is completed in stage 1 + AnnRestoreStoragedStepVal = "true" + + AnnRestoreStage1Val = "restore-stage-1" + AnnRestoreStage2Val = "restore-stage-2" ) +func IsRestoreNameNotEmpty(ann map[string]string) bool { + if ann != nil { + val, ok := ann[AnnRestoreNameKey] + if ok && val != "" { + return true + } + } + return false +} + +func IsRestoreMetadDone(ann map[string]string) bool { + if ann != nil { + val, ok := ann[AnnRestoreMetadStepKey] + if ok && val == AnnRestoreMetadStepVal { + return true + } + } + return false +} + +func IsRestoreStoragedDone(ann map[string]string) bool { + if ann != nil { + val, ok := ann[AnnRestoreStoragedStepKey] + if ok && val == AnnRestoreStoragedStepVal { + return true + } + } + return false +} + +func IsInRestoreStage2(ann map[string]string) bool { + if ann != nil { + val, ok := ann[AnnRestoreStageKey] + if ok && val == AnnRestoreStage2Val { + return true + } + } + return false +} + // IsInHaMode check whether in ha mode func IsInHaMode(ann map[string]string) bool { if ann != nil { diff --git a/pkg/controller/component/graphd_cluster.go b/pkg/controller/component/graphd_cluster.go index b8650843..e6991c71 100644 --- a/pkg/controller/component/graphd_cluster.go +++ b/pkg/controller/component/graphd_cluster.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" "github.com/vesoft-inc/nebula-operator/pkg/annotation" @@ -65,18 +66,16 @@ func (c *graphdCluster) Reconcile(nc *v1alpha1.NebulaCluster) error { func (c *graphdCluster) syncGraphdWorkload(nc *v1alpha1.NebulaCluster) error { namespace := nc.GetNamespace() - ncName := nc.GetName() componentName := nc.GraphdComponent().GetName() - log := getLog().WithValues("namespace", namespace, "name", ncName) gvk, err := resource.GetGVKFromDefinition(c.dm, nc.Spec.Reference) if err != nil { - return fmt.Errorf("get workload reference error: %v", err) + return fmt.Errorf("get workload kind failed: %v", err) } oldWorkloadTemp, err := c.clientSet.Workload().GetWorkload(namespace, componentName, gvk) if err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "failed to get workload") + klog.Errorf("get graphd cluster failed: %v", err) return err } @@ -90,6 +89,7 @@ func (c *graphdCluster) syncGraphdWorkload(nc *v1alpha1.NebulaCluster) error { newWorkload, err := nc.GraphdComponent().GenerateWorkload(gvk, cm, c.enableEvenPodsSpread) if err != nil { + klog.Errorf("generate graphd cluster template failed: %v", err) return err } @@ -100,7 +100,7 @@ func (c *graphdCluster) syncGraphdWorkload(nc *v1alpha1.NebulaCluster) error { } if err := c.syncNebulaClusterStatus(nc, newWorkload, oldWorkload); err != nil { - return fmt.Errorf("failed to sync graphd status status, error: %v", err) + return fmt.Errorf("sync graphd cluster status status failed: %v", err) } if notExist { diff --git a/pkg/controller/component/helper.go b/pkg/controller/component/helper.go index 12741bf0..b70f0357 100644 --- a/pkg/controller/component/helper.go +++ b/pkg/controller/component/helper.go @@ -80,7 +80,6 @@ func setWorkloadStatus(obj *unstructured.Unstructured, status *v1alpha1.Componen } func syncService(component v1alpha1.NebulaClusterComponentter, svcClient kube.Service) error { - log := getLog().WithValues("namespace", component.GetNamespace(), "name", component.GetName()) newSvc := component.GenerateService() if newSvc == nil { return nil @@ -94,7 +93,6 @@ func syncService(component v1alpha1.NebulaClusterComponentter, svcClient kube.Se return svcClient.CreateService(newSvc) } if err != nil { - log.Error(err, "failed to get svc cluster", "svcName", newSvc.Name) return err } @@ -241,12 +239,12 @@ func getNextUpdatePod(component v1alpha1.NebulaClusterComponentter, replicas int } revision, exist := pod.Labels[appsv1.ControllerRevisionHashLabelKey] if !exist { - return -1, &errors.ReconcileError{Msg: fmt.Sprintf("updated pod %s has no label: %s", + return -1, &errors.ReconcileError{Msg: fmt.Sprintf("rolling updated pod %s has no label: %s", podName, appsv1.ControllerRevisionHashLabelKey)} } if revision == updateRevision { if pod.Status.Phase != corev1.PodRunning { - return -1, &errors.ReconcileError{Msg: fmt.Sprintf("updated pod %s is not running", podName)} + return -1, &errors.ReconcileError{Msg: fmt.Sprintf("rolling updated pod %s is not running", podName)} } continue } diff --git a/pkg/controller/component/log.go b/pkg/controller/component/log.go deleted file mode 100644 index 44acebfa..00000000 --- a/pkg/controller/component/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package component - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("controller").WithName("component") - -func getLog() logr.Logger { return _log } diff --git a/pkg/controller/component/metad_cluster.go b/pkg/controller/component/metad_cluster.go index 8cd6cb7c..f955b303 100644 --- a/pkg/controller/component/metad_cluster.go +++ b/pkg/controller/component/metad_cluster.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" "github.com/vesoft-inc/nebula-go/v3/nebula/meta" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" @@ -72,18 +73,16 @@ func (c *metadCluster) syncMetadHeadlessService(nc *v1alpha1.NebulaCluster) erro func (c *metadCluster) syncMetadWorkload(nc *v1alpha1.NebulaCluster) error { namespace := nc.GetNamespace() - ncName := nc.GetName() componentName := nc.MetadComponent().GetName() - log := getLog().WithValues("namespace", namespace, "name", ncName) gvk, err := resource.GetGVKFromDefinition(c.dm, nc.Spec.Reference) if err != nil { - return fmt.Errorf("get workload reference error: %v", err) + return fmt.Errorf("get workload kind failed: %v", err) } oldWorkloadTemp, err := c.clientSet.Workload().GetWorkload(namespace, componentName, gvk) if err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "failed to get workload") + klog.Errorf("get metad cluster failed: %v", err) return err } @@ -97,7 +96,7 @@ func (c *metadCluster) syncMetadWorkload(nc *v1alpha1.NebulaCluster) error { newWorkload, err := nc.MetadComponent().GenerateWorkload(gvk, cm, c.enableEvenPodsSpread) if err != nil { - log.Error(err, "generate workload template failed") + klog.Errorf("generate metad cluster template failed: %v", err) return err } @@ -108,7 +107,7 @@ func (c *metadCluster) syncMetadWorkload(nc *v1alpha1.NebulaCluster) error { } if err := c.syncNebulaClusterStatus(nc, oldWorkload); err != nil { - return fmt.Errorf("failed to sync metad cluster status, error: %v", err) + return fmt.Errorf("sync metad cluster status failed: %v", err) } if notExist { diff --git a/pkg/controller/component/pvc_gc.go b/pkg/controller/component/pvc_gc.go index e2fb1bd7..867b07d0 100644 --- a/pkg/controller/component/pvc_gc.go +++ b/pkg/controller/component/pvc_gc.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" @@ -31,13 +32,13 @@ import ( func PvcGc(cli client.Client, namespace, clusterName string) error { selector, err := label.New().Cluster(clusterName).Selector() if err != nil { - return fmt.Errorf("get cluster %s/%s label selector failed: %v", namespace, clusterName, err) + return fmt.Errorf("get cluster [%s/%s] label selector failed: %v", namespace, clusterName, err) } pvcClient := kube.NewPVC(cli) pvcs, err := pvcClient.ListPVCs(namespace, selector) if err != nil { - return fmt.Errorf("cluster %s/%s list pvc failed: %v", namespace, clusterName, err) + return fmt.Errorf("cluster [%s/%s] list PVC failed: %v", namespace, clusterName, err) } for i := range pvcs { @@ -54,14 +55,14 @@ func PvcGc(cli client.Client, namespace, clusterName string) error { } func PvcMark(pvcClient kube.PersistentVolumeClaim, component v1alpha1.NebulaClusterComponentter, oldReplicas, newReplicas int32) error { + ns := component.GetNamespace() componentName := component.GetName() - log := getLog().WithValues("namespace", component.GetNamespace(), "name", componentName) for i := oldReplicas - 1; i >= newReplicas; i-- { pvcNames := ordinalPVCNames(component.Type(), componentName, i) for _, pvcName := range pvcNames { pvc, err := pvcClient.GetPVC(component.GetNamespace(), pvcName) if err != nil { - return fmt.Errorf("get pvc %s for cluster %s/%s failed: %s", + return fmt.Errorf("get PVC %s for cluster [%s/%s] failed: %s", pvcName, component.GetNamespace(), component.GetClusterName(), err) } @@ -71,17 +72,17 @@ func PvcMark(pvcClient kube.PersistentVolumeClaim, component v1alpha1.NebulaClus now := time.Now().Format(time.RFC3339) pvc.Annotations[annotation.AnnPVCDeferDeletingKey] = now if err := pvcClient.UpdatePVC(pvc); err != nil { - log.Error(err, "failed to set pvc annotation", "pvcName", pvcName) + klog.Errorf("component [%s/%s] set PVC %s annotations failed: %v", ns, componentName, pvcName, err) return err } - log.Info("set pvc annotation succeed", "pvcName", pvcName) + klog.Infof("component [%s/%s] set PVC %s annotations succeed", ns, componentName, pvcName) } } return nil } func ordinalPVCNames(componentType v1alpha1.ComponentType, setName string, ordinal int32) []string { - // TODO: here need a unified function to get logPVC and dataPVC name + // Todo: here need a unified function to get logPVC and dataPVC name logPVC := fmt.Sprintf("%s-log-%s-%d", componentType, setName, ordinal) dataPVC := fmt.Sprintf("%s-data-%s-%d", componentType, setName, ordinal) if componentType == v1alpha1.GraphdComponentType { diff --git a/pkg/controller/component/reclaimer/log.go b/pkg/controller/component/reclaimer/log.go deleted file mode 100644 index d37ca728..00000000 --- a/pkg/controller/component/reclaimer/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package reclaimer - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("controller").WithName("reclaimer") - -func getLog() logr.Logger { return _log } diff --git a/pkg/controller/component/reclaimer/meta_reconciler.go b/pkg/controller/component/reclaimer/meta_reconciler.go index 5dc5da8d..e2ec518d 100644 --- a/pkg/controller/component/reclaimer/meta_reconciler.go +++ b/pkg/controller/component/reclaimer/meta_reconciler.go @@ -21,6 +21,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" "github.com/vesoft-inc/nebula-operator/pkg/controller/component" @@ -37,7 +38,6 @@ func NewMetaReconciler(clientSet kube.ClientSet) component.ReconcileManager { } func (m *meta) Reconcile(nc *v1alpha1.NebulaCluster) error { - log := getLog().WithValues("namespace", nc.Namespace, "name", nc.Name) namespace := nc.GetNamespace() clusterName := nc.GetClusterName() selector, err := label.New().Cluster(clusterName).Selector() @@ -68,7 +68,7 @@ func (m *meta) Reconcile(nc *v1alpha1.NebulaCluster) error { } pv, err := m.clientSet.PV().GetPersistentVolume(pvc.Spec.VolumeName) if err != nil { - log.Error(err, "get pv failed", "pvName", pvc.Spec.VolumeName) + klog.Errorf("cluster [%s/%s] get PV %s failed: %v", namespace, clusterName, pvc.Spec.VolumeName, err) return err } if err := m.clientSet.PV().UpdateMetaInfo(nc, pv); err != nil { @@ -83,7 +83,6 @@ func (m *meta) Reconcile(nc *v1alpha1.NebulaCluster) error { func (m *meta) resolvePVCFromPod(pod *corev1.Pod) ([]*corev1.PersistentVolumeClaim, error) { var pvcs []*corev1.PersistentVolumeClaim var pvcName string - log := getLog().WithValues("namespace", pod.Namespace, "name", pod.Name) for i := range pod.Spec.Volumes { vol := pod.Spec.Volumes[i] if vol.PersistentVolumeClaim == nil { @@ -95,7 +94,7 @@ func (m *meta) resolvePVCFromPod(pod *corev1.Pod) ([]*corev1.PersistentVolumeCla } pvc, err := m.clientSet.PVC().GetPVC(pod.Namespace, pvcName) if err != nil { - log.Error(err, "get pvc failed", "pvcName", pvcName) + klog.Error(err, "pod [%s/%s] get PVC %s failed: %v", pod.Namespace, pod.Name, pvcName, err) continue } pvcs = append(pvcs, pvc) diff --git a/pkg/controller/component/reclaimer/pvc_reclaimer.go b/pkg/controller/component/reclaimer/pvc_reclaimer.go index 1d855eac..606c5a8f 100644 --- a/pkg/controller/component/reclaimer/pvc_reclaimer.go +++ b/pkg/controller/component/reclaimer/pvc_reclaimer.go @@ -21,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" "github.com/vesoft-inc/nebula-operator/pkg/annotation" @@ -45,7 +46,6 @@ func (p *pvcReclaimer) Reclaim(nc *v1alpha1.NebulaCluster) error { } func (p *pvcReclaimer) reclaimPV(nc *v1alpha1.NebulaCluster) error { - log := getLog().WithValues("namespace", nc.Namespace, "name", nc.Name) namespace := nc.GetNamespace() ncName := nc.GetName() @@ -58,38 +58,38 @@ func (p *pvcReclaimer) reclaimPV(nc *v1alpha1.NebulaCluster) error { pvc := pvcs[i] pvcName := pvc.GetName() if !label.Label(pvc.Labels).IsNebulaComponent() { - log.V(4).Info("skip reclaim for not nebula component", "pvcName", pvcName) + klog.V(4).Infof("skip reclaim for PVC %s is not associate with nebula component", pvcName) continue } if pvc.Status.Phase != corev1.ClaimBound { - log.V(4).Info("skip reclaim for pvc status is not bound", "pvcName", pvcName) + klog.V(4).Infof("skip reclaim for PVC %s %s status is not bound", pvcName) continue } if pvc.DeletionTimestamp != nil { - log.V(4).Info("skip reclaim for pvc has been deleted", "pvcName", pvcName) + klog.V(4).Infof("skip reclaim for PVC %s has been deleted", pvcName) continue } if pvc.Annotations[annotation.AnnPVCDeferDeletingKey] == "" { - log.V(4).Info("skip reclaim for pvc has not been marked as defer deleting pvc", "pvcName", pvcName) + klog.V(4).Infof("skip reclaim for PVC %s has not been marked as defer deleting pvc", pvcName) continue } podName, exist := pvc.Annotations[annotation.AnnPodNameKey] if !exist { - log.V(4).Info("skip reclaim for pvc has no pod name annotation", "pvcName", pvcName) + klog.V(4).Infof("skip reclaim for PVC %s has no pod name annotation", pvcName) continue } _, err := p.clientSet.Pod().GetPod(namespace, podName) if err == nil { - log.V(4).Info("skip reclaim for pvc is still referenced by a pod", "pvcName", pvcName) + klog.V(4).Infof("skip reclaim for PVC %s is still referenced by a pod", pvcName) continue } if !apierrors.IsNotFound(err) { - return fmt.Errorf("cluster %s/%s get pvc %s pod %s from cache failed: %v", namespace, ncName, pvcName, podName, err) + return fmt.Errorf("cluster [%s/%s] get PVC %s pod %s from cache failed: %v", namespace, ncName, pvcName, podName, err) } pvName := pvc.Spec.VolumeName @@ -98,21 +98,21 @@ func (p *pvcReclaimer) reclaimPV(nc *v1alpha1.NebulaCluster) error { if apierrors.IsNotFound(err) { continue } - return fmt.Errorf("cluster %s/%s get pvc %s pv %s failed: %v", namespace, ncName, pvcName, pvName, err) + return fmt.Errorf("cluster [%s/%s] get PVC %s PV %s failed: %v", namespace, ncName, pvcName, pvName, err) } if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimDelete { if err := p.clientSet.PV().PatchPVReclaimPolicy(pv, corev1.PersistentVolumeReclaimDelete); err != nil { - return fmt.Errorf("cluster %s/%s patch pv %s to %s failed: %v", namespace, ncName, pvName, + return fmt.Errorf("cluster [%s/%s] patch PV %s to %s failed: %v", namespace, ncName, pvName, corev1.PersistentVolumeReclaimDelete, err) } - log.Info("patch pv policy to Delete success", "pvName", pvName) + klog.Infof("patch PV %s policy to Delete successfully", pvName) } if err := p.clientSet.PVC().DeletePVC(pvc.Namespace, pvcName); err != nil { - return fmt.Errorf("cluster %s/%s delete pvc %s failed: %v", namespace, ncName, pvcName, err) + return fmt.Errorf("cluster [%s/%s] delete PVC %s failed: %v", namespace, ncName, pvcName, err) } - log.Info("cluster reclaim pv success", "pvcName", pvName, "pvcName", pvcName) + klog.Infof("cluster [%s/%s] reclaim PV %s successfully", namespace, ncName, pvName) } return nil } @@ -123,12 +123,12 @@ func (p *pvcReclaimer) listPVCs(nc *v1alpha1.NebulaCluster) ([]corev1.Persistent selector, err := label.New().Cluster(nc.GetClusterName()).Selector() if err != nil { - return nil, fmt.Errorf("get cluster %s/%s label selector failed: %v", namespace, ncName, err) + return nil, fmt.Errorf("get cluster [%s/%s] label selector failed: %v", namespace, ncName, err) } pvcs, err := p.clientSet.PVC().ListPVCs(namespace, selector) if err != nil { - return nil, fmt.Errorf("cluster %s/%s list pvc failed: %v", namespace, ncName, err) + return nil, fmt.Errorf("cluster [%s/%s] list PVC failed: %v", namespace, ncName, err) } return pvcs, nil } diff --git a/pkg/controller/component/storaged_cluster.go b/pkg/controller/component/storaged_cluster.go index 9323f9c7..ec9cede5 100644 --- a/pkg/controller/component/storaged_cluster.go +++ b/pkg/controller/component/storaged_cluster.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" nebulago "github.com/vesoft-inc/nebula-go/v3/nebula" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" @@ -75,18 +76,16 @@ func (c *storagedCluster) syncStoragedHeadlessService(nc *v1alpha1.NebulaCluster func (c *storagedCluster) syncStoragedWorkload(nc *v1alpha1.NebulaCluster) error { namespace := nc.GetNamespace() - ncName := nc.GetName() componentName := nc.StoragedComponent().GetName() - log := getLog().WithValues("namespace", namespace, "name", ncName, "componentName", componentName) gvk, err := resource.GetGVKFromDefinition(c.dm, nc.Spec.Reference) if err != nil { - return fmt.Errorf("get workload reference error: %v", err) + return fmt.Errorf("get workload kind failed: %v", err) } oldWorkloadTemp, err := c.clientSet.Workload().GetWorkload(namespace, componentName, gvk) if err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "failed to get workload") + klog.Errorf("get storaged cluster failed: %v", err) return err } @@ -100,7 +99,7 @@ func (c *storagedCluster) syncStoragedWorkload(nc *v1alpha1.NebulaCluster) error newWorkload, err := nc.StoragedComponent().GenerateWorkload(gvk, cm, c.enableEvenPodsSpread) if err != nil { - log.Error(err, "generate workload template failed") + klog.Errorf("generate storaged cluster template failed: %v", err) return err } @@ -111,7 +110,7 @@ func (c *storagedCluster) syncStoragedWorkload(nc *v1alpha1.NebulaCluster) error } if err := c.syncNebulaClusterStatus(nc, oldWorkload); err != nil { - return fmt.Errorf("failed to sync storaged cluster status, error: %v", err) + return fmt.Errorf("sync storaged cluster status failed: %v", err) } if notExist { @@ -122,7 +121,7 @@ func (c *storagedCluster) syncStoragedWorkload(nc *v1alpha1.NebulaCluster) error return err } nc.Status.Storaged.Workload = v1alpha1.WorkloadStatus{} - return utilerrors.ReconcileErrorf("waiting for storaged cluster %s running", newWorkload.GetName()) + return utilerrors.ReconcileErrorf("waiting for storaged cluster [%s/%s] running", namespace, componentName) } oldReplicas := extender.GetReplicas(oldWorkload) @@ -138,10 +137,10 @@ func (c *storagedCluster) syncStoragedWorkload(nc *v1alpha1.NebulaCluster) error if err := c.addStorageHosts(nc, *oldReplicas, *newReplicas); err != nil { return err } - log.Info("add storage hosts succeed") + klog.Infof("storaged cluster [%s/%s] add hosts succeed", namespace, componentName) } if err := c.scaleManager.Scale(nc, oldWorkload, newWorkload); err != nil { - log.Error(err, "failed to scale cluster") + klog.Errorf("scale storaged cluster [%s/%s] failed: %v", namespace, componentName, err) return err } diff --git a/pkg/controller/component/storaged_scaler.go b/pkg/controller/component/storaged_scaler.go index 172e1f29..96edb936 100644 --- a/pkg/controller/component/storaged_scaler.go +++ b/pkg/controller/component/storaged_scaler.go @@ -22,6 +22,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" @@ -58,19 +59,20 @@ func (ss *storageScaler) Scale(nc *v1alpha1.NebulaCluster, oldUnstruct, newUnstr } func (ss *storageScaler) ScaleOut(nc *v1alpha1.NebulaCluster) error { - log := getLog().WithValues("namespace", nc.GetNamespace(), "name", nc.GetName()) + ns := nc.GetNamespace() + componentName := nc.StoragedComponent().GetName() nc.Status.Storaged.Phase = v1alpha1.ScaleOutPhase if err := ss.clientSet.NebulaCluster().UpdateNebulaClusterStatus(nc.DeepCopy()); err != nil { return err } if !nc.StoragedComponent().IsReady() { - log.Info("storage cluster status not ready", "storage", nc.StoragedComponent().GetName()) + klog.Infof("cluster [%s/%s] status not ready", ns, componentName) return nil } if !pointer.BoolPtrDerefOr(nc.Spec.Storaged.EnableAutoBalance, false) { - log.Info("auto balance is disabled", "storage", nc.StoragedComponent().GetName()) + klog.Infof("cluster [%s/%s] auto balance is disabled", ns, componentName) nc.Status.Storaged.Phase = v1alpha1.RunningPhase return nil } @@ -78,13 +80,13 @@ func (ss *storageScaler) ScaleOut(nc *v1alpha1.NebulaCluster) error { endpoints := []string{nc.GetMetadThriftConnAddress()} metaClient, err := nebula.NewMetaClient(endpoints) if err != nil { - log.Error(err, "create meta client failed", "endpoints", endpoints) + klog.Errorf("create meta client failed: %v", err) return err } defer func() { err := metaClient.Disconnect() if err != nil { - log.Error(err, "disconnect meta client failed", "endpoints", endpoints) + klog.Errorf("disconnect meta client failed: %v", err) } }() @@ -114,7 +116,9 @@ func (ss *storageScaler) ScaleOut(nc *v1alpha1.NebulaCluster) error { // nolint: revive func (ss *storageScaler) ScaleIn(nc *v1alpha1.NebulaCluster, oldReplicas, newReplicas int32) error { - log := getLog().WithValues("namespace", nc.GetNamespace(), "name", nc.GetName()) + ns := nc.GetNamespace() + ncName := nc.GetName() + componentName := nc.StoragedComponent().GetName() nc.Status.Storaged.Phase = v1alpha1.ScaleInPhase if err := ss.clientSet.NebulaCluster().UpdateNebulaClusterStatus(nc.DeepCopy()); err != nil { return err @@ -128,7 +132,7 @@ func (ss *storageScaler) ScaleIn(nc *v1alpha1.NebulaCluster, oldReplicas, newRep defer func() { err := metaClient.Disconnect() if err != nil { - log.Error(err, "meta client disconnect", "endpoints", endpoints) + klog.Error("meta client disconnect failed: %v", err) } }() @@ -162,13 +166,13 @@ func (ss *storageScaler) ScaleIn(nc *v1alpha1.NebulaCluster, oldReplicas, newRep if err := ss.removeHost(metaClient, nc, *space.Id.SpaceID, hosts); err != nil { return err } - log.Info("remove hosts successfully", "space", space.Name) + klog.Infof("cluster [%s/%s] remove hosts from space %s successfully", ns, ncName, space.Name) } } if err := metaClient.DropHosts(hosts); err != nil { return err } - log.Info("drop hosts successfully") + klog.Infof("cluster [%s/%s] drop hosts successfully", ns, ncName) } if len(spaces) > 0 && nc.Status.Storaged.BalancedSpaces == nil { @@ -207,7 +211,7 @@ func (ss *storageScaler) ScaleIn(nc *v1alpha1.NebulaCluster, oldReplicas, newRep } if nc.StoragedComponent().IsReady() { - log.Info("all used pvcs were reclaimed", "storage", nc.StoragedComponent().GetName()) + klog.Infof("cluster [%s/%s] all used pvcs were reclaimed", ns, componentName) nc.Status.Storaged.BalancedSpaces = nil nc.Status.Storaged.LastBalanceJob = nil nc.Status.Storaged.Phase = v1alpha1.RunningPhase diff --git a/pkg/controller/component/storaged_updater.go b/pkg/controller/component/storaged_updater.go index f8beb867..c86f8d46 100644 --- a/pkg/controller/component/storaged_updater.go +++ b/pkg/controller/component/storaged_updater.go @@ -24,6 +24,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" nebulago "github.com/vesoft-inc/nebula-go/v3/nebula" @@ -40,7 +41,7 @@ const ( // TransLeaderBeginTime is the key of trans Leader begin time TransLeaderBeginTime = "transLeaderBeginTime" // TransLeaderTimeout is the timeout limit of trans leader - TransLeaderTimeout = 5 * time.Minute + TransLeaderTimeout = 30 * time.Minute ) type storagedUpdater struct { @@ -57,7 +58,6 @@ func (s *storagedUpdater) Update( oldUnstruct, newUnstruct *unstructured.Unstructured, gvk schema.GroupVersionKind, ) error { - log := getLog().WithValues("namespace", nc.GetNamespace(), "name", nc.GetName()) if *nc.Spec.Storaged.Replicas == int32(0) { return nil } @@ -91,7 +91,7 @@ func (s *storagedUpdater) Update( } defer func() { if err := mc.Disconnect(); err != nil { - log.Error(err, "meta client disconnect failed") + klog.Errorf("meta client disconnect failed: %v", err) } }() @@ -125,16 +125,15 @@ func (s *storagedUpdater) updateStoragedPod( advanced bool, empty bool, ) error { - log := getLog().WithValues("namespace", nc.GetNamespace(), "name", nc.GetName()) namespace := nc.GetNamespace() updatePodName := nc.StoragedComponent().GetPodName(ordinal) updatePod, err := s.clientSet.Pod().GetPod(namespace, updatePodName) if err != nil { - log.Error(err, "failed to get pod") + klog.Errorf("cluster [%s/%s] get pod failed: %v", nc.Namespace, nc.Name, err) return err } - if empty || *nc.Spec.Storaged.Replicas < 3 { + if empty || *nc.Spec.Storaged.Replicas < 3 || nc.IsForceUpdateEnabled() { return setPartition(newUnstruct, int64(ordinal), advanced) } @@ -152,28 +151,29 @@ func (s *storagedUpdater) updateStoragedPod( } func (s *storagedUpdater) readyToUpdate(mc nebula.MetaInterface, leaderHost string, updatePod *corev1.Pod) bool { - log := getLog().WithValues("updatePod", updatePod.GetName()) + ns := updatePod.GetNamespace() + podName := updatePod.GetName() count, err := mc.GetLeaderCount(leaderHost) if err != nil { - log.Error(err, "get leader count failed") + klog.Errorf("pod [%s/%s] get leader count failed: %v", ns, podName, err) return false } if count == 0 { - log.Info("leader count is 0, ready for updating", "host", leaderHost) + klog.Infof("pod [%s/%s] leader count is 0, ready for rolling update", ns, podName) return true } if timeStr, ok := updatePod.Annotations[TransLeaderBeginTime]; ok { transLeaderBeginTime, err := time.Parse(time.RFC3339, timeStr) if err != nil { - log.Error(err, "parse time formatted string failed") + klog.Errorf("parse time formatted string failed: %v", err) return false } if time.Now().After(transLeaderBeginTime.Add(TransLeaderTimeout)) { - log.Error(err, "trans leader timeout") - return true + klog.Errorf("pod [%s/%s] transfer leader timeout", ns, podName) + return false } } - return false + return true } func (s *storagedUpdater) transLeaderIfNecessary( @@ -182,7 +182,6 @@ func (s *storagedUpdater) transLeaderIfNecessary( ordinal int32, updatePod *corev1.Pod, ) error { - log := getLog().WithValues("namespace", nc.GetNamespace(), "name", nc.GetName()) if updatePod.Annotations == nil { updatePod.Annotations = map[string]string{} } @@ -192,6 +191,7 @@ func (s *storagedUpdater) transLeaderIfNecessary( if err := s.clientSet.Pod().UpdatePod(updatePod); err != nil { return err } + klog.Infof("set pod %s annotation %v successfully", updatePod.Name, TransLeaderBeginTime) podFQDN := nc.StoragedComponent().GetPodFQDN(ordinal) endpoint := fmt.Sprintf("%s:%d", podFQDN, nc.StoragedComponent().GetPort(v1alpha1.StoragedPortNameAdmin)) @@ -202,13 +202,13 @@ func (s *storagedUpdater) transLeaderIfNecessary( defer func() { if err := sc.Disconnect(); err != nil { - log.Error(err, "storage client disconnect failed") + klog.Errorf("storage client disconnect failed: %v", err) } }() spaceItems, err := mc.GetSpaceParts() if err != nil { - log.Error(err, "get space partition failed") + klog.Errorf("cluster [%s/%s] get space partition failed: %v", nc.Namespace, nc.Name, err) return err } for spaceID, partItems := range spaceItems { @@ -234,16 +234,16 @@ func (s *storagedUpdater) transLeader( partID nebulago.PartitionID, newLeader *nebulago.HostAddr, ) error { - log := getLog().WithValues("namespace", nc.GetNamespace(), "name", nc.GetName()) if err := storageClient.TransLeader(spaceID, partID, newLeader); err != nil { return err } - log.Info("transfer leader successfully", "space", spaceID, "partition", partID) + klog.Infof("cluster [%s/%s] transfer leader spaceID %d partitionID %d to host %s successfully", + nc.Namespace, nc.Name, spaceID, partID, newLeader.Host) return nil } func (s *storagedUpdater) updateRunningPhase(mc nebula.MetaInterface, nc *v1alpha1.NebulaCluster, spaces []*meta.IdName) error { - if len(spaces) == 0 { + if len(spaces) == 0 || *nc.Spec.Storaged.Replicas == 1 { nc.Status.Storaged.Phase = v1alpha1.RunningPhase return nil } diff --git a/pkg/controller/nebulacluster/log.go b/pkg/controller/nebulacluster/log.go deleted file mode 100644 index 690f16f0..00000000 --- a/pkg/controller/nebulacluster/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nebulacluster - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("controller").WithName("nebulacluster") - -func getLog() logr.Logger { return _log } diff --git a/pkg/controller/nebulacluster/nebula_cluster_control.go b/pkg/controller/nebulacluster/nebula_cluster_control.go index ef6ba77c..615ed0ff 100644 --- a/pkg/controller/nebulacluster/nebula_cluster_control.go +++ b/pkg/controller/nebulacluster/nebula_cluster_control.go @@ -19,11 +19,14 @@ package nebulacluster import ( apiequality "k8s.io/apimachinery/pkg/api/equality" errorutils "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + "github.com/vesoft-inc/nebula-operator/pkg/annotation" "github.com/vesoft-inc/nebula-operator/pkg/controller/component" "github.com/vesoft-inc/nebula-operator/pkg/controller/component/reclaimer" "github.com/vesoft-inc/nebula-operator/pkg/kube" + utilerrors "github.com/vesoft-inc/nebula-operator/pkg/util/errors" ) type ControlInterface interface { @@ -83,29 +86,40 @@ func (c *defaultNebulaClusterControl) UpdateNebulaCluster(nc *v1alpha1.NebulaClu } func (c *defaultNebulaClusterControl) updateNebulaCluster(nc *v1alpha1.NebulaCluster) error { - log := getLog().WithValues("namespace", nc.Namespace, "name", nc.Name) if err := c.metadCluster.Reconcile(nc); err != nil { - log.Error(err, "reconcile metad cluster failed") + klog.Errorf("reconcile metad cluster failed: %v", err) return err } + if nc.IsBREnabled() && + annotation.IsRestoreNameNotEmpty(nc.GetAnnotations()) && + !annotation.IsRestoreMetadDone(nc.GetAnnotations()) { + return utilerrors.ReconcileErrorf("waiting for metad cluster restore done") + } + if err := c.storagedCluster.Reconcile(nc); err != nil { - log.Error(err, "reconcile storaged cluster failed") + klog.Errorf("reconcile storaged cluster failed: %v", err) return err } + if nc.IsBREnabled() && + annotation.IsRestoreNameNotEmpty(nc.GetAnnotations()) && + !annotation.IsRestoreStoragedDone(nc.GetAnnotations()) { + return utilerrors.ReconcileErrorf("waiting for storaged cluster restore done") + } + if err := c.graphdCluster.Reconcile(nc); err != nil { - log.Error(err, "reconcile graphd cluster failed") + klog.Errorf("reconcile graphd cluster failed: %v", err) return err } if err := c.metaReconciler.Reconcile(nc); err != nil { - log.Error(err, "reconcile pv and pvc metadata cluster failed") + klog.Errorf("reconcile pv and pvc metadata cluster failed: %v", err) return err } if err := c.pvcReclaimer.Reclaim(nc); err != nil { - log.Error(err, "reclaim pvc failed") + klog.Errorf("reclaim pvc failed: %v", err) return err } diff --git a/pkg/controller/nebulacluster/nebula_cluster_control_test.go b/pkg/controller/nebulacluster/nebula_cluster_control_test.go index f4488bf4..867f50e6 100644 --- a/pkg/controller/nebulacluster/nebula_cluster_control_test.go +++ b/pkg/controller/nebulacluster/nebula_cluster_control_test.go @@ -172,7 +172,7 @@ func newNebulaCluster() *v1alpha1.NebulaCluster { }, }, Image: "vesoft/graphd", - Version: "v3.3.0", + Version: "v3.4.0", }, }, Metad: &v1alpha1.MetadSpec{ @@ -185,7 +185,7 @@ func newNebulaCluster() *v1alpha1.NebulaCluster { }, }, Image: "vesoft/metad", - Version: "v3.3.0", + Version: "v3.4.0", }, }, Storaged: &v1alpha1.StoragedSpec{ @@ -198,7 +198,7 @@ func newNebulaCluster() *v1alpha1.NebulaCluster { }, }, Image: "vesoft/storaged", - Version: "v3.3.0", + Version: "v3.4.0", }, }, }, diff --git a/pkg/controller/nebulacluster/nebula_cluster_controller.go b/pkg/controller/nebulacluster/nebula_cluster_controller.go index 7859b13d..934d9d10 100644 --- a/pkg/controller/nebulacluster/nebula_cluster_controller.go +++ b/pkg/controller/nebulacluster/nebula_cluster_controller.go @@ -19,7 +19,6 @@ package nebulacluster import ( "context" "fmt" - "strings" "time" "github.com/go-logr/logr" @@ -29,7 +28,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" errorutils "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apiserver/pkg/registry/generic/registry" + "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -44,13 +43,12 @@ import ( ) const ( + defaultTimeout = 5 * time.Second reconcileTimeOut = 10 * time.Second KruiseReferenceName = "statefulsets.apps.kruise.io" ) -var ReconcileWaitResult = reconcile.Result{RequeueAfter: reconcileTimeOut} - // ClusterReconciler reconciles a NebulaCluster object type ClusterReconciler struct { Control ControlInterface @@ -129,8 +127,8 @@ func NewClusterReconciler(mgr ctrl.Manager, enableKruise bool) (*ClusterReconcil // +kubebuilder:rbac:groups=apps.kruise.io,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res reconcile.Result, retErr error) { - log := getLog().WithValues("namespace", req.Namespace, "name", req.Name) var nebulaCluster v1alpha1.NebulaCluster + key := req.NamespacedName.String() subCtx, cancel := context.WithTimeout(ctx, reconcileTimeOut) defer cancel() @@ -138,51 +136,49 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re defer func() { if retErr == nil { if res.Requeue || res.RequeueAfter > 0 { - log.Info("Finished reconciling nebulaCluster", "spendTime", time.Since(startTime), "result", res) + klog.Infof("Finished reconciling NebulaCluster [%s] (%v), result: %v", key, time.Since(startTime), res) } else { - log.Info("Finished reconcile nebulaCluster", "spendTime", time.Since(startTime)) + klog.Infof("Finished reconciling NebulaCluster [%s], spendTime: (%v)", key, time.Since(startTime)) } } else { - log.Error(retErr, "Failed to reconcile nebulaCluster", "spendTime", time.Since(startTime)) + klog.Errorf("Failed to reconcile NebulaCluster [%s], spendTime: (%v)", key, time.Since(startTime)) } }() if err := r.Get(subCtx, req.NamespacedName, &nebulaCluster); err != nil { if apierrors.IsNotFound(err) { - log.Info("nebula cluster does not exist") + klog.Infof("Skipping because NebulaCluster [%s] has been deleted", key) if err := component.PvcGc(r.Client, req.Namespace, req.Name); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } } return ctrl.Result{}, client.IgnoreNotFound(err) } - log.Info("Start to reconcile") + + klog.Info("Start to reconcile NebulaCluster") if !r.EnableKruise && nebulaCluster.Spec.Reference.Name == KruiseReferenceName { - return ctrl.Result{}, errorsutil.ReconcileErrorf("openkruise scheme not registered") + return ctrl.Result{}, fmt.Errorf("openkruise scheme not registered") } if err := r.syncNebulaCluster(nebulaCluster.DeepCopy()); err != nil { - if strings.Contains(err.Error(), registry.OptimisticLockErrorMsg) { - return ReconcileWaitResult, nil - } - isReconcileError := func(err error) (b bool) { defer func() { if b { - log.Info("reconcile failed", "error", err) + klog.Infof("NebulaCluster [%s] reconcile details: %v", key, err) } }() return errorsutil.IsReconcileError(err) } - err := errorutils.FilterOut(err, isReconcileError, errorsutil.IsStatusError) + err := errorutils.FilterOut(err, isReconcileError, errorsutil.IsDNSError, errorsutil.IsStatusError) if err == nil { - log.Info("nebula cluster need reconcile") - return ReconcileWaitResult, nil + return ctrl.Result{RequeueAfter: reconcileTimeOut}, nil } - log.Error(err, "reconcile nebula cluster failed") - return ReconcileWaitResult, nil + + klog.Errorf("NebulaCluster [%s] reconcile failed: %v", key, err) + + return ctrl.Result{RequeueAfter: defaultTimeout}, nil } return ctrl.Result{}, nil } diff --git a/pkg/controller/nebularestore/nebula_restore_control.go b/pkg/controller/nebularestore/nebula_restore_control.go new file mode 100644 index 00000000..66cac3ca --- /dev/null +++ b/pkg/controller/nebularestore/nebula_restore_control.go @@ -0,0 +1,124 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nebularestore + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + "github.com/vesoft-inc/nebula-operator/pkg/kube" + "github.com/vesoft-inc/nebula-operator/pkg/label" + "github.com/vesoft-inc/nebula-operator/pkg/util/condition" + "github.com/vesoft-inc/nebula-operator/pkg/util/errors" +) + +type ControlInterface interface { + UpdateNebulaRestore(rt *v1alpha1.NebulaRestore) error +} + +var _ ControlInterface = (*defaultRestoreControl)(nil) + +type defaultRestoreControl struct { + clientSet kube.ClientSet + restoreManager Manager +} + +func NewRestoreControl(clientSet kube.ClientSet, restoreManager Manager) ControlInterface { + return &defaultRestoreControl{ + clientSet: clientSet, + restoreManager: restoreManager, + } +} + +func (c *defaultRestoreControl) UpdateNebulaRestore(rt *v1alpha1.NebulaRestore) error { + ns := rt.GetNamespace() + name := rt.GetName() + + if condition.IsRestoreInvalid(rt) { + klog.Infof("Skipping because NebulaRestore [%s/%s] is invalid.", ns, name) + return nil + } + + if condition.IsRestoreComplete(rt) { + klog.Infof("Skipping because NebulaRestore [%s/%s] is complete.", ns, name) + return nil + } + + if condition.IsRestoreFailed(rt) { + klog.Infof("Skipping because NebulaRestore [%s/%s] is failed.", ns, name) + return nil + } + + if rt.Status.ClusterName != "" { + selector, err := label.New().Cluster(rt.Status.ClusterName).Selector() + if err != nil { + klog.Errorf("Fail to generate selector for NebulaCluster [%s/%s], %v", ns, rt.Status.ClusterName, err) + return nil + } + pods, err := c.clientSet.Pod().ListPods(ns, selector) + if err != nil { + klog.Errorf("Fail to list pod for NebulaCluster [%s/%s] with selector %s, %v", ns, rt.Status.ClusterName, selector, err) + return nil + } + for _, pod := range pods { + if pod.Status.Phase == corev1.PodFailed { + klog.Infof("NebulaCluster [%s/%s] has failed pod %s.", ns, name, pod.Name) + if err := c.restoreManager.UpdateCondition(rt, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "PodFailed", + Message: fmt.Sprintf("Pod %s has failed", pod.Name), + }); err != nil { + klog.Errorf("Fail to update the condition of NebulaRestore [%s/%s], %v", ns, name, err) + } + if err := c.deleteRestoredCluster(ns, rt.Status.ClusterName); err != nil { + klog.Errorf("Fail to delete NebulaCluster [%s/%s], %v", ns, rt.Status.ClusterName, err) + } + return nil + } + } + } + + err := c.restoreManager.Sync(rt) + if err != nil && !errors.IsReconcileError(err) { + if err := c.restoreManager.UpdateCondition(rt, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ExecuteFailed", + Message: err.Error(), + }); err != nil { + klog.Errorf("Fail to update the condition of NebulaRestore [%s/%s], %v", ns, name, err) + } + updated, err := c.clientSet.NebulaRestore().GetNebulaRestore(ns, rt.Name) + if err != nil { + klog.Error("Fail to get NebulaRestore [%s/%s], %v", ns, name, err) + } + if err := c.deleteRestoredCluster(ns, updated.Status.ClusterName); err != nil { + klog.Errorf("Fail to delete NebulaCluster %v", err) + } + return nil + } + + return err +} + +func (c *defaultRestoreControl) deleteRestoredCluster(namespace, ncName string) error { + return c.clientSet.NebulaCluster().DeleteNebulaCluster(namespace, ncName) +} diff --git a/pkg/controller/nebularestore/nebula_restore_controller.go b/pkg/controller/nebularestore/nebula_restore_controller.go new file mode 100644 index 00000000..876e0b7f --- /dev/null +++ b/pkg/controller/nebularestore/nebula_restore_controller.go @@ -0,0 +1,120 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nebularestore + +import ( + "context" + "time" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + "github.com/vesoft-inc/nebula-operator/pkg/kube" + errorsutil "github.com/vesoft-inc/nebula-operator/pkg/util/errors" +) + +const ( + defaultTimeout = 5 * time.Second + reconcileTimeOut = 10 * time.Second +) + +var _ reconcile.Reconciler = (*Reconciler)(nil) + +// Reconciler reconciles a NebulaRestore object +type Reconciler struct { + Control ControlInterface + client.Client + Log logr.Logger +} + +func NewRestoreReconciler(mgr ctrl.Manager) (*Reconciler, error) { + clientSet, err := kube.NewClientSet(mgr.GetConfig()) + if err != nil { + return nil, err + } + + restoreMgr := NewRestoreManager(clientSet) + + return &Reconciler{ + Control: NewRestoreControl(clientSet, restoreMgr), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("NebulaRestore"), + }, nil +} + +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch;list +// +kubebuilder:rbac:groups=apps.nebula-graph.io,resources=restores/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.nebula-graph.io,resources=restores/finalizers,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.nebula-graph.io,resources=restores,verbs=get;list;watch;create;update;patch;delete + +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (res reconcile.Result, retErr error) { + var restore v1alpha1.NebulaRestore + key := req.NamespacedName.String() + subCtx, cancel := context.WithTimeout(ctx, reconcileTimeOut) + defer cancel() + + startTime := time.Now() + defer func() { + if retErr == nil { + if res.Requeue || res.RequeueAfter > 0 { + klog.Infof("Finished reconciling NebulaRestore [%s] (%v), result: %v", key, time.Since(startTime), res) + } else { + klog.Infof("Finished reconciling NebulaRestore [%s], spendTime: (%v)", key, time.Since(startTime)) + } + } else { + klog.Errorf("Failed to reconcile NebulaRestore [%s], spendTime: (%v)", key, time.Since(startTime)) + } + }() + + if err := r.Get(subCtx, req.NamespacedName, &restore); err != nil { + if apierrors.IsNotFound(err) { + klog.Infof("Skipping because NebulaRestore [%s] has been deleted", key) + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + klog.Info("Start to reconcile NebulaRestore") + + if err := r.syncNebulaRestore(restore.DeepCopy()); err != nil { + if errorsutil.IsReconcileError(err) { + klog.Infof("NebulaRestore [%s] reconcile details: %v", key, err) + return ctrl.Result{RequeueAfter: reconcileTimeOut}, nil + } + klog.Errorf("NebulaRestore [%s] reconcile failed: %v", key, err) + return ctrl.Result{RequeueAfter: defaultTimeout}, nil + } + + return ctrl.Result{}, nil +} + +func (r *Reconciler) syncNebulaRestore(restore *v1alpha1.NebulaRestore) error { + return r.Control.UpdateNebulaRestore(restore) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.NebulaRestore{}). + WithOptions(opts). + Complete(r) +} diff --git a/pkg/controller/nebularestore/nebula_restore_manager.go b/pkg/controller/nebularestore/nebula_restore_manager.go new file mode 100644 index 00000000..1d511192 --- /dev/null +++ b/pkg/controller/nebularestore/nebula_restore_manager.go @@ -0,0 +1,814 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nebularestore + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + pb "github.com/vesoft-inc/nebula-agent/pkg/proto" + "github.com/vesoft-inc/nebula-agent/pkg/storage" + ng "github.com/vesoft-inc/nebula-go/v3/nebula" + "github.com/vesoft-inc/nebula-go/v3/nebula/meta" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + "github.com/vesoft-inc/nebula-operator/pkg/annotation" + "github.com/vesoft-inc/nebula-operator/pkg/kube" + "github.com/vesoft-inc/nebula-operator/pkg/nebula" + "github.com/vesoft-inc/nebula-operator/pkg/util/async" + "github.com/vesoft-inc/nebula-operator/pkg/util/condition" + utilerrors "github.com/vesoft-inc/nebula-operator/pkg/util/errors" + "github.com/vesoft-inc/nebula-operator/pkg/util/randstr" + rtutil "github.com/vesoft-inc/nebula-operator/pkg/util/restore" +) + +const ( + S3AccessKey = "access-key" + S3SecretKey = "secret-key" +) + +type Restore struct { + ctx context.Context + cfg *rtutil.Config + sto storage.ExternalStorage + agentMgr *nebula.AgentManager + + // bakMetas store backup meta list for restore + bakMetas []*meta.BackupMeta + + rootURI string + backupName string + backSuffix string + + metaDir *ng.DirInfo + storageHosts []*meta.ServiceInfo +} + +type Manager interface { + // Sync implements the logic for syncing NebulaRestore. + Sync(restore *v1alpha1.NebulaRestore) error + + // UpdateCondition updates the condition for a NebulaRestore. + UpdateCondition(restore *v1alpha1.NebulaRestore, condition *v1alpha1.RestoreCondition) error +} + +var _ Manager = (*restoreManager)(nil) + +type restoreManager struct { + clientSet kube.ClientSet + restore *Restore +} + +func NewRestoreManager(clientSet kube.ClientSet) Manager { + return &restoreManager{clientSet: clientSet} +} + +func (rm *restoreManager) Sync(restore *v1alpha1.NebulaRestore) error { + return rm.syncRestoreProcess(restore) +} + +func (rm *restoreManager) UpdateCondition(restore *v1alpha1.NebulaRestore, condition *v1alpha1.RestoreCondition) error { + return rm.clientSet.NebulaRestore().UpdateNebulaRestoreStatus(restore, condition, nil) +} + +// syncRestoreProcess restores nebula cluster which storage is one engine for one part +/* +backup_root/backup_name + - meta + - xxx.sst + - ... + - data + - spaceId + - partId + - ... + - ... + - backup_name.meta +*/ + +func (rm *restoreManager) syncRestoreProcess(rt *v1alpha1.NebulaRestore) error { + var ready bool + ns := rt.GetNamespace() + originalName := rt.Spec.BR.ClusterName + if rt.Spec.BR.ClusterNamespace != nil { + ns = *rt.Spec.BR.ClusterNamespace + } + + restoredName, err := rm.getRestoredName(rt) + if err != nil { + return err + } + + nc, _ := rm.clientSet.NebulaCluster().GetNebulaCluster(ns, restoredName) + if nc != nil && annotation.IsInRestoreStage2(nc.Annotations) { + if !isReady(nc.Status.Conditions) { + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage2, waiting for cluster ready", ns, restoredName) + } + nc.Annotations = nil + if err := rm.clientSet.NebulaCluster().UpdateNebulaCluster(nc); err != nil { + klog.Error("remove cluster [%s/%s] annotations failed: %v", ns, restoredName, err) + } + + return rm.clientSet.NebulaRestore().UpdateNebulaRestoreStatus(rt, + &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreComplete, + Status: corev1.ConditionTrue, + }, &kube.RestoreUpdateStatus{ + TimeCompleted: &metav1.Time{Time: time.Now()}, + ConditionType: v1alpha1.RestoreComplete, + Partitions: nil, + Checkpoints: nil}) + } + + original, err := rm.clientSet.NebulaCluster().GetNebulaCluster(ns, originalName) + if err != nil { + return err + } + + restored := rm.genNebulaCluster(restoredName, rt, original) + if err := rm.clientSet.NebulaCluster().CreateNebulaCluster(restored); err != nil { + return err + } + + if err := rm.initRestore(rt); err != nil { + return err + } + + if err := rm.loadCluster(original, restored); err != nil { + return err + } + + if err := os.MkdirAll(rtutil.LocalTmpDir, os.ModePerm); err != nil { + return err + } + + if err := rm.restore.loadBakMetas(rm.restore.backupName); err != nil { + return err + } + klog.Infof("[%s/%s] load backup metad file successfully", ns, restoredName) + + if err := rm.restore.checkTopology(rm.restore.bakMetas[0], rm.restore.storageHosts); err != nil { + return err + } + + if !condition.IsRestoreMetadComplete(rt) { + if !rm.endpointsConnected(restored.GetMetadEndpoints()) { + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage1, waiting for metad init agent are connected", ns, restoredName) + } + + if err := rm.restore.downloadMetaData(restored.GetMetadEndpoints()); err != nil { + klog.Errorf("download metad files failed: %v", err) + return err + } + + klog.Infof("restoring [%s/%s] in stage1, download metad files successfully", ns, restoredName) + + ready, err = rm.metadReady(ns, restoredName) + if err != nil { + return err + } + if !ready { + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage1, waiting for metad running", ns, restoredName) + } + + hostPairs := rm.restore.genHostPairs(rm.restore.bakMetas[0], restored.GetStoragedEndpoints()) + restoreResp, err := rm.restore.restoreMeta(rm.restore.bakMetas[0], hostPairs, restored.GetMetadEndpoints()) + if err != nil { + klog.Errorf("restore metad data [%s/%s] failed, error: %v", ns, restoredName, err) + return err + } + + klog.Infof("restoring [%s/%s] in stage1, restore metad data successfully", ns, restoredName) + + hostPartMap := rtutil.FlattenRestoreMeta(restoreResp) + + rtutil.FlattenRestoreMeta(restoreResp) + + if err := rm.clientSet.NebulaRestore().UpdateNebulaRestoreStatus(rt, + &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreMetadComplete, + Status: corev1.ConditionTrue, + }, &kube.RestoreUpdateStatus{ + Partitions: hostPartMap, + ConditionType: v1alpha1.RestoreMetadComplete}); err != nil { + return err + } + } + + if err := rm.updateClusterAnnotations(ns, + restoredName, + map[string]string{annotation.AnnRestoreMetadStepKey: annotation.AnnRestoreMetadStepVal}); err != nil { + return err + } + + if !condition.IsRestoreStoragedComplete(rt) { + if !rm.endpointsConnected(restored.GetStoragedEndpoints()) { + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage1, waiting for storaged init agent are connected", ns, restoredName) + } + + checkpoints, err := rm.restore.downloadStorageData(rt.Status.Partitions, rm.restore.storageHosts) + if err != nil { + klog.Errorf("download storaged files failed: %v", err) + return err + } + + klog.Infof("restoring [%s/%s] in stage1, download storaged files successfully", ns, restoredName) + + if err := rm.restore.playBackStorageData(restored.GetMetadEndpoints(), rm.restore.storageHosts); err != nil { + return err + } + + klog.Infof("restoring [%s/%s] in stage1, playback storaged data successfully", ns, restoredName) + + if err := rm.clientSet.NebulaRestore().UpdateNebulaRestoreStatus(rt, + &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreStoragedCompleted, + Status: corev1.ConditionTrue, + }, &kube.RestoreUpdateStatus{ + Checkpoints: checkpoints, + ConditionType: v1alpha1.RestoreStoragedCompleted}); err != nil { + return err + } + } + + if err := rm.updateClusterAnnotations(ns, + restoredName, + map[string]string{annotation.AnnRestoreStoragedStepKey: annotation.AnnRestoreStoragedStepVal}); err != nil { + return err + } + + ready, err = rm.clusterReady(ns, restoredName) + if err != nil { + return err + } + if !ready { + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage1, waiting for cluster ready", ns, restoredName) + } + + if !rm.endpointsConnected(restored.GetStoragedEndpoints()) { + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage1, waiting for storaged sidecar agent are connected", ns, restoredName) + } + + if err := rm.restore.removeDownloadCheckpoints(rt.Status.Checkpoints); err != nil { + klog.Errorf("remove downloaded checkpoints failed: %v", err) + return err + } + klog.Infof("restoring [%s/%s] in stage1, remove download checkpoints successfully", ns, restoredName) + + if err := rm.removeInitAgentContainer(ns, restoredName); err != nil { + klog.Errorf("remove init agent containers failed: %v", err) + return err + } + klog.Infof("restoring [%s/%s] in stage1, remove init agent container successfully", ns, restoredName) + + return utilerrors.ReconcileErrorf("restoring [%s/%s] in stage2, waiting for cluster ready", ns, restoredName) +} + +func (rm *restoreManager) loadCluster(original, restored *v1alpha1.NebulaCluster) error { + mc, err := nebula.NewMetaClient([]string{original.GetMetadThriftConnAddress()}) + if err != nil { + return err + } + + resp, err := mc.ListCluster() + if err != nil { + return err + } + + hosts := &rtutil.NebulaHosts{} + if err := hosts.LoadFrom(resp); err != nil { + return err + } + + rm.restore.metaDir = hosts.GetMetas()[0].Dir + rm.replaceStorageHosts(hosts.GetStorages(), restored) + + klog.Infof("restore metad dir info, root: %s data: %s", string(rm.restore.metaDir.Root), string(rm.restore.metaDir.Data[0])) + + return nil +} + +func (rm *restoreManager) replaceStorageHosts(original []*meta.ServiceInfo, restored *v1alpha1.NebulaCluster) { + for i := range original { + original[i].Addr.Host = restored.StoragedComponent().GetPodFQDN(int32(i)) + } + rm.restore.storageHosts = original +} + +func (rm *restoreManager) genNebulaCluster(restoredName string, rt *v1alpha1.NebulaRestore, original *v1alpha1.NebulaCluster) *v1alpha1.NebulaCluster { + annotations := map[string]string{ + annotation.AnnRestoreNameKey: rt.Name, + annotation.AnnRestoreStageKey: annotation.AnnRestoreStage1Val, + } + nc := &v1alpha1.NebulaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: restoredName, + Namespace: original.Namespace, + Annotations: annotations, + }, + Spec: original.Spec, + } + + nc.Spec.Metad.InitContainers = append(nc.Spec.Metad.InitContainers, v1alpha1.GenerateInitAgentContainer(nc.MetadComponent())) + nc.Spec.Storaged.InitContainers = append(nc.Spec.Storaged.InitContainers, v1alpha1.GenerateInitAgentContainer(nc.StoragedComponent())) + + nc.Spec.Storaged.EnableForceUpdate = pointer.Bool(true) + if rt.Spec.NodeSelector != nil { + nc.Spec.NodeSelector = rt.Spec.NodeSelector + } + + return nc +} + +func (rm *restoreManager) initRestore(restore *v1alpha1.NebulaRestore) error { + var err error + backend := &pb.Backend{} + if err := backend.SetUri(fmt.Sprintf("s3://%s", restore.Spec.BR.S3.Bucket)); err != nil { + return err + } + backend.GetS3().Region = restore.Spec.BR.S3.Region + backend.GetS3().Endpoint = restore.Spec.BR.S3.Endpoint + accessKey, secretKey, err := rm.getS3Key(restore.Namespace, restore.Spec.BR.S3.SecretName) + if err != nil { + return fmt.Errorf("get S3 key failed: %v", err) + } + backend.GetS3().AccessKey = accessKey + backend.GetS3().SecretKey = secretKey + + cfg := &rtutil.Config{ + BackupName: restore.Spec.BR.BackupName, + Concurrency: restore.Spec.BR.Concurrency, + Backend: backend, + } + + if rm.restore == nil { + if rm.restore, err = newRestore(cfg); err != nil { + return err + } + } + + return nil +} + +func newRestore(cfg *rtutil.Config) (*Restore, error) { + sto, err := storage.New(cfg.Backend) + if err != nil { + return nil, err + } + + return &Restore{ + ctx: context.TODO(), + cfg: cfg, + sto: sto, + agentMgr: nebula.NewAgentManager(), + rootURI: cfg.Backend.Uri(), + backupName: cfg.BackupName, + }, nil +} + +func (r *Restore) checkTopology(bak *meta.BackupMeta, storageHosts []*meta.ServiceInfo) error { + if len(storageHosts) != len(bak.GetStorageHosts()) { + return fmt.Errorf("the cluster topology of storaged count must be consistent") + } + + return nil +} + +func (r *Restore) loadBakMetas(backupName string) error { + // check backup dir existence + rootURI, err := rtutil.UriJoin(r.cfg.Backend.Uri(), backupName) + if err != nil { + return err + } + exist := r.sto.ExistDir(r.ctx, rootURI) + if !exist { + return fmt.Errorf("backup dir %s does not exist", rootURI) + } + + // download and parse backup meta file + backupMetaName := fmt.Sprintf("%s.meta", backupName) + metaUri, _ := rtutil.UriJoin(rootURI, backupMetaName) + tmpPath := filepath.Join(rtutil.LocalTmpDir, backupMetaName) + if err := r.sto.Download(r.ctx, tmpPath, metaUri, false); err != nil { + return fmt.Errorf("download %s to %s failed: %v", metaUri, tmpPath, err) + } + + bakMeta, err := rtutil.ParseMetaFromFile(tmpPath) + if err != nil { + return fmt.Errorf("parse backup metad file %s failed: %v", tmpPath, err) + } + + r.bakMetas = append(r.bakMetas, bakMeta) + + if len(bakMeta.BaseBackupName) > 0 { + return r.loadBakMetas(string(bakMeta.BaseBackupName)) + } + + return nil +} + +func (r *Restore) downloadMetaData(metaEndpoints []string) error { + // {backupRoot}/{backupName}/meta/*.sst + externalUri, _ := rtutil.UriJoin(r.rootURI, r.backupName, "meta") + backend, err := r.sto.GetDir(r.ctx, externalUri) + if err != nil { + return err + } + + // download meta backup files to every meta service + for _, ep := range metaEndpoints { + addr, err := rtutil.ParseAddr(ep) + if err != nil { + return err + } + agent, err := r.agentMgr.GetAgent(addr) + if err != nil { + return err + } + + // meta kv data path: {nebulaData}/meta + req := &pb.DownloadFileRequest{ + SourceBackend: backend, + TargetPath: string(r.metaDir.Data[0]), + Recursively: true, + } + if _, err := agent.DownloadFile(req); err != nil { + return err + } + if _, err := agent.StopAgent(&pb.StopAgentRequest{}); err != nil { + return err + } + } + + return nil +} + +func (r *Restore) downloadStorageData(parts map[string][]*ng.HostAddr, storageHosts []*meta.ServiceInfo) (map[string]map[string]string, error) { + // checkpoints save the download checkpoint paths, make cleanup restore data easier + checkpoints := make(map[string]map[string]string) + + // dataPathSelector is a selector map which select current hosts' dataPath on average + dataPathSelector := rtutil.NewPathSelectorMap(storageHosts) + dataPathMap := make(map[string]string) + group := async.NewGroup(context.TODO(), r.cfg.Concurrency) + for _, backupMeta := range r.bakMetas { + storageUri, _ := rtutil.UriJoin(r.rootURI, string(backupMeta.GetBackupName()), "data") + partMap := rtutil.FlattenBackupMeta(backupMeta) + for _, info := range partMap { + // restore one part to multiple part + key := rtutil.GenPartKey(info.SpaceID, info.PartID) + for _, host := range parts[key] { + agent, err := r.agentMgr.GetAgent(host) + if err != nil { + return nil, err + } + + externalUri, _ := rtutil.UriJoin(storageUri, info.SpaceID, info.PartID) + // avoid agent.DownloadFile prefix bugs + externalUri += "/" + + source, err := r.sto.GetDir(r.ctx, externalUri) + if err != nil { + return nil, err + } + + // ensure every part's all checkpoint place in same dataPath + dataPathKey := rtutil.GenDataPathKey(host, key) + dataPath, hasPlace := dataPathMap[dataPathKey] + if !hasPlace { + dataPath = dataPathSelector[rtutil.StringifyAddr(host)].EvenlyGet() + dataPathMap[dataPathKey] = dataPath + } + + targetCk := filepath.Join(dataPath, "nebula", info.SpaceID, info.PartID, "checkpoints") + target := filepath.Join(targetCk, string(backupMeta.BackupName)) + + // source: {backupRoot}/{backupName}/data/{spaceID}/{partID}/ + // target: {nebulaDataPath}/nebula/{spaceID}/{partID}/checkpoints/{backupName}/ + req := &pb.DownloadFileRequest{ + SourceBackend: source, + TargetPath: target, + Recursively: true, + } + + if _, ok := checkpoints[rtutil.StringifyAddr(host)]; !ok { + checkpoints[rtutil.StringifyAddr(host)] = make(map[string]string) + } + checkpoints[rtutil.StringifyAddr(host)][targetCk] = "" + + worker := func() error { + if _, err := agent.DownloadFile(req); err != nil { + return err + } + return nil + } + + group.Add(func(stopCh chan interface{}) { + stopCh <- worker() + }) + } + } + } + + return checkpoints, group.Wait() +} + +func (r *Restore) restoreMeta(backup *meta.BackupMeta, storageMap map[string]string, metaEndpoints []string) (*meta.RestoreMetaResp, error) { + addrMap := make([]*meta.HostPair, 0, len(storageMap)) + for from, to := range storageMap { + fromAddr, err := rtutil.ParseAddr(from) + if err != nil { + return nil, err + } + toAddr, err := rtutil.ParseAddr(to) + if err != nil { + return nil, err + } + addrMap = append(addrMap, &meta.HostPair{FromHost: fromAddr, ToHost: toAddr}) + } + + var restoreRes *meta.RestoreMetaResp + for _, metaEndpoint := range metaEndpoints { + files := make([]string, 0, len(backup.GetMetaFiles())) + for _, file := range backup.GetMetaFiles() { + filePath := fmt.Sprintf("%s/%s", string(r.metaDir.Data[0]), string(file)) + files = append(files, filePath) + } + + mc, err := nebula.NewMetaClient([]string{metaEndpoint}) + if err != nil { + if utilerrors.IsDNSError(err) { + return nil, utilerrors.ReconcileErrorf("waiting for %s dns lookup is ok", metaEndpoint) + } + return nil, err + } + + resp, err := mc.RestoreMeta(addrMap, files) + klog.Infof("restore metad sst files %v", files) + if err != nil { + return nil, fmt.Errorf("restore metad service %s failed: %v", metaEndpoint, err) + } + if restoreRes == nil { + restoreRes = resp + } + + if err := mc.Disconnect(); err != nil { + klog.Error(err) + } + + klog.Infof("restore backup data on metad %s successfully", metaEndpoint) + } + + return restoreRes, nil +} + +// genHostPairs generate old:new storage host pairs +func (r *Restore) genHostPairs(backup *meta.BackupMeta, restoreHosts []string) map[string]string { + hostPairs := make(map[string]string) + backupHosts := make([]string, 0) + + for _, st := range backup.GetStorageHosts() { + addr := rtutil.StringifyAddr(st) + backupHosts = append(backupHosts, addr) + } + + for i := range restoreHosts { + hostPairs[backupHosts[i]] = restoreHosts[i] + } + + klog.Infof("restore meta hostPairs %v", hostPairs) + + return hostPairs +} + +func (r *Restore) playBackStorageData(metaEndpoints []string, storageHosts []*meta.ServiceInfo) error { + group := async.NewGroup(context.TODO(), r.cfg.Concurrency) + for _, s := range storageHosts { + agent, err := r.agentMgr.GetAgent(s.GetAddr()) + if err != nil { + return err + } + + dataPaths := make([]string, 0) + for _, d := range s.GetDir().GetData() { + dataPaths = append(dataPaths, string(d)) + } + + req := &pb.DataPlayBackRequest{ + Dir: string(s.GetDir().GetRoot()), + DataPath: strings.Join(dataPaths, ","), + MetaAddr: strings.Join(metaEndpoints, ","), + } + + worker := func() error { + _, err = agent.DataPlayBack(req) + if err != nil { + return fmt.Errorf("storaged data playback failed: %v", err) + } + klog.Infof("backup data playback on storaged %s successfully", s.GetAddr().GetHost()) + return nil + } + + group.Add(func(stopCh chan interface{}) { + stopCh <- worker() + }) + } + + if err := group.Wait(); err != nil { + return err + } + + for _, s := range storageHosts { + agent, err := r.agentMgr.GetAgent(s.GetAddr()) + if err != nil { + return err + } + if _, err := agent.StopAgent(&pb.StopAgentRequest{}); err != nil { + return fmt.Errorf("stop agent failed: %v", err) + } + } + + return nil +} + +func (r *Restore) removeDownloadCheckpoints(checkpoints map[string]map[string]string) error { + for addr, paths := range checkpoints { + host, err := rtutil.ParseAddr(addr) + if err != nil { + return err + } + agent, err := r.agentMgr.GetAgent(host) + if err != nil { + return fmt.Errorf("get agent for storaged %s failed: %v", addr, err) + } + for path := range paths { + if _, err = agent.RemoveDir(&pb.RemoveDirRequest{Path: path}); err != nil { + return err + } + } + } + + return nil +} + +func (rm *restoreManager) removeInitAgentContainer(namespace, ncName string) error { + updated, err := rm.clientSet.NebulaCluster().GetNebulaCluster(namespace, ncName) + if err != nil { + return err + } + + updated.SetAnnotations(map[string]string{annotation.AnnRestoreStageKey: annotation.AnnRestoreStage2Val}) + updated.Spec.Storaged.EnableForceUpdate = pointer.Bool(false) + + m := make([]corev1.Container, 0) + for _, c := range updated.Spec.Metad.InitContainers { + if c.Name == v1alpha1.AgentInitContainerName { + continue + } + m = append(m, c) + } + updated.Spec.Metad.InitContainers = m + + s := make([]corev1.Container, 0) + for _, c := range updated.Spec.Storaged.InitContainers { + if c.Name == v1alpha1.AgentInitContainerName { + continue + } + s = append(s, c) + } + updated.Spec.Storaged.InitContainers = s + + return rm.clientSet.NebulaCluster().UpdateNebulaCluster(updated) +} + +func (rm *restoreManager) updateClusterAnnotations(namespace, ncName string, annotations map[string]string) error { + updated, err := rm.clientSet.NebulaCluster().GetNebulaCluster(namespace, ncName) + if err != nil { + return err + } + + var needUpdate bool + if updated.GetAnnotations() == nil { + updated.SetAnnotations(annotations) + needUpdate = true + } else { + for annKey, annVal := range annotations { + v, exists := updated.Annotations[annKey] + if exists && annVal == v { + continue + } + updated.Annotations[annKey] = annVal + needUpdate = true + } + } + + if needUpdate { + klog.Infof("NebulaCluster %s/%s will update annotations %v", namespace, ncName, annotations) + return rm.clientSet.NebulaCluster().UpdateNebulaCluster(updated) + } + + return nil + +} + +func (rm *restoreManager) endpointsConnected(endpoints []string) bool { + for _, ep := range endpoints { + host, err := rtutil.ParseAddr(ep) + if err != nil { + klog.Error(err) + return false + } + agent, err := rm.restore.agentMgr.GetAgent(host) + if err != nil { + klog.Error(err) + return false + } + resp, err := agent.HealthCheck(&pb.HealthCheckRequest{}) + if err != nil { + return false + } + if resp != nil && resp.Status != "healthy" { + return false + } + } + + return true +} + +func (rm *restoreManager) metadReady(namespace, ncName string) (bool, error) { + nc, err := rm.clientSet.NebulaCluster().GetNebulaCluster(namespace, ncName) + if err != nil { + return false, err + } + return nc.MetadComponent().IsReady(), nil +} + +func (rm *restoreManager) clusterReady(namespace, ncName string) (bool, error) { + nc, err := rm.clientSet.NebulaCluster().GetNebulaCluster(namespace, ncName) + if err != nil { + return false, err + } + return isReady(nc.Status.Conditions), nil +} + +func (rm *restoreManager) getRestoredName(rt *v1alpha1.NebulaRestore) (string, error) { + if rt.Status.ClusterName == "" { + genName := "ng" + randstr.Hex(4) + newStatus := &kube.RestoreUpdateStatus{ + TimeStarted: &metav1.Time{Time: time.Now()}, + ClusterName: pointer.String(genName), + } + if err := rm.clientSet.NebulaRestore().UpdateNebulaRestoreStatus(rt, nil, newStatus); err != nil { + return "", err + } + + klog.Infof("generate restored nebula cluster name successfully") + return genName, nil + } + + return rt.Status.ClusterName, nil +} + +func (rm *restoreManager) getS3Key(namespace, secretName string) (accessKey string, secretKey string, err error) { + var secret *corev1.Secret + secret, err = rm.clientSet.Secret().GetSecret(namespace, secretName) + if err != nil { + return + } + accessKey = string(secret.Data[S3AccessKey]) + secretKey = string(secret.Data[S3SecretKey]) + + return +} + +func isReady(conditions []v1alpha1.NebulaClusterCondition) bool { + for i := range conditions { + c := conditions[i] + if c.Type == v1alpha1.NebulaClusterReady && + c.Status == corev1.ConditionTrue { + return true + } + } + return false +} diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 3ba849cb..703f19df 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -23,6 +23,7 @@ import ( ) type ClientSet interface { + Secret() Secret ConfigMap() ConfigMap PV() PersistentVolume PVC() PersistentVolumeClaim @@ -32,9 +33,11 @@ type ClientSet interface { Ingress() Ingress Workload() Workload NebulaCluster() NebulaCluster + NebulaRestore() NebulaRestore } type clientSet struct { + secretClient Secret cmClient ConfigMap pvClient PersistentVolume pvcClient PersistentVolumeClaim @@ -44,6 +47,7 @@ type clientSet struct { ingressClient Ingress workloadClient Workload nebulaClient NebulaCluster + restoreClient NebulaRestore } func NewClientSet(config *rest.Config) (ClientSet, error) { @@ -52,6 +56,7 @@ func NewClientSet(config *rest.Config) (ClientSet, error) { return nil, errors.Errorf("error building runtime client: %v", err) } return &clientSet{ + secretClient: NewSecret(cli), cmClient: NewConfigMap(cli), pvClient: NewPV(cli), pvcClient: NewPVC(cli), @@ -61,9 +66,14 @@ func NewClientSet(config *rest.Config) (ClientSet, error) { ingressClient: NewIngress(cli), workloadClient: NewWorkload(cli), nebulaClient: NewNebulaCluster(cli), + restoreClient: NewNebulaRestore(cli), }, nil } +func (c *clientSet) Secret() Secret { + return c.secretClient +} + func (c *clientSet) ConfigMap() ConfigMap { return c.cmClient } @@ -99,3 +109,7 @@ func (c *clientSet) Workload() Workload { func (c *clientSet) NebulaCluster() NebulaCluster { return c.nebulaClient } + +func (c *clientSet) NebulaRestore() NebulaRestore { + return c.restoreClient +} diff --git a/pkg/kube/cm.go b/pkg/kube/cm.go index e5023910..b7f971e9 100644 --- a/pkg/kube/cm.go +++ b/pkg/kube/cm.go @@ -18,11 +18,14 @@ package kube import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -77,15 +80,26 @@ func (c *cmClient) GetConfigMap(namespace, cmName string) (*corev1.ConfigMap, er } func (c *cmClient) updateConfigMap(cm *corev1.ConfigMap) error { - log := getLog().WithValues("namespace", cm.Namespace, "name", cm.Name) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return c.kubecli.Update(context.TODO(), cm) + ns := cm.GetNamespace() + cmName := cm.GetName() + cmData := cm.Data + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if updated, err := c.GetConfigMap(ns, cmName); err == nil { + cm = updated.DeepCopy() + cm.Data = cmData + } else { + utilruntime.HandleError(fmt.Errorf("get configMap [%s/%s] failed: %v", ns, cmName, err)) + return err + } + + updateErr := c.kubecli.Update(context.TODO(), cm) + if updateErr == nil { + klog.Infof("configMap [%s/%s] updated successfully", ns, cmName) + return nil + } + return updateErr }) - if err != nil { - return err - } - log.Info("configMap updated") - return nil } func (c *cmClient) getConfigMap(objKey client.ObjectKey) (*corev1.ConfigMap, error) { @@ -98,11 +112,9 @@ func (c *cmClient) getConfigMap(objKey client.ObjectKey) (*corev1.ConfigMap, err } func (c *cmClient) DeleteConfigMap(namespace, cmName string) error { - log := getLog().WithValues("namespace", namespace, "name", cmName) cm, err := c.getConfigMap(client.ObjectKey{Namespace: namespace, Name: cmName}) if err != nil { return err } - log.Info("configMap deleted") return c.kubecli.Delete(context.TODO(), cm) } diff --git a/pkg/kube/ingress.go b/pkg/kube/ingress.go index 0c35af26..95ea479e 100644 --- a/pkg/kube/ingress.go +++ b/pkg/kube/ingress.go @@ -24,6 +24,7 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vesoft-inc/nebula-operator/pkg/annotation" @@ -44,7 +45,6 @@ func NewIngress(kubecli client.Client) Ingress { } func (i *ingressClient) CreateOrUpdateIngress(ingress *networkingv1.Ingress) error { - log := getLog().WithValues("namespace", ingress.Namespace, "name", ingress.Name) if err := i.kubecli.Create(context.TODO(), ingress); err != nil { if apierrors.IsAlreadyExists(err) { merge := func(existing, desired *networkingv1.Ingress) error { @@ -89,7 +89,7 @@ func (i *ingressClient) CreateOrUpdateIngress(ingress *networkingv1.Ingress) err } return err } - log.Info("ingress created") + klog.Infof("ingress %s/%s created successfully", ingress.Namespace, ingress.Name) return nil } @@ -107,24 +107,16 @@ func (i *ingressClient) getIngress(objKey client.ObjectKey) (*networkingv1.Ingre } func (i *ingressClient) updateIngress(ingress *networkingv1.Ingress) error { - log := getLog().WithValues("namespace", ingress.Namespace, "name", ingress.Name) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { return i.kubecli.Update(context.TODO(), ingress) }) - if err != nil { - return err - } - log.Info("ingress updated") - return nil } func (i *ingressClient) DeleteIngress(namespace, ingressName string) error { - log := getLog().WithValues("namespace", namespace, "name", ingressName) ingress, err := i.getIngress(client.ObjectKey{Namespace: namespace, Name: ingressName}) if err != nil { return err } - log.Info("ingress deleted") return i.kubecli.Delete(context.TODO(), ingress) } diff --git a/pkg/kube/log.go b/pkg/kube/log.go deleted file mode 100644 index 03c9ede8..00000000 --- a/pkg/kube/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kube - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("kube") - -func getLog() logr.Logger { return _log } diff --git a/pkg/kube/nebulacluster.go b/pkg/kube/nebulacluster.go index ec16dea4..924d2734 100644 --- a/pkg/kube/nebulacluster.go +++ b/pkg/kube/nebulacluster.go @@ -18,31 +18,48 @@ package kube import ( "context" + "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" ) type NebulaCluster interface { + CreateNebulaCluster(nc *v1alpha1.NebulaCluster) error GetNebulaCluster(namespace, name string) (*v1alpha1.NebulaCluster, error) UpdateNebulaCluster(nc *v1alpha1.NebulaCluster) error UpdateNebulaClusterStatus(nc *v1alpha1.NebulaCluster) error + DeleteNebulaCluster(namespace, name string) error } type nebulaClusterClient struct { - cli client.Client + client client.Client } -func NewNebulaCluster(cli client.Client) NebulaCluster { - return &nebulaClusterClient{cli: cli} +func NewNebulaCluster(client client.Client) NebulaCluster { + return &nebulaClusterClient{client: client} +} + +func (c *nebulaClusterClient) CreateNebulaCluster(nc *v1alpha1.NebulaCluster) error { + if err := c.client.Create(context.TODO(), nc); err != nil { + if apierrors.IsAlreadyExists(err) { + klog.Infof("NebulaCluster %s/%s already exists", nc.Namespace, nc.Name) + return nil + } + return err + } + return nil } func (c *nebulaClusterClient) GetNebulaCluster(namespace, name string) (*v1alpha1.NebulaCluster, error) { nebulaCluster := &v1alpha1.NebulaCluster{} - err := c.cli.Get(context.TODO(), types.NamespacedName{ + err := c.client.Get(context.TODO(), types.NamespacedName{ Name: name, Namespace: namespace, }, nebulaCluster) @@ -53,53 +70,98 @@ func (c *nebulaClusterClient) GetNebulaCluster(namespace, name string) (*v1alpha } func (c *nebulaClusterClient) UpdateNebulaCluster(nc *v1alpha1.NebulaCluster) error { - log := getLog().WithValues("namespace", nc.Namespace, "name", nc.Name) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return c.cli.Update(context.TODO(), nc) + ns := nc.Namespace + ncName := nc.Name + ncSpec := nc.Spec.DeepCopy() + labels := nc.GetLabels() + annotations := nc.GetAnnotations() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + // Update the set with the latest resource version for the next poll + if updated, err := c.GetNebulaCluster(ns, ncName); err == nil { + nc = updated.DeepCopy() + nc.Spec = *ncSpec + nc.SetLabels(labels) + nc.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get NebulaCluster %s/%s failed: %v", ns, ncName, err)) + return err + } + + updateErr := c.client.Update(context.TODO(), nc) + if updateErr == nil { + klog.Infof("NebulaCluster %s/%s updated successfully", ns, ncName) + return nil + } + klog.Errorf("update NebulaCluster [%s/%s] failed: %v", ns, ncName, updateErr) + return updateErr }) - if err != nil { - return err - } - log.Info("nebulaCluster updated") - return nil } func (c *nebulaClusterClient) UpdateNebulaClusterStatus(nc *v1alpha1.NebulaCluster) error { - log := getLog().WithValues("namespace", nc.Namespace, "name", nc.Name) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return c.cli.Status().Update(context.TODO(), nc) + ns := nc.Namespace + ncName := nc.Name + status := nc.Status.DeepCopy() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if updated, err := c.GetNebulaCluster(ns, ncName); err == nil { + // make a copy, so we don't mutate the shared cache + nc = updated.DeepCopy() + nc.Status = *status + } else { + utilruntime.HandleError(fmt.Errorf("get NebulaCluster [%s/%s] failed: %v", ns, ncName, err)) + return err + } + + updateErr := c.client.Status().Update(context.TODO(), nc) + if updateErr == nil { + klog.Infof("NebulaCluster [%s/%s] updated successfully", ns, ncName) + return nil + } + klog.Errorf("update NebulaCluster [%s/%s] status failed: %v", ns, ncName, updateErr) + return updateErr }) +} + +func (c *nebulaClusterClient) DeleteNebulaCluster(namespace, name string) error { + nc, err := c.GetNebulaCluster(namespace, name) + if apierrors.IsNotFound(err) { + return nil + } if err != nil { return err } - log.Info("nebulaCluster updated") + if err := c.client.Delete(context.TODO(), nc); err != nil { + return err + } + klog.Infof("NebulaCluster [%s/%s] deleted successfully", namespace, name) return nil } type FakeNebulaCluster struct { - cli client.Client + client client.Client } -func NewFakeNebulaCluster(cli client.Client) NebulaCluster { - return &FakeNebulaCluster{cli: cli} +func NewFakeNebulaCluster(client client.Client) NebulaCluster { + return &FakeNebulaCluster{client: client} } -func (f *FakeNebulaCluster) GetNebulaCluster(namespace, name string) (*v1alpha1.NebulaCluster, error) { - nebulaCluster := &v1alpha1.NebulaCluster{} - err := f.cli.Get(context.TODO(), types.NamespacedName{ - Name: name, - Namespace: namespace, - }, nebulaCluster) - if err != nil { - return nil, err - } - return nebulaCluster, nil +func (f *FakeNebulaCluster) CreateNebulaCluster(_ *v1alpha1.NebulaCluster) error { + return nil } -func (f *FakeNebulaCluster) UpdateNebulaCluster(nc *v1alpha1.NebulaCluster) error { - return f.cli.Update(context.TODO(), nc) +func (f *FakeNebulaCluster) GetNebulaCluster(_, _ string) (*v1alpha1.NebulaCluster, error) { + return nil, nil } -func (f *FakeNebulaCluster) UpdateNebulaClusterStatus(nc *v1alpha1.NebulaCluster) error { - return f.cli.Status().Update(context.TODO(), nc) +func (f *FakeNebulaCluster) UpdateNebulaCluster(_ *v1alpha1.NebulaCluster) error { + return nil +} + +func (f *FakeNebulaCluster) UpdateNebulaClusterStatus(_ *v1alpha1.NebulaCluster) error { + return nil +} + +func (f *FakeNebulaCluster) DeleteNebulaCluster(_, _ string) error { + return nil } diff --git a/pkg/kube/nebularestore.go b/pkg/kube/nebularestore.go new file mode 100644 index 00000000..58db7eba --- /dev/null +++ b/pkg/kube/nebularestore.go @@ -0,0 +1,129 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + "fmt" + + "github.com/vesoft-inc/nebula-go/v3/nebula" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + condutil "github.com/vesoft-inc/nebula-operator/pkg/util/condition" +) + +type RestoreUpdateStatus struct { + TimeStarted *metav1.Time + TimeCompleted *metav1.Time + ClusterName *string + ConditionType v1alpha1.RestoreConditionType + Partitions map[string][]*nebula.HostAddr + Checkpoints map[string]map[string]string +} + +type NebulaRestore interface { + GetNebulaRestore(namespace, name string) (*v1alpha1.NebulaRestore, error) + UpdateNebulaRestoreStatus(restore *v1alpha1.NebulaRestore, condition *v1alpha1.RestoreCondition, newStatus *RestoreUpdateStatus) error +} + +type restoreClient struct { + cli client.Client +} + +func NewNebulaRestore(cli client.Client) NebulaRestore { + return &restoreClient{cli: cli} +} + +func (r *restoreClient) GetNebulaRestore(namespace, name string) (*v1alpha1.NebulaRestore, error) { + restore := &v1alpha1.NebulaRestore{} + err := r.cli.Get(context.TODO(), types.NamespacedName{ + Name: name, + Namespace: namespace, + }, restore) + if err != nil { + return nil, err + } + return restore, nil +} + +func (r *restoreClient) UpdateNebulaRestoreStatus(restore *v1alpha1.NebulaRestore, condition *v1alpha1.RestoreCondition, newStatus *RestoreUpdateStatus) error { + var isStatusUpdate bool + var isConditionUpdate bool + ns := restore.GetNamespace() + rtName := restore.GetName() + + return retry.OnError(retry.DefaultRetry, func(e error) bool { return e != nil }, func() error { + if updated, err := r.GetNebulaRestore(ns, rtName); err == nil { + restore = updated.DeepCopy() + } else { + utilruntime.HandleError(fmt.Errorf("get NebulaRestore [%s/%s] failed: %v", ns, rtName, err)) + return err + } + isStatusUpdate = updateRestoreStatus(&restore.Status, newStatus) + isConditionUpdate = condutil.UpdateNebulaRestoreCondition(&restore.Status, condition) + if isStatusUpdate || isConditionUpdate { + updateErr := r.cli.Status().Update(context.TODO(), restore) + if updateErr == nil { + klog.Infof("NebulaRestore [%s/%s] updated successfully", ns, rtName) + return nil + } + klog.Errorf("update NebulaRestore [%s/%s] status failed: %v", ns, rtName, updateErr) + return updateErr + } + return nil + }) +} + +func updateRestoreStatus(status *v1alpha1.RestoreStatus, newStatus *RestoreUpdateStatus) bool { + if newStatus == nil { + return false + } + + isUpdate := false + if status.Phase != newStatus.ConditionType { + status.Phase = newStatus.ConditionType + isUpdate = true + } + if newStatus.ClusterName != nil { + status.ClusterName = *newStatus.ClusterName + isUpdate = true + } + if newStatus.TimeStarted != nil { + status.TimeStarted = *newStatus.TimeStarted + isUpdate = true + } + if newStatus.TimeCompleted != nil { + status.TimeCompleted = *newStatus.TimeCompleted + isUpdate = true + } + if newStatus.Partitions != nil || (status.Partitions != nil && newStatus.Partitions == nil) { + status.Partitions = newStatus.Partitions + isUpdate = true + } + if newStatus.Checkpoints != nil || (status.Checkpoints != nil && newStatus.Checkpoints == nil) { + status.Checkpoints = newStatus.Checkpoints + isUpdate = true + } + + return isUpdate +} diff --git a/pkg/kube/pod.go b/pkg/kube/pod.go index d52ab9b3..17ff16fc 100644 --- a/pkg/kube/pod.go +++ b/pkg/kube/pod.go @@ -18,11 +18,14 @@ package kube import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -54,19 +57,31 @@ func (pd *podClient) GetPod(namespace, name string) (*corev1.Pod, error) { } func (pd *podClient) UpdatePod(pod *corev1.Pod) error { - log := getLog().WithValues("namespace", pod.GetNamespace(), "name", pod.GetName()) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return pd.kubecli.Update(context.TODO(), pod) + ns := pod.GetNamespace() + podName := pod.GetName() + podLabels := pod.GetLabels() + annotations := pod.GetAnnotations() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if updated, err := pd.GetPod(ns, podName); err == nil { + pod = updated.DeepCopy() + pod.SetLabels(podLabels) + pod.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get pod [%s/%s] failed: %v", ns, podName, err)) + return err + } + + updateErr := pd.kubecli.Update(context.TODO(), pod) + if updateErr == nil { + klog.Infof("pod [%s/%s] updated successfully", ns, podName) + return nil + } + return updateErr }) - if err != nil { - return err - } - log.Info("pod updated") - return nil } func (pd *podClient) DeletePod(namespace, name string) error { - log := getLog().WithValues("namespace", namespace, "name", name) pod := &corev1.Pod{} if err := pd.kubecli.Get(context.TODO(), types.NamespacedName{ Namespace: namespace, @@ -74,7 +89,6 @@ func (pd *podClient) DeletePod(namespace, name string) error { }, pod); err != nil { return err } - log.Info("pod deleted") return pd.kubecli.Delete(context.TODO(), pod) } diff --git a/pkg/kube/pv.go b/pkg/kube/pv.go index 46e98143..1f71f3fd 100644 --- a/pkg/kube/pv.go +++ b/pkg/kube/pv.go @@ -25,7 +25,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vesoft-inc/nebula-operator/pkg/annotation" @@ -49,12 +51,7 @@ func NewPV(kubecli client.Client) PersistentVolume { } func (p *pvClient) CreatePersistentVolume(pv *corev1.PersistentVolume) error { - log := getLog().WithValues("namespace", pv.Namespace, "name", pv.Name) - if err := p.kubecli.Create(context.TODO(), pv); err != nil { - return err - } - log.Info("pv created") - return nil + return p.kubecli.Create(context.TODO(), pv) } func (p *pvClient) GetPersistentVolume(name string) (*corev1.PersistentVolume, error) { @@ -69,7 +66,6 @@ func (p *pvClient) GetPersistentVolume(name string) (*corev1.PersistentVolume, e } func (p *pvClient) PatchPVReclaimPolicy(pv *corev1.PersistentVolume, reclaimPolicy corev1.PersistentVolumeReclaimPolicy) error { - log := getLog().WithValues("namespace", pv.Namespace, "name", pv.Name) patchBytes := []byte(fmt.Sprintf(`{"spec":{"persistentVolumeReclaimPolicy":"%s"}}`, reclaimPolicy)) patch := client.RawPatch(types.StrategicMergePatchType, patchBytes) err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { @@ -78,18 +74,16 @@ func (p *pvClient) PatchPVReclaimPolicy(pv *corev1.PersistentVolume, reclaimPoli if err != nil { return err } - log.Info("pv patched reclaim policy") + klog.Infof("PV %s patched reclaim policy", pv.GetName()) return nil } func (p *pvClient) UpdateMetaInfo(obj runtime.Object, pv *corev1.PersistentVolume) error { - log := getLog().WithValues("name", pv.GetName()) metaObj, ok := obj.(metav1.Object) if !ok { return fmt.Errorf("%+v is not a runtime.Object", obj) } namespace := metaObj.GetNamespace() - log = log.WithValues("namespace", namespace) if pv.Annotations == nil { pv.Annotations = make(map[string]string) } @@ -98,7 +92,7 @@ func (p *pvClient) UpdateMetaInfo(obj runtime.Object, pv *corev1.PersistentVolum } pvcRef := pv.Spec.ClaimRef if pvcRef == nil { - log.Info("pv doesn't have a ClaimRef, skipping") + klog.Infof("PV %s doesn't have a claimRef, skipping", pv.GetName()) return nil } @@ -112,7 +106,7 @@ func (p *pvClient) UpdateMetaInfo(obj runtime.Object, pv *corev1.PersistentVolum if !errors.IsNotFound(err) { return err } - log.Info("pv: pvc doesn't exist, skipping", "pvcName", pvcName) + klog.Infof("PV: PVC %s doesn't exist, skipping", pvcName) return nil } componentType := pvc.Labels[label.ComponentLabelKey] @@ -127,13 +121,25 @@ func (p *pvClient) UpdateMetaInfo(obj runtime.Object, pv *corev1.PersistentVolum } func (p *pvClient) UpdatePersistentVolume(pv *corev1.PersistentVolume) error { - log := getLog().WithValues("name", pv.GetName()) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return p.kubecli.Update(context.TODO(), pv) + pvName := pv.GetName() + labels := pv.GetLabels() + annotations := pv.GetAnnotations() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if updated, err := p.GetPersistentVolume(pvName); err == nil { + pv = updated.DeepCopy() + pv.SetLabels(labels) + pv.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get PV %s failed: %v", pvName, err)) + return err + } + + updateErr := p.kubecli.Update(context.TODO(), pv) + if updateErr == nil { + klog.V(4).Infof("PV %s updated successfully", pvName) + return nil + } + return updateErr }) - if err != nil { - return err - } - log.V(4).Info("pv updated") - return nil } diff --git a/pkg/kube/pvc.go b/pkg/kube/pvc.go index 9e22db2d..a36348f0 100644 --- a/pkg/kube/pvc.go +++ b/pkg/kube/pvc.go @@ -18,12 +18,16 @@ package kube import ( "context" + "fmt" "strconv" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vesoft-inc/nebula-operator/pkg/annotation" @@ -47,12 +51,7 @@ func NewPVC(kubecli client.Client) PersistentVolumeClaim { } func (p *pvcClient) CreatePVC(pvc *corev1.PersistentVolumeClaim) error { - log := getLog().WithValues("namespace", pvc.GetNamespace(), "name", pvc.GetName()) - if err := p.kubecli.Create(context.TODO(), pvc); err != nil { - return err - } - log.Info("pvc created ") - return nil + return p.kubecli.Create(context.TODO(), pvc) } func (p *pvcClient) GetPVC(namespace, name string) (*corev1.PersistentVolumeClaim, error) { @@ -91,26 +90,41 @@ func (p *pvcClient) UpdateMetaInfo(pvc *corev1.PersistentVolumeClaim, pod *corev } func (p *pvcClient) UpdatePVC(pvc *corev1.PersistentVolumeClaim) error { - log := getLog().WithValues("namespace", pvc.GetNamespace(), "name", pvc.GetName()) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return p.kubecli.Update(context.TODO(), pvc) + ns := pvc.GetNamespace() + pvcName := pvc.GetName() + pvcLabels := pvc.GetLabels() + annotations := pvc.GetAnnotations() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if updated, err := p.GetPVC(ns, pvcName); err == nil { + pvc = updated.DeepCopy() + pvc.SetLabels(pvcLabels) + pvc.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get PV [%s/%s] failed: %v", ns, pvcName, err)) + return err + } + + updateErr := p.kubecli.Update(context.TODO(), pvc) + if updateErr == nil { + klog.V(4).Infof("PVC [%s/%s] updated successfully", ns, pvcName) + return nil + } + return updateErr }) - if err != nil { - return err - } - log.V(4).Info("pvc updated") - return nil } func (p *pvcClient) DeletePVC(namespace, name string) error { - log := getLog().WithValues("namespace", namespace, "name", name) pvc, err := p.GetPVC(namespace, name) + if apierrors.IsNotFound(err) { + return nil + } if err != nil { return err } if err := p.kubecli.Delete(context.TODO(), pvc); err != nil { return err } - log.Info("pvc deleted") + klog.Infof("PVC [%s/%s] deleted successfully", namespace, name) return nil } diff --git a/pkg/kube/secret.go b/pkg/kube/secret.go new file mode 100644 index 00000000..71a82dd5 --- /dev/null +++ b/pkg/kube/secret.go @@ -0,0 +1,33 @@ +package kube + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Secret interface { + GetSecret(namespace, secretName string) (*corev1.Secret, error) +} + +type secretClient struct { + kubecli client.Client +} + +func NewSecret(kubecli client.Client) Secret { + return &secretClient{kubecli: kubecli} +} + +func (s *secretClient) GetSecret(namespace, secretName string) (*corev1.Secret, error) { + secret := &corev1.Secret{} + err := s.kubecli.Get(context.TODO(), types.NamespacedName{ + Name: secretName, + Namespace: namespace, + }, secret) + if err != nil { + return nil, err + } + return secret, nil +} diff --git a/pkg/kube/service.go b/pkg/kube/service.go index 7cf6b656..e33249af 100644 --- a/pkg/kube/service.go +++ b/pkg/kube/service.go @@ -18,11 +18,14 @@ package kube import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -42,16 +45,13 @@ func NewService(kubecli client.Client) Service { } func (s *serviceClient) CreateService(service *corev1.Service) error { - log := getLog().WithValues("namespace", service.Namespace, "name", service.Name) if err := s.kubecli.Create(context.TODO(), service); err != nil { if apierrors.IsAlreadyExists(err) { - log.Info("service already exists") + klog.Infof("service [%s/%s] already exists", service.Namespace, service.Name) return nil } - log.Error(err, "service created failed") return err } - log.Info("service created") return nil } @@ -68,19 +68,33 @@ func (s *serviceClient) GetService(namespace, name string) (*corev1.Service, err } func (s *serviceClient) UpdateService(service *corev1.Service) error { - log := getLog().WithValues("namespace", service.Namespace, "name", service.Name) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return s.kubecli.Update(context.TODO(), service) + ns := service.GetNamespace() + svcName := service.GetName() + svcSpec := service.Spec.DeepCopy() + labels := service.GetLabels() + annotations := service.GetAnnotations() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if updated, err := s.GetService(ns, svcName); err == nil { + service = updated.DeepCopy() + service.Spec = *svcSpec + service.SetLabels(labels) + service.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get service [%s/%s] failed: %v", ns, svcName, err)) + return err + } + + updateErr := s.kubecli.Update(context.TODO(), service) + if updateErr == nil { + klog.Infof("service [%s/%s] updated successfully", ns, svcName) + return nil + } + return updateErr }) - if err != nil { - return err - } - log.Info("service updated") - return nil } func (s *serviceClient) DeleteService(namespace, name string) error { - log := getLog().WithValues("namespace", namespace, "name", name) service := &corev1.Service{} err := s.kubecli.Get(context.TODO(), types.NamespacedName{ Name: name, @@ -90,6 +104,6 @@ func (s *serviceClient) DeleteService(namespace, name string) error { return err } - log.Info("service deleted") + klog.Infof("service [%s/%s] deleted successfully", namespace, name) return s.kubecli.Delete(context.TODO(), service) } diff --git a/pkg/kube/workload.go b/pkg/kube/workload.go index 5b205fb0..5562933f 100644 --- a/pkg/kube/workload.go +++ b/pkg/kube/workload.go @@ -24,8 +24,12 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vesoft-inc/nebula-operator/pkg/util/resource" ) type Workload interface { @@ -57,26 +61,63 @@ func (w *workloadClient) GetWorkload(namespace, name string, gvk schema.GroupVer } func (w *workloadClient) CreateWorkload(obj *unstructured.Unstructured) error { - log := getLog().WithValues("kind", obj.GetKind(), "namespace", obj.GetNamespace(), "name", obj.GetName()) if err := w.kubecli.Create(context.TODO(), obj); err != nil { if apierrors.IsAlreadyExists(err) { - log.Error(err, "workload already exists") + klog.Error(err, "workload already exists") return nil } return err } - log.Info("workload created") + klog.Infof("workload %s/%s created successfully", obj.GetNamespace(), obj.GetName()) return nil } func (w *workloadClient) UpdateWorkload(obj *unstructured.Unstructured) error { - log := getLog().WithValues("kind", obj.GetKind(), "namespace", obj.GetNamespace(), "name", obj.GetName()) - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - return w.kubecli.Update(context.TODO(), obj) + kind := obj.GetKind() + ns := obj.GetNamespace() + objName := obj.GetName() + spec := getSpec(obj) + labels := obj.GetLabels() + annotations := obj.GetAnnotations() + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + updated, err := w.GetWorkload(ns, objName, resource.StatefulSetKind) + if err == nil { + obj = updated.DeepCopy() + setSpec(obj, spec) + obj.SetLabels(labels) + obj.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get workload %s/%s failed: %v", ns, objName, err)) + } + + updated, err = w.GetWorkload(ns, objName, resource.AdvancedStatefulSetKind) + if err == nil { + obj = updated.DeepCopy() + setSpec(obj, spec) + obj.SetLabels(labels) + obj.SetAnnotations(annotations) + } else { + utilruntime.HandleError(fmt.Errorf("get workload %s/%s failed: %v", ns, objName, err)) + } + + updateErr := w.kubecli.Update(context.TODO(), obj) + if updateErr == nil { + klog.Infof("workload %s %s/%s updated successfully", kind, ns, objName) + return nil + } + return updateErr }) - if err != nil { - return fmt.Errorf("workload kind %s %s/%s update failed: %v", obj.GetKind(), obj.GetNamespace(), obj.GetName(), err) +} + +func getSpec(obj *unstructured.Unstructured) map[string]interface{} { + spec, found, err := unstructured.NestedMap(obj.Object, "spec") + if err != nil || !found { + return nil } - log.Info("workload updated") - return nil + return spec +} + +func setSpec(obj *unstructured.Unstructured, value interface{}) error { + return unstructured.SetNestedField(obj.Object, value, "spec") } diff --git a/pkg/logging/logging.go b/pkg/logging/logging.go deleted file mode 100644 index 9706504f..00000000 --- a/pkg/logging/logging.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package logging - -import ( - "github.com/go-logr/logr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log" - runtimezap "sigs.k8s.io/controller-runtime/pkg/log/zap" -) - -type ( - Logger = logr.Logger - Options = runtimezap.Options - Opts = runtimezap.Opts -) - -var ( - Log = log.Log - LoggerFrom = ctrl.LoggerFrom - LoggerInto = ctrl.LoggerInto - SetLogger = ctrl.SetLogger - - New = runtimezap.New - NewRaw = runtimezap.NewRaw - Level = runtimezap.Level - StacktraceLevel = runtimezap.StacktraceLevel - RawZapOpts = runtimezap.RawZapOpts - UseDevMode = runtimezap.UseDevMode - UseFlagOptions = runtimezap.UseFlagOptions -) diff --git a/pkg/nebula/agent.go b/pkg/nebula/agent.go new file mode 100644 index 00000000..eb58d9f4 --- /dev/null +++ b/pkg/nebula/agent.go @@ -0,0 +1,72 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nebula + +import ( + "context" + "fmt" + + agentclient "github.com/vesoft-inc/nebula-agent/pkg/client" + "github.com/vesoft-inc/nebula-go/v3/nebula" + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" + rtutil "github.com/vesoft-inc/nebula-operator/pkg/util/restore" +) + +type Agent struct { + agentclient.Client +} + +func NewAgent(agentAddr *nebula.HostAddr) (*Agent, error) { + agentAddr.Port = v1alpha1.DefaultAgentPortGRPC + cfg := &agentclient.Config{ + Addr: agentAddr, + } + c, err := agentclient.New(context.TODO(), cfg) + if err != nil { + return nil, err + } + + a := &Agent{ + Client: c, + } + + return a, nil +} + +type AgentManager struct { + agents map[string]*Agent +} + +func NewAgentManager() *AgentManager { + return &AgentManager{ + agents: make(map[string]*Agent), + } +} + +func (a *AgentManager) GetAgent(agentAddr *nebula.HostAddr) (*Agent, error) { + if agent, ok := a.agents[agentAddr.Host]; ok { + return agent, nil + } + + agent, err := NewAgent(agentAddr) + if err != nil { + return nil, fmt.Errorf("create agent %s failed: %v", rtutil.StringifyAddr(agentAddr), err) + } + + a.agents[agentAddr.Host] = agent + return agent, nil +} diff --git a/pkg/nebula/log.go b/pkg/nebula/log.go deleted file mode 100644 index d5e373d2..00000000 --- a/pkg/nebula/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nebula - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("nebula") - -func getLog() logr.Logger { return _log } diff --git a/pkg/nebula/meta_client.go b/pkg/nebula/meta_client.go index 0bb81eb4..5bde297b 100644 --- a/pkg/nebula/meta_client.go +++ b/pkg/nebula/meta_client.go @@ -19,17 +19,17 @@ package nebula import ( "fmt" "strconv" - "sync" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/sets" - "github.com/vesoft-inc/nebula-go/v3/nebula" "github.com/vesoft-inc/nebula-go/v3/nebula/meta" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + utilerrors "github.com/vesoft-inc/nebula-operator/pkg/util/errors" ) -var ErrNoAvailableMetadEndpoints = errors.New("metadclient: no available endpoints") +var ErrNoAvailableMetadEndpoints = errors.New("metadclient: no available hosts") var _ MetaInterface = (*metaClient)(nil) @@ -38,9 +38,11 @@ type ( MetaInterface interface { GetSpace(spaceName []byte) (*meta.SpaceItem, error) + DropSpace(spaceName []byte) error ListSpaces() ([]*meta.IdName, error) - AddHosts(endpoints []*nebula.HostAddr) error - DropHosts(endpoints []*nebula.HostAddr) error + ListCluster() (*meta.ListClusterInfoResp, error) + AddHosts(hosts []*nebula.HostAddr) error + DropHosts(hosts []*nebula.HostAddr) error ListHosts(hostType meta.ListHostType) ([]*meta.HostItem, error) ListParts(spaceID nebula.GraphSpaceID, partIDs []nebula.PartitionID) ([]*meta.PartItem, error) GetSpaceParts() (map[nebula.GraphSpaceID][]*meta.PartItem, error) @@ -49,21 +51,21 @@ type ( BalanceStatus(jobID int32, spaceID nebula.GraphSpaceID) error BalanceLeader(spaceID nebula.GraphSpaceID) error BalanceData(spaceID nebula.GraphSpaceID) (int32, error) - RemoveHost(spaceID nebula.GraphSpaceID, endpoints []*nebula.HostAddr) (int32, error) + RemoveHost(spaceID nebula.GraphSpaceID, hosts []*nebula.HostAddr) (int32, error) + RestoreMeta(hosts []*meta.HostPair, files []string) (*meta.RestoreMetaResp, error) Disconnect() error } metaClient struct { - mutex sync.Mutex client *meta.MetaServiceClient } ) -func NewMetaClient(endpoints []string, options ...Option) (MetaInterface, error) { - if len(endpoints) == 0 { +func NewMetaClient(hosts []string, options ...Option) (MetaInterface, error) { + if len(hosts) == 0 { return nil, ErrNoAvailableMetadEndpoints } - mc, err := newMetaConnection(endpoints[0], options...) + mc, err := newMetaConnection(hosts[0], options...) if err != nil { return nil, err } @@ -121,8 +123,6 @@ func (m *metaClient) Disconnect() error { } func (m *metaClient) GetSpace(spaceName []byte) (*meta.SpaceItem, error) { - m.mutex.Lock() - defer m.mutex.Unlock() req := &meta.GetSpaceReq{SpaceName: spaceName} resp, err := m.retryOnError(req, func(req interface{}) (interface{}, error) { resp, err := m.client.GetSpace(req.(*meta.GetSpaceReq)) @@ -134,9 +134,16 @@ func (m *metaClient) GetSpace(spaceName []byte) (*meta.SpaceItem, error) { return resp.(*meta.GetSpaceResp).Item, nil } +func (m *metaClient) DropSpace(spaceName []byte) error { + req := &meta.DropSpaceReq{SpaceName: spaceName, IfExists: true} + _, err := m.retryOnError(req, func(req interface{}) (interface{}, error) { + resp, err := m.client.DropSpace(req.(*meta.DropSpaceReq)) + return resp, err + }) + return err +} + func (m *metaClient) ListSpaces() ([]*meta.IdName, error) { - m.mutex.Lock() - defer m.mutex.Unlock() resp, err := m.retryOnError(meta.NewListSpacesReq(), func(req interface{}) (interface{}, error) { resp, err := m.client.ListSpaces(req.(*meta.ListSpacesReq)) return resp, err @@ -147,6 +154,38 @@ func (m *metaClient) ListSpaces() ([]*meta.IdName, error) { return resp.(*meta.ListSpacesResp).Spaces, nil } +func (m *metaClient) ListCluster() (*meta.ListClusterInfoResp, error) { + r, err := m.retryOnError(meta.NewListClusterInfoReq(), func(req interface{}) (interface{}, error) { + resp, err := m.client.ListCluster(req.(*meta.ListClusterInfoReq)) + return resp, err + }) + if err != nil { + return nil, err + } + resp := r.(*meta.ListClusterInfoResp) + for _, services := range resp.GetHostServices() { + for _, s := range services { + if s.Role == meta.HostRole_META { + dir, err := m.getMetaDirInfo() + if err != nil { + return nil, err + } + s.Dir = dir + } + } + } + return resp, nil +} + +func (m *metaClient) getMetaDirInfo() (*nebula.DirInfo, error) { + resp, err := m.client.GetMetaDirInfo(meta.NewGetMetaDirInfoReq()) + if err != nil { + return nil, err + } + + return resp.GetDir(), nil +} + func (m *metaClient) AddHosts(hosts []*nebula.HostAddr) error { req := &meta.AddHostsReq{ Hosts: hosts, @@ -170,8 +209,6 @@ func (m *metaClient) DropHosts(hosts []*nebula.HostAddr) error { } func (m *metaClient) ListHosts(hostType meta.ListHostType) ([]*meta.HostItem, error) { - m.mutex.Lock() - defer m.mutex.Unlock() req := &meta.ListHostsReq{Type: hostType} resp, err := m.retryOnError(req, func(req interface{}) (interface{}, error) { resp, err := m.client.ListHosts(req.(*meta.ListHostsReq)) @@ -184,8 +221,6 @@ func (m *metaClient) ListHosts(hostType meta.ListHostType) ([]*meta.HostItem, er } func (m *metaClient) ListParts(spaceID nebula.GraphSpaceID, partIDs []nebula.PartitionID) ([]*meta.PartItem, error) { - m.mutex.Lock() - defer m.mutex.Unlock() req := &meta.ListPartsReq{ SpaceID: spaceID, PartIds: partIDs, @@ -253,7 +288,6 @@ func (m *metaClient) GetSpaceLeaderHosts(space []byte) ([]string, error) { } func (m *metaClient) GetLeaderCount(leaderHost string) (int, error) { - log := getLog() spaceItems, err := m.GetSpaceParts() if err != nil { return 0, err @@ -265,8 +299,7 @@ func (m *metaClient) GetLeaderCount(leaderHost string) (int, error) { continue } if partItem.Leader.Host == leaderHost { - log.Info("space's partition still distribute this node", - "space", spaceID, "partition", partItem.PartID) + klog.Infof("space %d partition %d still distribute this node", spaceID, partItem.PartID) count++ } } @@ -275,7 +308,6 @@ func (m *metaClient) GetLeaderCount(leaderHost string) (int, error) { } func (m *metaClient) BalanceLeader(spaceID nebula.GraphSpaceID) error { - log := getLog().WithValues("SpaceID", spaceID) req := &meta.AdminJobReq{ SpaceID: spaceID, Op: meta.JobOp_ADD, @@ -288,13 +320,12 @@ func (m *metaClient) BalanceLeader(spaceID nebula.GraphSpaceID) error { if err != nil { return err } - log.Info("balance leader successfully") + klog.Infof("spaceID %d balance leader successfully", spaceID) return nil } func (m *metaClient) balance(req *meta.AdminJobReq) (int32, error) { - log := getLog() - log.Info("start balance job") + klog.Info("start balance job") resp, err := m.retryOnError(req, func(req interface{}) (interface{}, error) { resp, err := m.client.RunAdminJob(req.(*meta.AdminJobReq)) return resp, err @@ -302,7 +333,7 @@ func (m *metaClient) balance(req *meta.AdminJobReq) (int32, error) { if err != nil { return 0, err } - log.Info("balance job running now") + klog.Info("balance job running now") jobID := resp.(*meta.AdminJobResp).GetResult_().GetJobID() return jobID, utilerrors.ReconcileErrorf("waiting for balance job %d finished", jobID) } @@ -317,9 +348,9 @@ func (m *metaClient) BalanceData(spaceID nebula.GraphSpaceID) (int32, error) { return m.balance(req) } -func (m *metaClient) RemoveHost(spaceID nebula.GraphSpaceID, endpoints []*nebula.HostAddr) (int32, error) { +func (m *metaClient) RemoveHost(spaceID nebula.GraphSpaceID, hosts []*nebula.HostAddr) (int32, error) { paras := make([][]byte, 0) - for _, endpoint := range endpoints { + for _, endpoint := range hosts { // The back quote need here to consistent with the host addr registered in meta paras = append(paras, []byte(fmt.Sprintf(`"%s":%d`, endpoint.Host, endpoint.Port))) } @@ -334,7 +365,6 @@ func (m *metaClient) RemoveHost(spaceID nebula.GraphSpaceID, endpoints []*nebula } func (m *metaClient) BalanceStatus(jobID int32, spaceID nebula.GraphSpaceID) error { - log := getLog().WithValues("JobID", jobID, "SpaceID", spaceID) req := &meta.AdminJobReq{ SpaceID: spaceID, Op: meta.JobOp_SHOW, @@ -354,8 +384,26 @@ func (m *metaClient) BalanceStatus(jobID int32, spaceID nebula.GraphSpaceID) err return nil } } - log.Info("Balance job in progress") - return &utilerrors.ReconcileError{Msg: fmt.Sprintf("Balance job %d still in progress", jobID)} + return &utilerrors.ReconcileError{Msg: fmt.Sprintf("Balance job still in progress, jobID %d, spaceID %d", jobID, spaceID)} +} + +func (m *metaClient) RestoreMeta(hostPairs []*meta.HostPair, files []string) (*meta.RestoreMetaResp, error) { + byteFiles := make([][]byte, 0, len(files)) + for _, f := range files { + byteFiles = append(byteFiles, []byte(f)) + } + req := &meta.RestoreMetaReq{ + Hosts: hostPairs, + Files: byteFiles, + } + resp, err := m.retryOnError(req, func(req interface{}) (interface{}, error) { + resp, err := m.client.RestoreMeta(req.(*meta.RestoreMetaReq)) + return resp, err + }) + if err != nil { + return nil, err + } + return resp.(*meta.RestoreMetaResp), nil } func (m *metaClient) retryOnError(req interface{}, fn Fn) (interface{}, error) { @@ -382,9 +430,12 @@ func (m *metaClient) retryOnError(req interface{}, fn Fn) (interface{}, error) { } code := getResponseCode(resp) if code != nebula.ErrorCode_SUCCEEDED { - return nil, fmt.Errorf("retry response code %d", code) - } else if code == nebula.ErrorCode_E_EXISTED { - return resp, nil + if code == nebula.ErrorCode_E_EXISTED { + return resp, nil + } else if code == nebula.ErrorCode_E_NO_HOSTS { + return resp, nil + } + return nil, fmt.Errorf("metad client retry response code %d name %s", code, code.String()) } return resp, nil } else if code == nebula.ErrorCode_E_EXISTED { @@ -392,7 +443,7 @@ func (m *metaClient) retryOnError(req interface{}, fn Fn) (interface{}, error) { } else if code == nebula.ErrorCode_E_NO_HOSTS { return resp, nil } - return nil, fmt.Errorf("response code %d", code) + return nil, fmt.Errorf("metad client response code %d name %s", code, code.String()) } return resp, nil } @@ -405,12 +456,16 @@ func getResponseCode(resp interface{}) nebula.ErrorCode { return r.Code case *meta.ListSpacesResp: return r.Code + case *meta.ListClusterInfoResp: + return r.Code case *meta.ListHostsResp: return r.Code case *meta.ListPartsResp: return r.Code case *meta.AdminJobResp: return r.Code + case *meta.RestoreMetaResp: + return r.Code default: return nebula.ErrorCode_E_UNKNOWN } @@ -424,12 +479,16 @@ func getResponseLeader(resp interface{}) *nebula.HostAddr { return r.Leader case *meta.ListSpacesResp: return r.Leader + case *meta.ListClusterInfoResp: + return r.Leader case *meta.ListHostsResp: return r.Leader case *meta.ListPartsResp: return r.Leader case *meta.AdminJobResp: return r.Leader + case *meta.RestoreMetaResp: + return r.Leader default: return nil } diff --git a/pkg/nebula/storage_client.go b/pkg/nebula/storage_client.go index 620369c8..2cb0648b 100644 --- a/pkg/nebula/storage_client.go +++ b/pkg/nebula/storage_client.go @@ -17,12 +17,10 @@ limitations under the License. package nebula import ( - "sync" - "github.com/pkg/errors" - "github.com/vesoft-inc/nebula-go/v3/nebula" "github.com/vesoft-inc/nebula-go/v3/nebula/storage" + "k8s.io/klog/v2" ) var ErrNoAvailableStoragedEndpoints = errors.New("storagedclient: no available endpoints") @@ -37,13 +35,12 @@ type StorageInterface interface { } type storageClient struct { - mutex sync.Mutex client *storage.StorageAdminServiceClient } func NewStorageClient(endpoints []string, options ...Option) (StorageInterface, error) { if len(endpoints) == 0 { - return nil, ErrNoAvailableMetadEndpoints + return nil, ErrNoAvailableStoragedEndpoints } sc, err := newStorageConnection(endpoints[0], options...) if err != nil { @@ -66,60 +63,48 @@ func newStorageConnection(endpoint string, options ...Option) (*storageClient, e } func (s *storageClient) connect() error { - log := getLog() if err := s.client.Open(); err != nil { - log.Error(err, "open transport failed") return err } - log.Info("storaged connection opened", "isOpen", s.client.IsOpen()) - return nil -} - -func (s *storageClient) disconnect() error { - if err := s.client.Close(); err != nil { - getLog().Error(err, "close transport failed") + if !s.client.IsOpen() { + return errors.Errorf("transport is not open") } return nil } func (s *storageClient) Disconnect() error { - return s.disconnect() + return s.client.Close() } func (s *storageClient) TransLeader(spaceID nebula.GraphSpaceID, partID nebula.PartitionID, newLeader *nebula.HostAddr) error { - s.mutex.Lock() - defer s.mutex.Unlock() req := &storage.TransLeaderReq{ SpaceID: spaceID, PartID: partID, NewLeader_: newLeader, } - log := getLog() - log.Info("start TransLeader") + klog.Infof("start transfer leader spaceID %d partitionID %d", spaceID, partID) resp, err := s.client.TransLeader(req) if err != nil { - log.Error(err, "TransLeader failed") + klog.Errorf("TransLeader failed: %v", err) return err } if len(resp.Result_.FailedParts) > 0 { if resp.Result_.FailedParts[0].Code == nebula.ErrorCode_E_PART_NOT_FOUND { - return errors.Errorf("part %d not found", partID) + return errors.Errorf("partition %d not found", partID) } else if resp.Result_.FailedParts[0].Code == nebula.ErrorCode_E_SPACE_NOT_FOUND { return errors.Errorf("space %d not found", partID) } else if resp.Result_.FailedParts[0].Code == nebula.ErrorCode_E_LEADER_CHANGED { - log.Info("request leader changed", "result", resp.Result_.FailedParts[0].String()) + klog.Infof("request leader changed, result: %v", resp.Result_.FailedParts[0].String()) return nil } else { - return errors.Errorf("TransLeader space %d part %d code %d", spaceID, partID, resp.Result_.FailedParts[0].Code) + return errors.Errorf("TransLeader space %d partition %d code %d", spaceID, partID, resp.Result_.FailedParts[0].Code) } } return nil } func (s *storageClient) RemovePart(spaceID nebula.GraphSpaceID, partID nebula.PartitionID) error { - s.mutex.Lock() - defer s.mutex.Unlock() req := &storage.RemovePartReq{ SpaceID: spaceID, PartID: partID, @@ -135,8 +120,6 @@ func (s *storageClient) RemovePart(spaceID nebula.GraphSpaceID, partID nebula.Pa } func (s *storageClient) GetLeaderParts() (map[nebula.GraphSpaceID][]nebula.PartitionID, error) { - s.mutex.Lock() - defer s.mutex.Unlock() req := &storage.GetLeaderReq{} resp, err := s.client.GetLeaderParts(req) if err != nil { diff --git a/pkg/scheduler/extender/extender.go b/pkg/scheduler/extender/extender.go index 1f832632..993eb213 100644 --- a/pkg/scheduler/extender/extender.go +++ b/pkg/scheduler/extender/extender.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" extender "k8s.io/kube-scheduler/extender/v1" "github.com/vesoft-inc/nebula-operator/pkg/label" @@ -49,7 +50,7 @@ func NewScheduleExtender(kubeCli kubernetes.Interface, predicate predicates.Pred eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "nebula-scheduler"}) eventBroadcaster.StartLogging(func(format string, args ...interface{}) { - getLog().Info(fmt.Sprintf(format, args...)) + klog.Info(fmt.Sprintf(format, args...)) }) eventBroadcaster.StartRecordingToSink( &typedcorev1.EventSinkImpl{ @@ -59,9 +60,6 @@ func NewScheduleExtender(kubeCli kubernetes.Interface, predicate predicates.Pred } func (se *scheduleExtender) Filter(args *extender.ExtenderArgs) *extender.ExtenderFilterResult { - log := getLog() - log.Info("schedule pod", "namespace", args.Pod.Namespace, "name", args.Pod.Name) - l := label.Label(args.Pod.Labels) if !l.IsManagedByNebulaOperator() && !l.IsNebulaComponent() { diff --git a/pkg/scheduler/extender/log.go b/pkg/scheduler/extender/log.go deleted file mode 100644 index 0a8e2964..00000000 --- a/pkg/scheduler/extender/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package extender - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("scheduler").WithName("extender") - -func getLog() logr.Logger { return _log } diff --git a/pkg/scheduler/extender/predicates/log.go b/pkg/scheduler/extender/predicates/log.go deleted file mode 100644 index 0840c302..00000000 --- a/pkg/scheduler/extender/predicates/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package predicates - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("scheduler").WithName("predicates") - -func getLog() logr.Logger { return _log } diff --git a/pkg/scheduler/extender/predicates/topology.go b/pkg/scheduler/extender/predicates/topology.go index 5642a415..a738c200 100644 --- a/pkg/scheduler/extender/predicates/topology.go +++ b/pkg/scheduler/extender/predicates/topology.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" @@ -49,8 +50,6 @@ func (t *topology) Filter(pod *corev1.Pod, preNodes []corev1.Node) ([]corev1.Nod componentType := pod.Labels[label.ComponentLabelKey] clusterName := pod.Labels[label.ClusterLabelKey] - log := getLog() - nc := &v1alpha1.NebulaCluster{} if err := t.client.Get(context.TODO(), types.NamespacedName{ Namespace: pod.Namespace, @@ -61,7 +60,7 @@ func (t *topology) Filter(pod *corev1.Pod, preNodes []corev1.Node) ([]corev1.Nod component, err := nc.ComponentByType(v1alpha1.ComponentType(componentType)) if err != nil { - log.Error(err, "get component %s failed", "componentType", componentType) + klog.Errorf("get component %s failed: %v", componentType, err) return nil, err } @@ -119,7 +118,7 @@ func (t *topology) Filter(pod *corev1.Pod, preNodes []corev1.Node) ([]corev1.Nod } } - log.Info("candidate nodes", "nodes", resNames) + klog.Infof("candidate nodes %v", resNames) return resNodes, nil } @@ -151,7 +150,6 @@ func (t *topology) acquireLock(pod *corev1.Pod, pods []corev1.Pod) error { var currentPod, schedulingPod *corev1.Pod namespace := pod.GetNamespace() podName := pod.GetName() - log := getLog().WithValues("namespace", namespace, "podName", podName) for i := range pods { if pods[i].GetName() == podName { @@ -163,7 +161,7 @@ func (t *topology) acquireLock(pod *corev1.Pod, pods []corev1.Pod) error { } if currentPod == nil { - log.Info("not found") + klog.Infof("pod %s not found", podName) return fmt.Errorf("can't find current Pod %s/%s", namespace, podName) } @@ -175,12 +173,10 @@ func (t *topology) acquireLock(pod *corev1.Pod, pods []corev1.Pod) error { } currentPod.Annotations[annotation.AnnPodSchedulingKey] = now err := t.client.Update(context.TODO(), currentPod) - log := log.WithValues("key", annotation.AnnPodSchedulingKey, "value", now) if err != nil { - log.Error(err, "set annotation failed") return err } - log.Info("set annotation successfully") + klog.Infof("pod [%s/%s] set annotation successfully", namespace, podName) return nil } @@ -192,13 +188,11 @@ func (t *topology) acquireLock(pod *corev1.Pod, pods []corev1.Pod) error { // If the scheduling pod already bind with node, and then remove the AnnPodSchedulingKey annotation if schedulingPod.Spec.NodeName != "" { delete(schedulingPod.Annotations, annotation.AnnPodSchedulingKey) - log := log.WithValues("key", annotation.AnnPodSchedulingKey) err := t.client.Update(context.TODO(), schedulingPod) if err != nil { - log.Error(err, "delete annotation failed") return err } - log.Error(err, "delete annotation successfully") + klog.Infof("delete pod [%s/%s] annotation successfully", namespace, podName) return nil } diff --git a/pkg/util/async/async.go b/pkg/util/async/async.go new file mode 100644 index 00000000..556a37e6 --- /dev/null +++ b/pkg/util/async/async.go @@ -0,0 +1,66 @@ +package async + +import ( + "context" + "fmt" + + errorutils "k8s.io/apimachinery/pkg/util/errors" +) + +type Group struct { + ctx context.Context + concurrency int32 + workers []func(stopCh chan interface{}) +} + +func NewGroup(ctx context.Context, concurrency int32) *Group { + return &Group{ + ctx: ctx, + concurrency: concurrency, + } +} + +func (g *Group) Add(worker func(stopCh chan interface{})) { + g.workers = append(g.workers, worker) +} + +func (g *Group) Wait() error { + if g.workers == nil || len(g.workers) == 0 { + return nil + } + + stopCh := make(chan interface{}, len(g.workers)) + concurCh := make(chan interface{}, g.concurrency) + + go func() { + for _, worker := range g.workers { + concurCh <- struct{}{} + go worker(stopCh) + } + }() + + var errs []error + res := 0 + for { + select { + case <-g.ctx.Done(): + return fmt.Errorf("group waiting time out") + case val := <-stopCh: + switch v := val.(type) { + case error: + errs = append(errs, v) + case []error: + errs = append(errs, v...) + default: + } + <-concurCh + res++ + if res == len(g.workers) { + if len(errs) == 0 { + return nil + } + return errorutils.NewAggregate(errs) + } + } + } +} diff --git a/pkg/util/condition/condition.go b/pkg/util/condition/cluster_condition.go similarity index 99% rename from pkg/util/condition/condition.go rename to pkg/util/condition/cluster_condition.go index c00d4a52..54733c68 100644 --- a/pkg/util/condition/condition.go +++ b/pkg/util/condition/cluster_condition.go @@ -25,7 +25,7 @@ import ( const ( WorkloadReady = "Ready" - // WorkloadNotUpToDate is added when one of workloads is not up to date. + // WorkloadNotUpToDate is added when one of workloads is not up-to-date. WorkloadNotUpToDate = "WorkloadNotUpToDate" // MetadUnhealthy is added when one of metad pods is unhealthy. MetadUnhealthy = "MetadUnhealthy" diff --git a/pkg/util/condition/restore_condition.go b/pkg/util/condition/restore_condition.go new file mode 100644 index 00000000..95c695f5 --- /dev/null +++ b/pkg/util/condition/restore_condition.go @@ -0,0 +1,72 @@ +package condition + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" +) + +func UpdateNebulaRestoreCondition(status *v1alpha1.RestoreStatus, condition *v1alpha1.RestoreCondition) bool { + if condition == nil { + return false + } + + condition.LastTransitionTime = metav1.Now() + conditionIndex, oldCondition := getRestoreCondition(status, condition.Type) + + if oldCondition == nil { + status.Conditions = append(status.Conditions, *condition) + return true + } + + if condition.Status == oldCondition.Status { + condition.LastTransitionTime = oldCondition.LastTransitionTime + } + + isUpdate := condition.Status == oldCondition.Status && + condition.Reason == oldCondition.Reason && + condition.Message == oldCondition.Message && + condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime) + + status.Conditions[conditionIndex] = *condition + + return !isUpdate +} + +func IsRestoreInvalid(restore *v1alpha1.NebulaRestore) bool { + _, condition := getRestoreCondition(&restore.Status, v1alpha1.RestoreInvalid) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +func IsRestoreMetadComplete(restore *v1alpha1.NebulaRestore) bool { + _, condition := getRestoreCondition(&restore.Status, v1alpha1.RestoreMetadComplete) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +func IsRestoreStoragedComplete(restore *v1alpha1.NebulaRestore) bool { + _, condition := getRestoreCondition(&restore.Status, v1alpha1.RestoreStoragedCompleted) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +func IsRestoreComplete(restore *v1alpha1.NebulaRestore) bool { + _, condition := getRestoreCondition(&restore.Status, v1alpha1.RestoreComplete) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +func IsRestoreFailed(restore *v1alpha1.NebulaRestore) bool { + _, condition := getRestoreCondition(&restore.Status, v1alpha1.RestoreFailed) + return condition != nil && condition.Status == corev1.ConditionTrue +} + +func getRestoreCondition(status *v1alpha1.RestoreStatus, conditionType v1alpha1.RestoreConditionType) (int, *v1alpha1.RestoreCondition) { + if status == nil { + return -1, nil + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType { + return i, &status.Conditions[i] + } + } + return -1, nil +} diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index b15f33d5..c97592a2 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -23,12 +23,13 @@ import ( "regexp" "sort" "strings" + + "k8s.io/klog/v2" ) var paramPattern = regexp.MustCompile(`--(\w+)=(.+)`) func AppendCustomConfig(data string, custom map[string]string) string { - log := getLog() if len(custom) == 0 { return data } @@ -61,7 +62,7 @@ func AppendCustomConfig(data string, custom map[string]string) string { } } if err := scanner.Err(); err != nil { - log.Error(err, "reading input failed") + klog.Errorf("reading input failed, error: %v", err) } if len(custom) > 0 { diff --git a/pkg/util/config/log.go b/pkg/util/config/log.go deleted file mode 100644 index affeff89..00000000 --- a/pkg/util/config/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("config") - -func getLog() logr.Logger { return _log } diff --git a/pkg/util/errors/errors.go b/pkg/util/errors/errors.go index 605e5b38..de906871 100644 --- a/pkg/util/errors/errors.go +++ b/pkg/util/errors/errors.go @@ -18,6 +18,7 @@ package errors import ( "fmt" + "net" "k8s.io/apimachinery/pkg/api/errors" ) @@ -45,3 +46,8 @@ func IsStatusError(err error) bool { _, ok := err.(*errors.StatusError) return ok } + +func IsDNSError(err error) bool { + _, ok := err.(*net.DNSError) + return ok +} diff --git a/pkg/util/extender/log.go b/pkg/util/extender/log.go deleted file mode 100644 index 152d99cb..00000000 --- a/pkg/util/extender/log.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package extender - -import "github.com/vesoft-inc/nebula-operator/pkg/logging" - -var log = logging.Log.WithName("extender") diff --git a/pkg/util/extender/unstructured.go b/pkg/util/extender/unstructured.go index a9aa6a92..a15f5e6b 100644 --- a/pkg/util/extender/unstructured.go +++ b/pkg/util/extender/unstructured.go @@ -22,6 +22,7 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/klog/v2" "k8s.io/utils/pointer" "github.com/vesoft-inc/nebula-operator/pkg/annotation" @@ -144,12 +145,12 @@ func templateEqual(oldTemplate, newTemplate map[string]interface{}) bool { var newVal, oldVal interface{} oldApply, _ := codec.Encode(oldTemplate) if err := json.Unmarshal([]byte(oldApply), &oldVal); err != nil { - log.Error(err, "unmarshal failed") + klog.Errorf("unmarshal failed, error: %v", err) return false } newApply, _ := codec.Encode(newTemplate) if err := json.Unmarshal([]byte(newApply), &newVal); err != nil { - log.Error(err, "unmarshal failed") + klog.Errorf("unmarshal failed, error: %v", err) return false } return apiequality.Semantic.DeepEqual(oldVal, newVal) @@ -161,9 +162,7 @@ func PodTemplateEqual(newUnstruct, oldUnstruct *unstructured.Unstructured) bool lastAppliedConfig, ok := oldUnstruct.GetAnnotations()[annotation.AnnLastAppliedConfigKey] if ok { if err := json.Unmarshal([]byte(lastAppliedConfig), &oldSpec); err != nil { - log.Error(err, "applied config failed", - "namespace", oldUnstruct.GetNamespace(), - "name", oldUnstruct.GetName()) + klog.Errorf("applied config failed, error: %v", err) return false } oldPodTemplate, _, _ := unstructured.NestedMap(oldSpec, "template", "spec") @@ -185,10 +184,7 @@ func ObjectEqual(newUnstruct, oldUnstruct *unstructured.Unstructured) bool { oldSpec := make(map[string]interface{}) if lastAppliedConfig, ok := oldUnstruct.GetAnnotations()[annotation.AnnLastAppliedConfigKey]; ok { if err := json.Unmarshal([]byte(lastAppliedConfig), &oldSpec); err != nil { - log.Error(err, "unmarshal failed", - "kind", oldUnstruct.GetKind(), - "namespace", oldUnstruct.GetNamespace(), - "name", oldUnstruct.GetName()) + klog.Errorf("unmarshal failed, error: %v", err) return false } newSpec := GetSpec(newUnstruct) diff --git a/pkg/util/randstr/randstr.go b/pkg/util/randstr/randstr.go new file mode 100644 index 00000000..ba735f74 --- /dev/null +++ b/pkg/util/randstr/randstr.go @@ -0,0 +1,19 @@ +package randstr + +import ( + "crypto/rand" + "encoding/hex" +) + +// Bytes generates n random bytes +func Bytes(n int) []byte { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + panic(err) + } + return b +} + +// Hex generates a random hex string with length of n +func Hex(n int) string { return hex.EncodeToString(Bytes(n)) } diff --git a/pkg/util/resource/resource.go b/pkg/util/resource/resource.go index 6cf9a675..dcfe7225 100644 --- a/pkg/util/resource/resource.go +++ b/pkg/util/resource/resource.go @@ -66,7 +66,7 @@ func GetUniteDeploymentGVR() schema.GroupVersionResource { } func GetGVKFromDefinition(dm discovery.Interface, ref v1alpha1.WorkloadReference) (schema.GroupVersionKind, error) { - // if given definitionRef is empty return an default GVK + // if given definitionRef is empty return a default GVK if ref.Name == "" { return StatefulSetKind, nil } diff --git a/pkg/util/restore/flatten.go b/pkg/util/restore/flatten.go new file mode 100644 index 00000000..156a8124 --- /dev/null +++ b/pkg/util/restore/flatten.go @@ -0,0 +1,84 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "fmt" + "strconv" + + "github.com/vesoft-inc/nebula-go/v3/nebula" + "github.com/vesoft-inc/nebula-go/v3/nebula/meta" +) + +// PartInfo save a unique part's information for FlattenBackupMeta +type PartInfo struct { + SpaceID string + PartID string + HostAddr *nebula.HostAddr + DataPath string + CheckpointPath string + CommitLogId int64 + LastLogId int64 +} + +// FlattenBackupMeta flatten backup meta to a map for convenience +// because of (spaceId + partId) can specify a unique part +func FlattenBackupMeta(backup *meta.BackupMeta) map[string]*PartInfo { + backupMap := make(map[string]*PartInfo) + for sid, sb := range backup.GetSpaceBackups() { + for _, hb := range sb.GetHostBackups() { + for _, ck := range hb.GetCheckpoints() { + for pid, part := range ck.GetParts() { + key := GenPartKey(strconv.Itoa(int(sid)), strconv.Itoa(int(pid))) + backupMap[key] = &PartInfo{ + SpaceID: strconv.Itoa(int(sid)), + PartID: strconv.Itoa(int(pid)), + HostAddr: hb.GetHost(), + DataPath: string(ck.DataPath), + CheckpointPath: string(part.CheckpointPath), + CommitLogId: part.CommitLogID, + LastLogId: part.LogID, + } + } + } + } + } + return backupMap +} + +// FlattenRestoreMeta flatten meta.RestoreMetaResp to a map for convenience +// because of (spaceId + partId) can specify a unique part +func FlattenRestoreMeta(resp *meta.RestoreMetaResp) map[string][]*nebula.HostAddr { + hostMap := make(map[string][]*nebula.HostAddr) + for sid, parts := range resp.GetPartHosts() { + for _, part := range parts { + key := GenPartKey(strconv.Itoa(int(sid)), strconv.Itoa(int(part.PartID))) + hostMap[key] = part.GetHosts() + } + } + return hostMap +} + +// GenPartKey generate a unique key for part +func GenPartKey(spaceId, partId string) string { + return fmt.Sprintf("%s-%s", spaceId, partId) +} + +// GenDataPathKey generate a unique key for data path +func GenDataPathKey(host *nebula.HostAddr, partKey string) string { + return fmt.Sprintf("%s-%s", StringifyAddr(host), partKey) +} diff --git a/pkg/util/restore/hosts.go b/pkg/util/restore/hosts.go new file mode 100644 index 00000000..7fa9bda1 --- /dev/null +++ b/pkg/util/restore/hosts.go @@ -0,0 +1,94 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "fmt" + + "github.com/vesoft-inc/nebula-go/v3/nebula" + "github.com/vesoft-inc/nebula-go/v3/nebula/meta" +) + +type NebulaHosts struct { + hosts map[string][]*meta.ServiceInfo // ip -> (agent, [storaged, metad, graphd, listener]) +} + +type HostDir struct { + Host string // ip + Dir string // nebula root dir +} + +func (h *NebulaHosts) LoadFrom(resp *meta.ListClusterInfoResp) error { + if resp.Code != nebula.ErrorCode_SUCCEEDED { + return fmt.Errorf("response is not successful, code is %s", resp.GetCode().String()) + } + + // only load metad、garphd、storaged、agent role + h.hosts = make(map[string][]*meta.ServiceInfo) + for host, services := range resp.GetHostServices() { + for _, s := range services { + switch s.GetRole() { + case meta.HostRole_GRAPH, meta.HostRole_META, meta.HostRole_STORAGE, meta.HostRole_AGENT: + h.hosts[host] = append(h.hosts[host], s) + } + + } + } + + // check only one agent in each host + for _, services := range h.hosts { + var agentAddr *nebula.HostAddr + for _, s := range services { + if s.GetRole() == meta.HostRole_AGENT { + if agentAddr == nil { + agentAddr = s.GetAddr() + } else { + return fmt.Errorf("there are more than one agent in host %s: %s, %s", s.GetAddr().GetHost(), + StringifyAddr(agentAddr), StringifyAddr(s.GetAddr())) + } + } + } + } + + return nil +} + +func (h *NebulaHosts) GetMetas() []*meta.ServiceInfo { + var sl []*meta.ServiceInfo + for _, services := range h.hosts { + for _, s := range services { + if s.Role == meta.HostRole_META { + sl = append(sl, s) + } + } + } + + return sl +} + +func (h *NebulaHosts) GetStorages() []*meta.ServiceInfo { + var sl []*meta.ServiceInfo + for _, services := range h.hosts { + for _, s := range services { + if s.Role == meta.HostRole_STORAGE { + sl = append(sl, s) + } + } + } + + return sl +} diff --git a/pkg/util/restore/restore.go b/pkg/util/restore/restore.go new file mode 100644 index 00000000..7a0348c1 --- /dev/null +++ b/pkg/util/restore/restore.go @@ -0,0 +1,100 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "fmt" + "net/url" + "os" + "path" + "strconv" + "strings" + + "github.com/facebook/fbthrift/thrift/lib/go/thrift" + pb "github.com/vesoft-inc/nebula-agent/pkg/proto" + "github.com/vesoft-inc/nebula-go/v3/nebula" + "github.com/vesoft-inc/nebula-go/v3/nebula/meta" +) + +const LocalTmpDir = "/tmp/nebula-br" + +type Config struct { + BackupName string + Concurrency int32 + Backend *pb.Backend +} + +func ParseAddr(addrStr string) (*nebula.HostAddr, error) { + ipAddr := strings.Split(addrStr, ":") + if len(ipAddr) != 2 { + return nil, fmt.Errorf("invalid address format: %s", addrStr) + } + + port, err := strconv.ParseInt(ipAddr[1], 10, 32) + if err != nil { + return nil, err + } + + return &nebula.HostAddr{Host: ipAddr[0], Port: nebula.Port(port)}, nil +} + +func ParseMetaFromFile(filename string) (*meta.BackupMeta, error) { + file, err := os.OpenFile(filename, os.O_RDONLY, 0644) + if err != nil { + return nil, err + } + defer file.Close() + + st := thrift.NewStreamTransport(file, file) + defer st.Close() + + ft := thrift.NewFramedTransport(st) + defer ft.Close() + + bp := thrift.NewBinaryProtocol(ft, false, true) + m := meta.NewBackupMeta() + if err = m.Read(bp); err != nil { + return nil, err + } + return m, nil +} + +func UriJoin(elem ...string) (string, error) { + if len(elem) == 0 { + return "", fmt.Errorf("empty paths") + } + + if len(elem) == 1 { + return elem[0], nil + } + + u, err := url.Parse(elem[0]) + if err != nil { + return "", fmt.Errorf("parse base uri %s failed: %v", elem[0], err) + } + + elem[0] = u.Path + u.Path = path.Join(elem...) + return u.String(), nil +} + +func StringifyAddr(addr *nebula.HostAddr) string { + if addr == nil { + return "nil" + } + return fmt.Sprintf("%s:%d", addr.GetHost(), addr.GetPort()) +} diff --git a/pkg/util/restore/selector.go b/pkg/util/restore/selector.go new file mode 100644 index 00000000..50ccc3a2 --- /dev/null +++ b/pkg/util/restore/selector.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 Vesoft Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import "github.com/vesoft-inc/nebula-go/v3/nebula/meta" + +type PathSelector struct { + paths [][]byte + index int +} + +func NewPathSelector(paths [][]byte) *PathSelector { + return &PathSelector{ + paths: paths, + index: 0, + } +} + +func (p *PathSelector) EvenlyGet() string { + path := string(p.paths[p.index]) + p.index = (p.index + 1) % len(p.paths) + return path +} + +func NewPathSelectorMap(storages []*meta.ServiceInfo) map[string]*PathSelector { + mappings := make(map[string]*PathSelector) + for _, storage := range storages { + mappings[StringifyAddr(storage.GetAddr())] = NewPathSelector(storage.GetDir().GetData()) + } + return mappings +} diff --git a/pkg/webhook/log.go b/pkg/webhook/log.go deleted file mode 100644 index d3e5c0ac..00000000 --- a/pkg/webhook/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("webhook") - -func getLog() logr.Logger { return _log } diff --git a/pkg/webhook/nebulacluster/validating/log.go b/pkg/webhook/nebulacluster/validating/log.go deleted file mode 100644 index d77bf5df..00000000 --- a/pkg/webhook/nebulacluster/validating/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validating - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("webhook").WithName("validating") - -func getLog() logr.Logger { return _log } diff --git a/pkg/webhook/nebulacluster/validating/nebulagraph_create_update_handler.go b/pkg/webhook/nebulacluster/validating/nebulagraph_create_update_handler.go index 075b6d8a..6d2f6b6a 100644 --- a/pkg/webhook/nebulacluster/validating/nebulagraph_create_update_handler.go +++ b/pkg/webhook/nebulacluster/validating/nebulagraph_create_update_handler.go @@ -21,6 +21,7 @@ import ( "net/http" admissionv1 "k8s.io/api/admission/v1" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -44,12 +45,11 @@ var _ admission.Handler = &NebulaClusterCreateUpdateHandler{} // Handle handles admission requests. func (h *NebulaClusterCreateUpdateHandler) Handle(_ context.Context, req admission.Request) (resp admission.Response) { - log := getLog().WithValues("resource", req.Resource, - "namespace", req.Namespace, "name", req.Name, "operation", req.Operation) - log.Info("start validating") + klog.Infof("start validating resource %v [%s/%s] operation %s", req.Resource, req.Namespace, req.Name, req.Operation) + defer func() { - log.Info("end validating", "allowed", resp.Allowed, - "reason", resp.Result.Reason, "message", resp.Result.Message) + klog.Infof("end validating, allowed %v, reason %v, message %s", "allowed", resp.Allowed, + resp.Result.Reason, resp.Result.Message) }() obj := &v1alpha1.NebulaCluster{} diff --git a/pkg/webhook/nebulacluster/validating/nebulagraph_validation.go b/pkg/webhook/nebulacluster/validating/nebulagraph_validation.go index 1d1e98fc..e974fa1a 100644 --- a/pkg/webhook/nebulacluster/validating/nebulagraph_validation.go +++ b/pkg/webhook/nebulacluster/validating/nebulagraph_validation.go @@ -21,6 +21,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog/v2" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" "github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1" @@ -96,10 +97,8 @@ func validateNebulaClusterCreate(nc *v1alpha1.NebulaCluster) (allErrs field.Erro name := nc.Name namespace := nc.Namespace - log := getLog().WithValues("gvk", nc.GroupVersionKind(), - "namespace", namespace, "name", name, "operation", admissionv1.Create) - - log.Info("receive admission") + klog.Infof("receive admission with resource [%s/%s], GVK %s, operation %s", namespace, name, + nc.GroupVersionKind().String(), admissionv1.Create) allErrs = append(allErrs, validateNebulaClusterCreateGraphd(nc)...) allErrs = append(allErrs, validateNebulaClusterCreateMetad(nc)...) @@ -130,14 +129,6 @@ func validateNebulaClusterUpdateMetad(nc, oldNC *v1alpha1.NebulaCluster) (allErr // validateNebulaClusterStoraged validates a NebulaCluster for Storaged on Update. func validateNebulaClusterUpdateStoraged(nc, oldNC *v1alpha1.NebulaCluster) (allErrs field.ErrorList) { - name := nc.Name - namespace := nc.Namespace - log := getLog().WithValues("gvk", nc.GroupVersionKind(), - "namespace", namespace, "name", name, "operation", admissionv1.Update) - - log.Info("replicas conversion info", "phase", nc.Status.Storaged.Phase, - "old replicas", *oldNC.Spec.Storaged.Replicas, "new replicas", *nc.Spec.Storaged.Replicas) - if nc.Status.Storaged.Phase != v1alpha1.RunningPhase { if *nc.Spec.Storaged.Replicas != *oldNC.Spec.Storaged.Replicas { allErrs = append(allErrs, field.Invalid( @@ -157,10 +148,9 @@ func validateNebulaClusterUpdateStoraged(nc, oldNC *v1alpha1.NebulaCluster) (all func validateNebulaClusterUpdate(nc, oldNC *v1alpha1.NebulaCluster) (allErrs field.ErrorList) { name := nc.Name namespace := nc.Namespace - log := getLog().WithValues("gvk", nc.GroupVersionKind(), - "namespace", namespace, "name", name, "operation", admissionv1.Update) - log.Info("receive admission") + klog.Infof("receive admission with resource [%s/%s], GVK %s, operation %s", namespace, name, + nc.GroupVersionKind().String(), admissionv1.Update) allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate( &nc.ObjectMeta, diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index e2f64c1b..5434b1fc 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -17,6 +17,7 @@ limitations under the License. package webhook import ( + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -26,10 +27,9 @@ import ( var HandlerMap = map[string]admission.Handler{} func registerHandlers(m map[string]admission.Handler) { - log := getLog() for path, handler := range m { if path == "" { - log.Info("Skip handler with empty path.") + klog.Info("Skip handler with empty path.") continue } if path[0] != '/' { @@ -37,20 +37,19 @@ func registerHandlers(m map[string]admission.Handler) { } _, found := HandlerMap[path] if found { - log.V(1).Info("conflicting webhook path in handler map", "path", path) + klog.V(1).Infof("conflicting webhook path %s in handler map", path) } HandlerMap[path] = handler } } func SetupWithManager(mgr manager.Manager) error { - log := getLog() server := mgr.GetWebhookServer() // register admission handlers for path, handler := range HandlerMap { server.Register(path, &webhook.Admission{Handler: handler}) - log.V(3).Info("Registered webhook handler", "path", path) + klog.V(3).Infof("Registered webhook handler with path %s", path) } return nil diff --git a/pkg/webhook/statefulset/mutating/log.go b/pkg/webhook/statefulset/mutating/log.go deleted file mode 100644 index 665077c5..00000000 --- a/pkg/webhook/statefulset/mutating/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mutating - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("webhook").WithName("mutating") - -func getLog() logr.Logger { return _log } diff --git a/pkg/webhook/statefulset/mutating/statefulset_create_update_handler.go b/pkg/webhook/statefulset/mutating/statefulset_create_update_handler.go index 63a7dd9f..2ac3b38d 100644 --- a/pkg/webhook/statefulset/mutating/statefulset_create_update_handler.go +++ b/pkg/webhook/statefulset/mutating/statefulset_create_update_handler.go @@ -38,9 +38,6 @@ var _ admission.Handler = &StatefulSetCreateUpdateHandler{} // Handle handles admission requests. func (h *StatefulSetCreateUpdateHandler) Handle(_ context.Context, req admission.Request) admission.Response { - log := getLog().WithValues("resource", req.Resource, - "namespace", req.Namespace, "name", req.Name, "operation", req.Operation) - log.Info("start mutating") return admission.Patched("") } diff --git a/pkg/webhook/statefulset/validating/log.go b/pkg/webhook/statefulset/validating/log.go deleted file mode 100644 index d77bf5df..00000000 --- a/pkg/webhook/statefulset/validating/log.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 Vesoft Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validating - -import ( - "github.com/go-logr/logr" - - "github.com/vesoft-inc/nebula-operator/pkg/logging" -) - -// Please don't use directly, but use getLog. -// Examples: -// log := getLog().WithName("name").WithValues("key", "value") -// log.Info(...) -var _log = logging.Log.WithName("webhook").WithName("validating") - -func getLog() logr.Logger { return _log } diff --git a/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go b/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go index e1b7825a..1b1cce45 100644 --- a/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go +++ b/pkg/webhook/statefulset/validating/statefulset_create_update_handler.go @@ -27,6 +27,7 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -48,12 +49,11 @@ var _ admission.Handler = &StatefulSetCreateUpdateHandler{} // Handle handles admission requests. func (h *StatefulSetCreateUpdateHandler) Handle(ctx context.Context, req admission.Request) (resp admission.Response) { - log := getLog().WithValues("resource", req.Resource, - "namespace", req.Namespace, "name", req.Name, "operation", req.Operation) - log.Info("start validating") + klog.Infof("start validating resource %v [%s/%s] operation %s", req.Resource, req.Namespace, req.Name, req.Operation) + defer func() { - log.Info("end validating", "allowed", resp.Allowed, - "reason", resp.Result.Reason, "message", resp.Result.Message) + klog.Infof("finish validating, allowed %v, reason %v, message %s", "allowed", resp.Allowed, + resp.Result.Reason, resp.Result.Message) }() var obj client.Object = &appsv1.StatefulSet{} diff --git a/pkg/webhook/statefulset/validating/statefulset_validation.go b/pkg/webhook/statefulset/validating/statefulset_validation.go index 54ba2401..ab99a823 100644 --- a/pkg/webhook/statefulset/validating/statefulset_validation.go +++ b/pkg/webhook/statefulset/validating/statefulset_validation.go @@ -23,6 +23,7 @@ import ( appsv1 "k8s.io/api/apps/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog/v2" appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation" apivalidation "k8s.io/kubernetes/pkg/apis/core/validation" @@ -95,10 +96,8 @@ func validateStatefulSetCreate(statefulSet *appsv1.StatefulSet) (allErrs field.E namespace := statefulSet.Namespace l := label.Label(statefulSet.Labels) - log := getLog().WithValues("gvk", statefulSet.GroupVersionKind(), - "namespace", namespace, "name", name, "operation", admissionv1.Create) - - log.Info("receive admission") + klog.Infof("receive admission with resource [%s/%s], GVK %s, operation %s", namespace, name, + statefulSet.GroupVersionKind().String(), admissionv1.Create) allErrs = append(allErrs, apivalidation.ValidateObjectMeta( &statefulSet.ObjectMeta, @@ -156,10 +155,8 @@ func validateStatefulSetUpdate(statefulSet, oldStatefulSet *appsv1.StatefulSet) namespace := statefulSet.Namespace l := label.Label(statefulSet.Labels) - log := getLog().WithValues("gvk", statefulSet.GroupVersionKind(), - "namespace", namespace, "name", name, "operation", admissionv1.Update) - - log.Info("receive admission") + klog.Infof("receive admission with resource [%s/%s], GVK %s, operation %s", namespace, name, + statefulSet.GroupVersionKind().String(), admissionv1.Create) allErrs = append(allErrs, apivalidation.ValidateObjectMetaUpdate( &statefulSet.ObjectMeta, @@ -221,16 +218,13 @@ func isManaged(statefulSet *appsv1.StatefulSet) bool { namespace := statefulSet.Namespace l := label.Label(statefulSet.Labels) - log := getLog().WithValues("gvk", statefulSet.GroupVersionKind(), - "namespace", namespace, "name", name, "operation", admissionv1.Create) - if !l.IsManagedByNebulaOperator() { - log.Info("not managed by Nebula Operator, admit") + klog.Infof("resource [%s/%s] not managed by Nebula Operator, admit", namespace, name) return false } if !(l.IsGraphd() || l.IsMetad() || l.IsStoraged()) { - log.Info("not Nebula component, admit") + klog.Infof("resource [%s/%s] not Nebula component, admit", namespace, name) return false }