From d8b0b4fbc9f31de93478bd04980c4cc8701a6669 Mon Sep 17 00:00:00 2001 From: Daniel Shubin Date: Tue, 28 Jan 2025 20:32:03 +0000 Subject: [PATCH] [PLAT-16383] K8S operator Storage config and Release to use secrets Summary: The operator previously took tokens, passwords, and other secrets as plain text directly in the storage config and release CRDs. This updates them to use secrets instead. The old method is not deprecated, but if a secret is provided it is used instead of the plain text secret Test Plan: created storage config and release with secrets Reviewers: anijhawan, vkumar Reviewed By: anijhawan Subscribers: yugaware Differential Revision: https://phorge.dev.yugabyte.com/D41525 --- .../yw/common/operator/ReleaseReconciler.java | 33 ++++++++++++ .../operator/StorageConfigReconciler.java | 45 ++++++++++++++-- .../common/operator/resources/releaseCrd.yaml | 28 +++++++++- .../operator/resources/storageConfigCrd.yaml | 54 +++++++++++++++++-- .../common/operator/utils/OperatorUtils.java | 35 ++++++++++++ 5 files changed, 187 insertions(+), 8 deletions(-) diff --git a/managed/src/main/java/com/yugabyte/yw/common/operator/ReleaseReconciler.java b/managed/src/main/java/com/yugabyte/yw/common/operator/ReleaseReconciler.java index f70eeba46d58..97bc142025e7 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/operator/ReleaseReconciler.java +++ b/managed/src/main/java/com/yugabyte/yw/common/operator/ReleaseReconciler.java @@ -22,6 +22,8 @@ import io.fabric8.kubernetes.client.informers.cache.Lister; import io.yugabyte.operator.v1alpha1.Release; import io.yugabyte.operator.v1alpha1.ReleaseStatus; +import io.yugabyte.operator.v1alpha1.releasespec.config.downloadconfig.gcs.CredentialsJsonSecret; +import io.yugabyte.operator.v1alpha1.releasespec.config.downloadconfig.s3.SecretAccessKeySecret; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -198,6 +200,11 @@ private void updateStatus(Release release, String status, Boolean success) { private List createReleaseArtifacts(Release release) throws Exception { ReleaseArtifact dbArtifact = null; ReleaseArtifact helmArtifact = null; + if (release.getSpec().getConfig() == null + || release.getSpec().getConfig().getDownloadConfig() == null) { + throw new Exception( + String.format("No download config found release %s", release.getMetadata().getName())); + } if (release.getSpec().getConfig().getDownloadConfig().getS3() != null) { ReleaseArtifact.S3File s3File = new ReleaseArtifact.S3File(); s3File.path = @@ -206,6 +213,19 @@ private List createReleaseArtifacts(Release release) throws Exc release.getSpec().getConfig().getDownloadConfig().getS3().getAccessKeyId(); s3File.secretAccessKey = release.getSpec().getConfig().getDownloadConfig().getS3().getSecretAccessKey(); + if (release.getSpec().getConfig().getDownloadConfig().getS3().getSecretAccessKeySecret() + != null) { + SecretAccessKeySecret awsSecret = + release.getSpec().getConfig().getDownloadConfig().getS3().getSecretAccessKeySecret(); + String secret = + operatorUtils.getAndParseSecretForKey( + awsSecret.getName(), awsSecret.getNamespace(), "AWS_SECRET_ACCESS_KEY"); + if (secret != null) { + s3File.secretAccessKey = secret; + } else { + log.warn("Aws secret access key secret {} not found", awsSecret.getName()); + } + } dbArtifact = ReleaseArtifact.create( release @@ -238,6 +258,19 @@ private List createReleaseArtifacts(Release release) throws Exc release.getSpec().getConfig().getDownloadConfig().getGcs().getPaths().getX86_64(); gcsFile.credentialsJson = release.getSpec().getConfig().getDownloadConfig().getGcs().getCredentialsJson(); + if (release.getSpec().getConfig().getDownloadConfig().getGcs().getCredentialsJsonSecret() + != null) { + CredentialsJsonSecret gcsSecret = + release.getSpec().getConfig().getDownloadConfig().getGcs().getCredentialsJsonSecret(); + String secret = + operatorUtils.getAndParseSecretForKey( + gcsSecret.getName(), gcsSecret.getNamespace(), "CREDENTIALS_JSON"); + if (secret != null) { + gcsFile.credentialsJson = secret; + } else { + log.warn("Gcs credentials json secret {} not found", gcsSecret.getName()); + } + } dbArtifact = ReleaseArtifact.create( release diff --git a/managed/src/main/java/com/yugabyte/yw/common/operator/StorageConfigReconciler.java b/managed/src/main/java/com/yugabyte/yw/common/operator/StorageConfigReconciler.java index 0a0046fca792..1a221e83101d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/operator/StorageConfigReconciler.java +++ b/managed/src/main/java/com/yugabyte/yw/common/operator/StorageConfigReconciler.java @@ -11,6 +11,7 @@ import com.yugabyte.yw.models.configs.CustomerConfig; import com.yugabyte.yw.models.helpers.CustomerConfigConsts; import io.fabric8.kubernetes.api.model.KubernetesResourceList; +import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.Resource; import io.fabric8.kubernetes.client.informers.ResourceEventHandler; @@ -18,7 +19,10 @@ import io.fabric8.kubernetes.client.informers.cache.Lister; import io.yugabyte.operator.v1alpha1.StorageConfig; import io.yugabyte.operator.v1alpha1.StorageConfigStatus; +import io.yugabyte.operator.v1alpha1.storageconfigspec.AwsSecretAccessKeySecret; +import io.yugabyte.operator.v1alpha1.storageconfigspec.AzureStorageSasTokenSecret; import io.yugabyte.operator.v1alpha1.storageconfigspec.Data; +import io.yugabyte.operator.v1alpha1.storageconfigspec.GcsCredentialsJsonSecret; import java.util.Objects; import java.util.UUID; import lombok.extern.slf4j.Slf4j; @@ -55,7 +59,7 @@ public String getCustomerUUID() throws Exception { return cust.getUuid().toString(); } - public static JsonNode getConfigPayloadFromCRD(StorageConfig sc) { + public JsonNode getConfigPayloadFromCRD(StorageConfig sc) { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); @@ -91,6 +95,7 @@ public static JsonNode getConfigPayloadFromCRD(StorageConfig sc) { } object.put(iamFieldName, useIAM); } + parseSecrets(object, sc); return dataJson; } @@ -118,6 +123,41 @@ private void updateStatus(StorageConfig sc, boolean success, String configUUID, resourceClient.inNamespace(namespace).resource(sc).replaceStatus(); } + private void parseSecrets(ObjectNode configObject, StorageConfig sc) { + AwsSecretAccessKeySecret awsSecret = sc.getSpec().getAwsSecretAccessKeySecret(); + if (awsSecret != null) { + Secret secret = operatorUtils.getSecret(awsSecret.getName(), awsSecret.getNamespace()); + if (secret != null) { + String awsSecretKey = operatorUtils.parseSecretForKey(secret, "AWS_SECRET_ACCESS_KEY"); + configObject.put("AWS_SECRET_ACCESS_KEY", awsSecretKey); + } else { + log.warn("AWS secret access key secret {} not found", awsSecret.getName()); + } + } + + GcsCredentialsJsonSecret gcsSecret = sc.getSpec().getGcsCredentialsJsonSecret(); + if (gcsSecret != null) { + Secret secret = operatorUtils.getSecret(gcsSecret.getName(), gcsSecret.getNamespace()); + if (secret != null) { + String gcsSecretKey = operatorUtils.parseSecretForKey(secret, "GCS_CREDENTIALS_JSON"); + configObject.put("GCS_CREDENTIALS_JSON", gcsSecretKey); + } else { + log.warn("GCS credentials json secret {} not found", gcsSecret.getName()); + } + } + + AzureStorageSasTokenSecret azureSecret = sc.getSpec().getAzureStorageSasTokenSecret(); + if (azureSecret != null) { + Secret secret = operatorUtils.getSecret(azureSecret.getName(), azureSecret.getNamespace()); + if (secret != null) { + String azureSecretKey = operatorUtils.parseSecretForKey(secret, "AZURE_STORAGE_SAS_TOKEN"); + configObject.put("AZURE_STORAGE_SAS_TOKEN", azureSecretKey); + } else { + log.warn("Azure storage sas token secret {} not found", azureSecret.getName()); + } + } + } + @Override public void onAdd(StorageConfig sc) { if (sc.getStatus() != null) { @@ -180,8 +220,7 @@ public void onUpdate(StorageConfig oldSc, StorageConfig newSc) { cc.setData((ObjectNode) payload); this.ccs.edit(cc); } catch (Exception e) { - log.error("Got Error {}", e); - log.info("Failed updating storageconfig {}, ", oldSc.getMetadata().getName()); + log.error("Failed updating storageconfig {}, ", oldSc.getMetadata().getName(), e); updateStatus(newSc, false, configUUID, e.getMessage()); return; } diff --git a/managed/src/main/java/com/yugabyte/yw/common/operator/resources/releaseCrd.yaml b/managed/src/main/java/com/yugabyte/yw/common/operator/resources/releaseCrd.yaml index a7f6fc5e9e2b..0a865bf220dc 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/operator/resources/releaseCrd.yaml +++ b/managed/src/main/java/com/yugabyte/yw/common/operator/resources/releaseCrd.yaml @@ -79,8 +79,20 @@ spec: description: S3 access key type: string secretAccessKey: - description: S3 secret key + description: S3 secret key. Deprecated, use secretAccessKeySecret type: string + secretAccessKeySecret: + description: S3 secret key secret. Overrides secretAccessKey + type: object + properties: + name: + description: Name of the secret + type: string + namespace: + description: Namespace of the secret + type: string + required: + - name paths: description: S3 paths to download the release from type: object @@ -117,7 +129,19 @@ spec: description: Optional checksum for Helm chart package credentialsJson: type: string - description: GCS service key JSON + description: | + GCS service key JSON. Deprecated, use credentialsJsonSecret instead. + credentialsJsonSecret: + type: object + properties: + name: + description: Name of the secret + type: string + namespace: + description: Namespace of the secret + type: string + required: + - name http: type: object properties: diff --git a/managed/src/main/java/com/yugabyte/yw/common/operator/resources/storageConfigCrd.yaml b/managed/src/main/java/com/yugabyte/yw/common/operator/resources/storageConfigCrd.yaml index e0cad9827b97..9c0b1f1c2313 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/operator/resources/storageConfigCrd.yaml +++ b/managed/src/main/java/com/yugabyte/yw/common/operator/resources/storageConfigCrd.yaml @@ -62,7 +62,9 @@ spec: description: AWS access key id for the S3 storage configuration. type: string AWS_SECRET_ACCESS_KEY: - description: AWS secret access key for the S3 storage configuration. + description: | + AWS secret access key for the S3 storage configuration. Deprecated, use + aws_secret_access_key_secret instead type: string USE_IAM: description: Use IAM for storage account access. Valid for S3/GCS. @@ -74,10 +76,56 @@ spec: - message: BACKUP_LOCATION cannot be changed rule: self == oldSelf GCS_CREDENTIALS_JSON: - description: GCS credentials JSON for the GCS storage configuration. + description: | + GCS credentials JSON for the GCS storage configuration. Deprecated, use + gcs_credentials_json_secret instead type: string AZURE_STORAGE_SAS_TOKEN: - description: Azure SAS token for the Azure storage configuration. + description: | + Azure SAS token for the Azure storage configuration. Deprecated, use + azure_storage_sas_token_secret instead type: string required: - BACKUP_LOCATION + awsSecretAccessKeySecret: + type: object + description: | + Name of the secret containing AWS_SECRET_ACCESS_KEY for the S3 storage + configuration. The secret will take precedence over data.AWS_SECRET_ACCESS_KEY + properties: + name: + type: string + description: Name of the secret + namespace: + type: string + description: Namespace of the secret + required: + - name + gcsCredentialsJsonSecret: + type: object + description: | + Name of the secret containing GCS_CREDENTIALS_JSON for the GCS storage + configuration. The secret will take precedence over data.GCS_CREDENTIALS_JSON + properties: + name: + type: string + description: Name of the secret + namespace: + type: string + description: Namespace of the secret + required: + - name + azureStorageSasTokenSecret: + type: object + description: | + Name of the secret containing AZURE_STORAGE_SAS_TOKEN for the Azure storage + configuration. The secret will take precedence over data.AZURE_STORAGE_SAS_TOKEN + properties: + name: + type: string + description: Name of the secret + namespace: + type: string + description: Namespace of the secret + required: + - name diff --git a/managed/src/main/java/com/yugabyte/yw/common/operator/utils/OperatorUtils.java b/managed/src/main/java/com/yugabyte/yw/common/operator/utils/OperatorUtils.java index 1262662ef95a..7d4267f97dc1 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/operator/utils/OperatorUtils.java +++ b/managed/src/main/java/com/yugabyte/yw/common/operator/utils/OperatorUtils.java @@ -28,6 +28,7 @@ import com.yugabyte.yw.models.helpers.DeviceInfo; import com.yugabyte.yw.models.helpers.TaskType; import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.KubernetesClient; @@ -35,6 +36,7 @@ import io.yugabyte.operator.v1alpha1.Release; import io.yugabyte.operator.v1alpha1.YBUniverse; import io.yugabyte.operator.v1alpha1.releasespec.config.DownloadConfig; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -421,4 +423,37 @@ public void deleteReleaseCr(Release release) { } log.info("Removed release {}", release.getMetadata().getName()); } + + public String getAndParseSecretForKey(String name, @Nullable String namespace, String key) { + Secret secret = getSecret(name, namespace); + if (secret == null) { + log.warn("Secret {} not found", name); + return null; + } + return parseSecretForKey(secret, key); + } + + public Secret getSecret(String name, @Nullable String namespace) { + try (final KubernetesClient kubernetesClient = + new KubernetesClientBuilder().withConfig(k8sClientConfig).build()) { + if (StringUtils.isBlank(namespace)) { + log.info("Getting secret '{}' from default namespace", name); + namespace = "default"; + } + return kubernetesClient.secrets().inNamespace(namespace).withName(name).get(); + } + } + + // parseSecretForKey checks secret data for the key. If not found, it will then check stringData. + // Returns null if the key is not found at all. + // Also handles null secret. + public String parseSecretForKey(Secret secret, String key) { + if (secret == null) { + return null; + } + if (secret.getData().get(key) != null) { + return new String(Base64.getDecoder().decode(secret.getData().get(key))); + } + return secret.getStringData().get(key); + } }