Skip to content

Commit

Permalink
[PLAT-16383] K8S operator Storage config and Release to use secrets
Browse files Browse the repository at this point in the history 
Summary:
The operator previously took tokens, passwords, and other secrets as plain text directly
in the storage config and release CRDs. This updates them to use secrets instead.
The old method is not deprecated, but if a secret is provided it is used instead of
the plain text secret

Test Plan: created storage config and release with secrets

Reviewers: anijhawan, vkumar

Reviewed By: anijhawan

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D41525
  • Loading branch information
@shubin-yb
shubin-yb committed Feb 3, 2025
1 parent 3c955c0 commit d8b0b4f
Show file tree
Hide file tree
Showing 5 changed files with 187 additions and 8 deletions.
33 changes: 33 additions & 0 deletions managed/src/main/java/com/yugabyte/yw/common/operator/ReleaseReconciler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import io.fabric8.kubernetes.client.informers.cache.Lister;
import io.yugabyte.operator.v1alpha1.Release;
import io.yugabyte.operator.v1alpha1.ReleaseStatus;
import io.yugabyte.operator.v1alpha1.releasespec.config.downloadconfig.gcs.CredentialsJsonSecret;
import io.yugabyte.operator.v1alpha1.releasespec.config.downloadconfig.s3.SecretAccessKeySecret;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -198,6 +200,11 @@ private void updateStatus(Release release, String status, Boolean success) {
private List<ReleaseArtifact> createReleaseArtifacts(Release release) throws Exception {
ReleaseArtifact dbArtifact = null;
ReleaseArtifact helmArtifact = null;
if (release.getSpec().getConfig() == null
|| release.getSpec().getConfig().getDownloadConfig() == null) {
throw new Exception(
String.format("No download config found release %s", release.getMetadata().getName()));
}
if (release.getSpec().getConfig().getDownloadConfig().getS3() != null) {
ReleaseArtifact.S3File s3File = new ReleaseArtifact.S3File();
s3File.path =
Expand All @@ -206,6 +213,19 @@ private List<ReleaseArtifact> createReleaseArtifacts(Release release) throws Exc
release.getSpec().getConfig().getDownloadConfig().getS3().getAccessKeyId();
s3File.secretAccessKey =
release.getSpec().getConfig().getDownloadConfig().getS3().getSecretAccessKey();
if (release.getSpec().getConfig().getDownloadConfig().getS3().getSecretAccessKeySecret()
!= null) {
SecretAccessKeySecret awsSecret =
release.getSpec().getConfig().getDownloadConfig().getS3().getSecretAccessKeySecret();
String secret =
operatorUtils.getAndParseSecretForKey(
awsSecret.getName(), awsSecret.getNamespace(), "AWS_SECRET_ACCESS_KEY");
if (secret != null) {
s3File.secretAccessKey = secret;
} else {
log.warn("Aws secret access key secret {} not found", awsSecret.getName());
}
}
dbArtifact =
ReleaseArtifact.create(
release
Expand Down Expand Up @@ -238,6 +258,19 @@ private List<ReleaseArtifact> createReleaseArtifacts(Release release) throws Exc
release.getSpec().getConfig().getDownloadConfig().getGcs().getPaths().getX86_64();
gcsFile.credentialsJson =
release.getSpec().getConfig().getDownloadConfig().getGcs().getCredentialsJson();
if (release.getSpec().getConfig().getDownloadConfig().getGcs().getCredentialsJsonSecret()
!= null) {
CredentialsJsonSecret gcsSecret =
release.getSpec().getConfig().getDownloadConfig().getGcs().getCredentialsJsonSecret();
String secret =
operatorUtils.getAndParseSecretForKey(
gcsSecret.getName(), gcsSecret.getNamespace(), "CREDENTIALS_JSON");
if (secret != null) {
gcsFile.credentialsJson = secret;
} else {
log.warn("Gcs credentials json secret {} not found", gcsSecret.getName());
}
}
dbArtifact =
ReleaseArtifact.create(
release
Expand Down
45 changes: 42 additions & 3 deletions managed/src/main/java/com/yugabyte/yw/common/operator/StorageConfigReconciler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,18 @@
import com.yugabyte.yw.models.configs.CustomerConfig;
import com.yugabyte.yw.models.helpers.CustomerConfigConsts;
import io.fabric8.kubernetes.api.model.KubernetesResourceList;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.client.dsl.MixedOperation;
import io.fabric8.kubernetes.client.dsl.Resource;
import io.fabric8.kubernetes.client.informers.ResourceEventHandler;
import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
import io.fabric8.kubernetes.client.informers.cache.Lister;
import io.yugabyte.operator.v1alpha1.StorageConfig;
import io.yugabyte.operator.v1alpha1.StorageConfigStatus;
import io.yugabyte.operator.v1alpha1.storageconfigspec.AwsSecretAccessKeySecret;
import io.yugabyte.operator.v1alpha1.storageconfigspec.AzureStorageSasTokenSecret;
import io.yugabyte.operator.v1alpha1.storageconfigspec.Data;
import io.yugabyte.operator.v1alpha1.storageconfigspec.GcsCredentialsJsonSecret;
import java.util.Objects;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -55,7 +59,7 @@ public String getCustomerUUID() throws Exception {
return cust.getUuid().toString();
}

public static JsonNode getConfigPayloadFromCRD(StorageConfig sc) {
public JsonNode getConfigPayloadFromCRD(StorageConfig sc) {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true);

Expand Down Expand Up @@ -91,6 +95,7 @@ public static JsonNode getConfigPayloadFromCRD(StorageConfig sc) {
}
object.put(iamFieldName, useIAM);
}
parseSecrets(object, sc);

return dataJson;
}
Expand Down Expand Up @@ -118,6 +123,41 @@ private void updateStatus(StorageConfig sc, boolean success, String configUUID,
resourceClient.inNamespace(namespace).resource(sc).replaceStatus();
}

private void parseSecrets(ObjectNode configObject, StorageConfig sc) {
AwsSecretAccessKeySecret awsSecret = sc.getSpec().getAwsSecretAccessKeySecret();
if (awsSecret != null) {
Secret secret = operatorUtils.getSecret(awsSecret.getName(), awsSecret.getNamespace());
if (secret != null) {
String awsSecretKey = operatorUtils.parseSecretForKey(secret, "AWS_SECRET_ACCESS_KEY");
configObject.put("AWS_SECRET_ACCESS_KEY", awsSecretKey);
} else {
log.warn("AWS secret access key secret {} not found", awsSecret.getName());
}
}

GcsCredentialsJsonSecret gcsSecret = sc.getSpec().getGcsCredentialsJsonSecret();
if (gcsSecret != null) {
Secret secret = operatorUtils.getSecret(gcsSecret.getName(), gcsSecret.getNamespace());
if (secret != null) {
String gcsSecretKey = operatorUtils.parseSecretForKey(secret, "GCS_CREDENTIALS_JSON");
configObject.put("GCS_CREDENTIALS_JSON", gcsSecretKey);
} else {
log.warn("GCS credentials json secret {} not found", gcsSecret.getName());
}
}

AzureStorageSasTokenSecret azureSecret = sc.getSpec().getAzureStorageSasTokenSecret();
if (azureSecret != null) {
Secret secret = operatorUtils.getSecret(azureSecret.getName(), azureSecret.getNamespace());
if (secret != null) {
String azureSecretKey = operatorUtils.parseSecretForKey(secret, "AZURE_STORAGE_SAS_TOKEN");
configObject.put("AZURE_STORAGE_SAS_TOKEN", azureSecretKey);
} else {
log.warn("Azure storage sas token secret {} not found", azureSecret.getName());
}
}
}

@Override
public void onAdd(StorageConfig sc) {
if (sc.getStatus() != null) {
Expand Down Expand Up @@ -180,8 +220,7 @@ public void onUpdate(StorageConfig oldSc, StorageConfig newSc) {
cc.setData((ObjectNode) payload);
this.ccs.edit(cc);
} catch (Exception e) {
log.error("Got Error {}", e);
log.info("Failed updating storageconfig {}, ", oldSc.getMetadata().getName());
log.error("Failed updating storageconfig {}, ", oldSc.getMetadata().getName(), e);
updateStatus(newSc, false, configUUID, e.getMessage());
return;
}
Expand Down
28 changes: 26 additions & 2 deletions managed/src/main/java/com/yugabyte/yw/common/operator/resources/releaseCrd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,20 @@ spec:
description: S3 access key
type: string
secretAccessKey:
description: S3 secret key
description: S3 secret key. Deprecated, use secretAccessKeySecret
type: string
secretAccessKeySecret:
description: S3 secret key secret. Overrides secretAccessKey
type: object
properties:
name:
description: Name of the secret
type: string
namespace:
description: Namespace of the secret
type: string
required:
- name
paths:
description: S3 paths to download the release from
type: object
Expand Down Expand Up @@ -117,7 +129,19 @@ spec:
description: Optional checksum for Helm chart package
credentialsJson:
type: string
description: GCS service key JSON
description: |
GCS service key JSON. Deprecated, use credentialsJsonSecret instead.
credentialsJsonSecret:
type: object
properties:
name:
description: Name of the secret
type: string
namespace:
description: Namespace of the secret
type: string
required:
- name
http:
type: object
properties:
Expand Down
54 changes: 51 additions & 3 deletions managed/src/main/java/com/yugabyte/yw/common/operator/resources/storageConfigCrd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,9 @@ spec:
description: AWS access key id for the S3 storage configuration.
type: string
AWS_SECRET_ACCESS_KEY:
description: AWS secret access key for the S3 storage configuration.
description: |
AWS secret access key for the S3 storage configuration. Deprecated, use
aws_secret_access_key_secret instead
type: string
USE_IAM:
description: Use IAM for storage account access. Valid for S3/GCS.
Expand All @@ -74,10 +76,56 @@ spec:
- message: BACKUP_LOCATION cannot be changed
rule: self == oldSelf
GCS_CREDENTIALS_JSON:
description: GCS credentials JSON for the GCS storage configuration.
description: |
GCS credentials JSON for the GCS storage configuration. Deprecated, use
gcs_credentials_json_secret instead
type: string
AZURE_STORAGE_SAS_TOKEN:
description: Azure SAS token for the Azure storage configuration.
description: |
Azure SAS token for the Azure storage configuration. Deprecated, use
azure_storage_sas_token_secret instead
type: string
required:
- BACKUP_LOCATION
awsSecretAccessKeySecret:
type: object
description: |
Name of the secret containing AWS_SECRET_ACCESS_KEY for the S3 storage
configuration. The secret will take precedence over data.AWS_SECRET_ACCESS_KEY
properties:
name:
type: string
description: Name of the secret
namespace:
type: string
description: Namespace of the secret
required:
- name
gcsCredentialsJsonSecret:
type: object
description: |
Name of the secret containing GCS_CREDENTIALS_JSON for the GCS storage
configuration. The secret will take precedence over data.GCS_CREDENTIALS_JSON
properties:
name:
type: string
description: Name of the secret
namespace:
type: string
description: Namespace of the secret
required:
- name
azureStorageSasTokenSecret:
type: object
description: |
Name of the secret containing AZURE_STORAGE_SAS_TOKEN for the Azure storage
configuration. The secret will take precedence over data.AZURE_STORAGE_SAS_TOKEN
properties:
name:
type: string
description: Name of the secret
namespace:
type: string
description: Namespace of the secret
required:
- name
35 changes: 35 additions & 0 deletions managed/src/main/java/com/yugabyte/yw/common/operator/utils/OperatorUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,15 @@
import com.yugabyte.yw.models.helpers.DeviceInfo;
import com.yugabyte.yw.models.helpers.TaskType;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.ConfigBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
import io.yugabyte.operator.v1alpha1.Release;
import io.yugabyte.operator.v1alpha1.YBUniverse;
import io.yugabyte.operator.v1alpha1.releasespec.config.DownloadConfig;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -421,4 +423,37 @@ public void deleteReleaseCr(Release release) {
}
log.info("Removed release {}", release.getMetadata().getName());
}

public String getAndParseSecretForKey(String name, @Nullable String namespace, String key) {
Secret secret = getSecret(name, namespace);
if (secret == null) {
log.warn("Secret {} not found", name);
return null;
}
return parseSecretForKey(secret, key);
}

public Secret getSecret(String name, @Nullable String namespace) {
try (final KubernetesClient kubernetesClient =
new KubernetesClientBuilder().withConfig(k8sClientConfig).build()) {
if (StringUtils.isBlank(namespace)) {
log.info("Getting secret '{}' from default namespace", name);
namespace = "default";
}
return kubernetesClient.secrets().inNamespace(namespace).withName(name).get();
}
}

// parseSecretForKey checks secret data for the key. If not found, it will then check stringData.
// Returns null if the key is not found at all.
// Also handles null secret.
public String parseSecretForKey(Secret secret, String key) {
if (secret == null) {
return null;
}
if (secret.getData().get(key) != null) {
return new String(Base64.getDecoder().decode(secret.getData().get(key)));
}
return secret.getStringData().get(key);
}
}

0 comments on commit d8b0b4f

Please sign in to comment.