From eb717db3ae5ebd502fbf58eced3fb2f72d8e83ee Mon Sep 17 00:00:00 2001 From: Benevolent Benjamin Powell Date: Thu, 2 Feb 2023 20:41:28 -0600 Subject: [PATCH] feat(artifacts): Adds ArtifactStore logic to clouddriver This commit adds artifact storage to clouddriver. The idea in this commit is artifact will automatically be expanded when the server receives a request since clouddriver is more than likely interested in the artifact. Signed-off-by: benjamin-j-powell --- ...loyAppengineConfigAtomicOperationTest.java | 3 +- .../embedded/EmbeddedArtifactCredentials.java | 13 +++++++-- .../clouddriver/config/CloudDriverConfig.java | 14 +++++++++- .../netflix/spinnaker/clouddriver/Main.groovy | 24 ++++++++++++++++ .../controllers/ArtifactController.java | 28 +++++++++++++++++-- 5 files changed, 75 insertions(+), 7 deletions(-) diff --git a/clouddriver-appengine/src/test/java/com/netflix/spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java b/clouddriver-appengine/src/test/java/com/netflix/spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java index 205b26420c0..346268d42d2 100644 --- a/clouddriver-appengine/src/test/java/com/netflix/spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java +++ b/clouddriver-appengine/src/test/java/com/netflix/spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spinnaker.clouddriver.appengine.deploy.description.DeployAppengineConfigDescription; import com.netflix.spinnaker.clouddriver.artifacts.ArtifactDownloader; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import java.io.ByteArrayInputStream; import java.io.File; @@ -73,7 +74,7 @@ public void shouldDownloadFiletoDirectory() throws IOException { artifactMap.put("artifactAccount", "embedded-artifact"); artifactMap.put("id", "123abc"); artifactMap.put("reference", "ZG9zb21ldGhpbmc="); - artifactMap.put("type", "embedded/base64"); + artifactMap.put("type", ArtifactTypes.EMBEDDED_BASE64.getMimeType()); Artifact artifact = mapper.convertValue(artifactMap, Artifact.class); Path path = null; diff --git a/clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java b/clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java index 20773c9d82e..1ddd38ee774 100644 --- a/clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java +++ b/clouddriver-artifacts/src/main/java/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials; import com.netflix.spinnaker.kork.annotations.NonnullByDefault; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -34,7 +35,11 @@ final class EmbeddedArtifactCredentials implements ArtifactCredentials { public static final String CREDENTIALS_TYPE = "artifacts-embedded"; @Getter private final String name; - @Getter private final ImmutableList types = ImmutableList.of("embedded/base64"); + + @Getter + private final ImmutableList types = + ImmutableList.of( + ArtifactTypes.EMBEDDED_BASE64.getMimeType(), ArtifactTypes.REMOTE_BASE64.getMimeType()); @JsonIgnore private final Base64.Decoder base64Decoder; @@ -45,7 +50,9 @@ final class EmbeddedArtifactCredentials implements ArtifactCredentials { public InputStream download(Artifact artifact) { String type = artifact.getType(); - if (type.equals("embedded/base64")) { + if (ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(type)) { + return fromBase64(artifact); + } else if (ArtifactTypes.REMOTE_BASE64.getMimeType().equals(type)) { return fromBase64(artifact); } else { throw new NotImplementedException("Embedded type '" + type + "' is not handled."); @@ -59,7 +66,7 @@ private InputStream fromBase64(Artifact artifact) { @Override public boolean handlesType(String type) { - return type.startsWith("embedded/"); + return type.startsWith("embedded/") || type.startsWith("remote/"); } @Override diff --git a/clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java b/clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java index fa9cf131279..b943c8d2135 100644 --- a/clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java +++ b/clouddriver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java @@ -99,6 +99,9 @@ import com.netflix.spinnaker.credentials.poller.PollerConfiguration; import com.netflix.spinnaker.credentials.poller.PollerConfigurationProperties; import com.netflix.spinnaker.fiat.shared.FiatPermissionEvaluator; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDeserializer; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration; import com.netflix.spinnaker.kork.core.RetrySupport; import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService; import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer; @@ -108,6 +111,7 @@ import java.util.List; import java.util.Optional; import javax.inject.Provider; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -128,7 +132,8 @@ RedisConfig.class, CacheConfig.class, SearchExecutorConfig.class, - PluginsAutoConfiguration.class + PluginsAutoConfiguration.class, + ArtifactStoreConfiguration.class, }) @PropertySource( value = "classpath:META-INF/clouddriver-core.properties", @@ -418,4 +423,11 @@ ThreadPoolTaskScheduler threadPoolTaskScheduler( threadPoolTaskScheduler.setThreadNamePrefix("ThreadPoolTaskScheduler"); return threadPoolTaskScheduler; } + + @Bean + @ConditionalOnExpression("${artifact-store.enabled:false}") + ArtifactDeserializer artifactDeserializer( + ArtifactStore storage, @Qualifier("artifactObjectMapper") ObjectMapper objectMapper) { + return new ArtifactDeserializer(objectMapper, storage); + } } diff --git a/clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy b/clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy index 5350b2f7e62..c26744e4c40 100644 --- a/clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy +++ b/clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy @@ -17,13 +17,19 @@ package com.netflix.spinnaker.clouddriver import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.netflix.spinnaker.clouddriver.security.config.SecurityConfig +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDeserializer +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration +import com.netflix.spinnaker.kork.artifacts.model.Artifact import com.netflix.spinnaker.kork.boot.DefaultPropertiesBuilder import com.netflix.spinnaker.kork.configserver.ConfigServerBootstrap import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthContributorAutoConfiguration import org.springframework.boot.autoconfigure.EnableAutoConfiguration import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration @@ -44,6 +50,7 @@ import java.security.Security @Import([ WebConfig, SecurityConfig, + ArtifactStoreConfiguration, ]) @ComponentScan([ 'com.netflix.spinnaker.config', @@ -81,6 +88,16 @@ class Main extends SpringBootServletInitializer { @Bean @Primary + @ConditionalOnBean(value = ArtifactDeserializer.class) + ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) { + return builder.createXmlMapper(false) + .mixIn(Artifact.class, ArtifactMixin.class) + .build() + } + + @Bean + @Primary + @ConditionalOnMissingBean(value = ArtifactDeserializer.class) ObjectMapper jacksonObjectMapper(Jackson2ObjectMapperBuilder builder) { return builder.createXmlMapper(false).build() } @@ -91,5 +108,12 @@ class Main extends SpringBootServletInitializer { .properties(DEFAULT_PROPS) .sources(Main) } + + /** + * Used to deserialize artifacts utilizing an artifact store if it is enabled, + * and thus bypassing the default deserializer on the artifact object itself. + */ + @JsonDeserialize(using = ArtifactDeserializer.class) + private static interface ArtifactMixin{} } diff --git a/clouddriver-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java b/clouddriver-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java index 100b991d589..5d0c3cbd999 100644 --- a/clouddriver-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java +++ b/clouddriver-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java @@ -20,6 +20,8 @@ import com.netflix.spinnaker.clouddriver.artifacts.ArtifactCredentialsRepository; import com.netflix.spinnaker.clouddriver.artifacts.ArtifactDownloader; import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreURIBuilder; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.kork.exceptions.MissingCredentialsException; import java.io.InputStream; @@ -30,7 +32,14 @@ import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; @Slf4j @@ -39,13 +48,19 @@ public class ArtifactController { private ArtifactCredentialsRepository artifactCredentialsRepository; private ArtifactDownloader artifactDownloader; + private final ArtifactStore storage; + private final ArtifactStoreURIBuilder artifactStoreURIBuilder; @Autowired public ArtifactController( Optional artifactCredentialsRepository, - Optional artifactDownloader) { + Optional artifactDownloader, + Optional storage, + Optional artifactStoreURIBuilder) { this.artifactCredentialsRepository = artifactCredentialsRepository.orElse(null); this.artifactDownloader = artifactDownloader.orElse(null); + this.storage = storage.orElse(null); + this.artifactStoreURIBuilder = artifactStoreURIBuilder.orElse(null); } @RequestMapping(method = RequestMethod.GET, value = "/credentials") @@ -72,6 +87,15 @@ StreamingResponseBody fetch(@RequestBody Artifact artifact) { }; } + @RequestMapping(method = RequestMethod.GET, value = "/content-address/{application}/{hash}") + Artifact.StoredView getStoredArtifact( + @PathVariable(value = "application") String application, + @PathVariable(value = "hash") String hash) { + Artifact artifact = storage.get(artifactStoreURIBuilder.buildRawURI(application, hash)); + Artifact.StoredView view = new Artifact.StoredView(artifact.getReference()); + return view; + } + @RequestMapping(method = RequestMethod.GET, value = "/account/{accountName}/names") List getNames( @PathVariable("accountName") String accountName,