feat(artifacts): Adds ArtifactStore logic to clouddriver

This commit adds artifact storage to clouddriver. The idea in this commit is artifact will automatically be expanded when the server receives a request since clouddriver is more than likely interested in the artifact. Signed-off-by: benjamin-j-powell <[email protected]>
spinnaker · Jul 6, 2023 · eb717db · eb717db
1 parent af887e9
commit eb717db
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 7 deletions.
diff --git a/.../spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java b/.../spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java
@@ -25,6 +25,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.spinnaker.clouddriver.appengine.deploy.description.DeployAppengineConfigDescription;
 import com.netflix.spinnaker.clouddriver.artifacts.ArtifactDownloader;
+import com.netflix.spinnaker.kork.artifacts.ArtifactTypes;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -73,7 +74,7 @@ public void shouldDownloadFiletoDirectory() throws IOException {
     artifactMap.put("artifactAccount", "embedded-artifact");
     artifactMap.put("id", "123abc");
     artifactMap.put("reference", "ZG9zb21ldGhpbmc=");
-    artifactMap.put("type", "embedded/base64");
+    artifactMap.put("type", ArtifactTypes.EMBEDDED_BASE64.getMimeType());
     Artifact artifact = mapper.convertValue(artifactMap, Artifact.class);
 
     Path path = null;

diff --git a/...ava/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java b/...ava/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java
@@ -21,6 +21,7 @@
 import com.google.common.collect.ImmutableList;
 import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
 import com.netflix.spinnaker.kork.annotations.NonnullByDefault;
+import com.netflix.spinnaker.kork.artifacts.ArtifactTypes;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
@@ -34,7 +35,11 @@
 final class EmbeddedArtifactCredentials implements ArtifactCredentials {
   public static final String CREDENTIALS_TYPE = "artifacts-embedded";
   @Getter private final String name;
-  @Getter private final ImmutableList<String> types = ImmutableList.of("embedded/base64");
+
+  @Getter
+  private final ImmutableList<String> types =
+      ImmutableList.of(
+          ArtifactTypes.EMBEDDED_BASE64.getMimeType(), ArtifactTypes.REMOTE_BASE64.getMimeType());
 
   @JsonIgnore private final Base64.Decoder base64Decoder;
 
@@ -45,7 +50,9 @@ final class EmbeddedArtifactCredentials implements ArtifactCredentials {
 
   public InputStream download(Artifact artifact) {
     String type = artifact.getType();
-    if (type.equals("embedded/base64")) {
+    if (ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(type)) {
+      return fromBase64(artifact);
+    } else if (ArtifactTypes.REMOTE_BASE64.getMimeType().equals(type)) {
       return fromBase64(artifact);
     } else {
       throw new NotImplementedException("Embedded type '" + type + "' is not handled.");
@@ -59,7 +66,7 @@ private InputStream fromBase64(Artifact artifact) {
 
   @Override
   public boolean handlesType(String type) {
-    return type.startsWith("embedded/");
+    return type.startsWith("embedded/") || type.startsWith("remote/");
   }
 
   @Override

diff --git a/...driver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java b/...driver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java
@@ -99,6 +99,9 @@
 import com.netflix.spinnaker.credentials.poller.PollerConfiguration;
 import com.netflix.spinnaker.credentials.poller.PollerConfigurationProperties;
 import com.netflix.spinnaker.fiat.shared.FiatPermissionEvaluator;
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDeserializer;
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration;
 import com.netflix.spinnaker.kork.core.RetrySupport;
 import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService;
 import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer;
@@ -108,6 +111,7 @@
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Provider;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -128,7 +132,8 @@
   RedisConfig.class,
   CacheConfig.class,
   SearchExecutorConfig.class,
-  PluginsAutoConfiguration.class
+  PluginsAutoConfiguration.class,
+  ArtifactStoreConfiguration.class,
 })
 @PropertySource(
     value = "classpath:META-INF/clouddriver-core.properties",
@@ -418,4 +423,11 @@ ThreadPoolTaskScheduler threadPoolTaskScheduler(
     threadPoolTaskScheduler.setThreadNamePrefix("ThreadPoolTaskScheduler");
     return threadPoolTaskScheduler;
   }
+
+  @Bean
+  @ConditionalOnExpression("${artifact-store.enabled:false}")
+  ArtifactDeserializer artifactDeserializer(
+      ArtifactStore storage, @Qualifier("artifactObjectMapper") ObjectMapper objectMapper) {
+    return new ArtifactDeserializer(objectMapper, storage);
+  }
 }
diff --git a/clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy b/clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy
@@ -17,13 +17,19 @@
 package com.netflix.spinnaker.clouddriver
 
 import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import com.netflix.spinnaker.clouddriver.security.config.SecurityConfig
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDeserializer
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration
+import com.netflix.spinnaker.kork.artifacts.model.Artifact
 import com.netflix.spinnaker.kork.boot.DefaultPropertiesBuilder
 import com.netflix.spinnaker.kork.configserver.ConfigServerBootstrap
 import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthContributorAutoConfiguration
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration
 import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration
 import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
 import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration
 import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
@@ -44,6 +50,7 @@ import java.security.Security
 @Import([
   WebConfig,
   SecurityConfig,
+  ArtifactStoreConfiguration,
 ])
 @ComponentScan([
   'com.netflix.spinnaker.config',
@@ -81,6 +88,16 @@ class Main extends SpringBootServletInitializer {
 
   @Bean
   @Primary
+  @ConditionalOnBean(value = ArtifactDeserializer.class)
+  ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) {
+    return builder.createXmlMapper(false)
+      .mixIn(Artifact.class, ArtifactMixin.class)
+      .build()
+  }
+
+  @Bean
+  @Primary
+  @ConditionalOnMissingBean(value = ArtifactDeserializer.class)
   ObjectMapper jacksonObjectMapper(Jackson2ObjectMapperBuilder builder) {
     return builder.createXmlMapper(false).build()
   }
@@ -91,5 +108,12 @@ class Main extends SpringBootServletInitializer {
       .properties(DEFAULT_PROPS)
       .sources(Main)
   }
+
+  /**
+   * Used to deserialize artifacts utilizing an artifact store if it is enabled,
+   * and thus bypassing the default deserializer on the artifact object itself.
+   */
+  @JsonDeserialize(using = ArtifactDeserializer.class)
+  private static interface ArtifactMixin{}
 }
 
diff --git a/...r-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java b/...r-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java
@@ -20,6 +20,8 @@
 import com.netflix.spinnaker.clouddriver.artifacts.ArtifactCredentialsRepository;
 import com.netflix.spinnaker.clouddriver.artifacts.ArtifactDownloader;
 import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
+import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreURIBuilder;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import com.netflix.spinnaker.kork.exceptions.MissingCredentialsException;
 import java.io.InputStream;
@@ -30,7 +32,14 @@
 import org.apache.commons.io.IOUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
 
 @Slf4j
@@ -39,13 +48,19 @@
 public class ArtifactController {
   private ArtifactCredentialsRepository artifactCredentialsRepository;
   private ArtifactDownloader artifactDownloader;
+  private final ArtifactStore storage;
+  private final ArtifactStoreURIBuilder artifactStoreURIBuilder;
 
   @Autowired
   public ArtifactController(
       Optional<ArtifactCredentialsRepository> artifactCredentialsRepository,
-      Optional<ArtifactDownloader> artifactDownloader) {
+      Optional<ArtifactDownloader> artifactDownloader,
+      Optional<ArtifactStore> storage,
+      Optional<ArtifactStoreURIBuilder> artifactStoreURIBuilder) {
     this.artifactCredentialsRepository = artifactCredentialsRepository.orElse(null);
     this.artifactDownloader = artifactDownloader.orElse(null);
+    this.storage = storage.orElse(null);
+    this.artifactStoreURIBuilder = artifactStoreURIBuilder.orElse(null);
   }
 
   @RequestMapping(method = RequestMethod.GET, value = "/credentials")
@@ -72,6 +87,15 @@ StreamingResponseBody fetch(@RequestBody Artifact artifact) {
     };
   }
 
+  @RequestMapping(method = RequestMethod.GET, value = "/content-address/{application}/{hash}")
+  Artifact.StoredView getStoredArtifact(
+      @PathVariable(value = "application") String application,
+      @PathVariable(value = "hash") String hash) {
+    Artifact artifact = storage.get(artifactStoreURIBuilder.buildRawURI(application, hash));
+    Artifact.StoredView view = new Artifact.StoredView(artifact.getReference());
+    return view;
+  }
+
   @RequestMapping(method = RequestMethod.GET, value = "/account/{accountName}/names")
   List<String> getNames(
       @PathVariable("accountName") String accountName,