Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(artifacts): Adds ArtifactStore logic to clouddriver #5976

Merged
merged 1 commit into from
Jul 6, 2023
Merged

feat(artifacts): Adds ArtifactStore logic to clouddriver #5976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .../spinnaker/clouddriver/appengine/deploy/ops/DeployAppengineConfigAtomicOperationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.netflix.spinnaker.clouddriver.appengine.deploy.description.DeployAppengineConfigDescription;
import com.netflix.spinnaker.clouddriver.artifacts.ArtifactDownloader;
import com.netflix.spinnaker.kork.artifacts.ArtifactTypes;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import java.io.ByteArrayInputStream;
import java.io.File;
Expand Down Expand Up @@ -73,7 +74,7 @@ public void shouldDownloadFiletoDirectory() throws IOException {
artifactMap.put("artifactAccount", "embedded-artifact");
artifactMap.put("id", "123abc");
artifactMap.put("reference", "ZG9zb21ldGhpbmc=");
artifactMap.put("type", "embedded/base64");
artifactMap.put("type", ArtifactTypes.EMBEDDED_BASE64.getMimeType());
Artifact artifact = mapper.convertValue(artifactMap, Artifact.class);

Path path = null;
Expand Down
13 changes: 10 additions & 3 deletions ...ava/com/netflix/spinnaker/clouddriver/artifacts/embedded/EmbeddedArtifactCredentials.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.common.collect.ImmutableList;
import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
import com.netflix.spinnaker.kork.annotations.NonnullByDefault;
import com.netflix.spinnaker.kork.artifacts.ArtifactTypes;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
Expand All @@ -34,7 +35,11 @@
final class EmbeddedArtifactCredentials implements ArtifactCredentials {
public static final String CREDENTIALS_TYPE = "artifacts-embedded";
@Getter private final String name;
@Getter private final ImmutableList<String> types = ImmutableList.of("embedded/base64");

@Getter
private final ImmutableList<String> types =
ImmutableList.of(
ArtifactTypes.EMBEDDED_BASE64.getMimeType(), ArtifactTypes.REMOTE_BASE64.getMimeType());

@JsonIgnore private final Base64.Decoder base64Decoder;

Expand All @@ -45,7 +50,9 @@ final class EmbeddedArtifactCredentials implements ArtifactCredentials {

public InputStream download(Artifact artifact) {
String type = artifact.getType();
if (type.equals("embedded/base64")) {
if (ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(type)) {
return fromBase64(artifact);
} else if (ArtifactTypes.REMOTE_BASE64.getMimeType().equals(type)) {
return fromBase64(artifact);
} else {
throw new NotImplementedException("Embedded type '" + type + "' is not handled.");
Copy link
Contributor

@dbyron-sf dbyron-sf Jul 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Maybe it never was, but this isn't necessarily an embedded type, is it?

Expand All @@ -59,7 +66,7 @@ private InputStream fromBase64(Artifact artifact) {

@Override
public boolean handlesType(String type) {
return type.startsWith("embedded/");
return type.startsWith("embedded/") || type.startsWith("remote/");
}

@Override
Expand Down
14 changes: 13 additions & 1 deletion ...driver-core/src/main/java/com/netflix/spinnaker/clouddriver/config/CloudDriverConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@
import com.netflix.spinnaker.credentials.poller.PollerConfiguration;
import com.netflix.spinnaker.credentials.poller.PollerConfigurationProperties;
import com.netflix.spinnaker.fiat.shared.FiatPermissionEvaluator;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDeserializer;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration;
import com.netflix.spinnaker.kork.core.RetrySupport;
import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService;
import com.netflix.spinnaker.kork.jackson.ObjectMapperSubtypeConfigurer;
Expand All @@ -108,6 +111,7 @@
import java.util.List;
import java.util.Optional;
import javax.inject.Provider;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
Expand All @@ -128,7 +132,8 @@
RedisConfig.class,
CacheConfig.class,
SearchExecutorConfig.class,
PluginsAutoConfiguration.class
PluginsAutoConfiguration.class,
ArtifactStoreConfiguration.class,
})
@PropertySource(
value = "classpath:META-INF/clouddriver-core.properties",
Expand Down Expand Up @@ -418,4 +423,11 @@ ThreadPoolTaskScheduler threadPoolTaskScheduler(
threadPoolTaskScheduler.setThreadNamePrefix("ThreadPoolTaskScheduler");
return threadPoolTaskScheduler;
}

@Bean
@ConditionalOnExpression("${artifact-store.enabled:false}")
ArtifactDeserializer artifactDeserializer(
ArtifactStore storage, @Qualifier("artifactObjectMapper") ObjectMapper objectMapper) {
return new ArtifactDeserializer(objectMapper, storage);
}
}
24 changes: 24 additions & 0 deletions clouddriver-web/src/main/groovy/com/netflix/spinnaker/clouddriver/Main.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,19 @@
package com.netflix.spinnaker.clouddriver

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.netflix.spinnaker.clouddriver.security.config.SecurityConfig
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDeserializer
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreConfiguration
import com.netflix.spinnaker.kork.artifacts.model.Artifact
import com.netflix.spinnaker.kork.boot.DefaultPropertiesBuilder
import com.netflix.spinnaker.kork.configserver.ConfigServerBootstrap
import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthContributorAutoConfiguration
import org.springframework.boot.autoconfigure.EnableAutoConfiguration
import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration
import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration
import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
Expand All @@ -44,6 +50,7 @@ import java.security.Security
@Import([
WebConfig,
SecurityConfig,
ArtifactStoreConfiguration,
])
@ComponentScan([
'com.netflix.spinnaker.config',
Expand Down Expand Up @@ -81,6 +88,16 @@ class Main extends SpringBootServletInitializer {

@Bean
@Primary
@ConditionalOnBean(value = ArtifactDeserializer.class)
ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) {
return builder.createXmlMapper(false)
.mixIn(Artifact.class, ArtifactMixin.class)
.build()
}

@Bean
@Primary
@ConditionalOnMissingBean(value = ArtifactDeserializer.class)
ObjectMapper jacksonObjectMapper(Jackson2ObjectMapperBuilder builder) {
return builder.createXmlMapper(false).build()
}
Expand All @@ -91,5 +108,12 @@ class Main extends SpringBootServletInitializer {
.properties(DEFAULT_PROPS)
.sources(Main)
}

/**
* Used to deserialize artifacts utilizing an artifact store if it is enabled,
* and thus bypassing the default deserializer on the artifact object itself.
*/
@JsonDeserialize(using = ArtifactDeserializer.class)
private static interface ArtifactMixin{}
}

28 changes: 26 additions & 2 deletions ...r-web/src/main/java/com/netflix/spinnaker/clouddriver/controllers/ArtifactController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import com.netflix.spinnaker.clouddriver.artifacts.ArtifactCredentialsRepository;
import com.netflix.spinnaker.clouddriver.artifacts.ArtifactDownloader;
import com.netflix.spinnaker.clouddriver.artifacts.config.ArtifactCredentials;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore;
import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreURIBuilder;
import com.netflix.spinnaker.kork.artifacts.model.Artifact;
import com.netflix.spinnaker.kork.exceptions.MissingCredentialsException;
import java.io.InputStream;
Expand All @@ -30,7 +32,14 @@
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;

@Slf4j
Expand All @@ -39,13 +48,19 @@
public class ArtifactController {
private ArtifactCredentialsRepository artifactCredentialsRepository;
private ArtifactDownloader artifactDownloader;
private final ArtifactStore storage;
private final ArtifactStoreURIBuilder artifactStoreURIBuilder;

@Autowired
public ArtifactController(
Optional<ArtifactCredentialsRepository> artifactCredentialsRepository,
Optional<ArtifactDownloader> artifactDownloader) {
Optional<ArtifactDownloader> artifactDownloader,
Optional<ArtifactStore> storage,
Optional<ArtifactStoreURIBuilder> artifactStoreURIBuilder) {
this.artifactCredentialsRepository = artifactCredentialsRepository.orElse(null);
this.artifactDownloader = artifactDownloader.orElse(null);
this.storage = storage.orElse(null);
this.artifactStoreURIBuilder = artifactStoreURIBuilder.orElse(null);
}

@RequestMapping(method = RequestMethod.GET, value = "/credentials")
Expand All @@ -72,6 +87,15 @@ StreamingResponseBody fetch(@RequestBody Artifact artifact) {
};
}

@RequestMapping(method = RequestMethod.GET, value = "/content-address/{application}/{hash}")
Artifact.StoredView getStoredArtifact(
@PathVariable(value = "application") String application,
@PathVariable(value = "hash") String hash) {
Artifact artifact = storage.get(artifactStoreURIBuilder.buildRawURI(application, hash));
Artifact.StoredView view = new Artifact.StoredView(artifact.getReference());
return view;
}

@RequestMapping(method = RequestMethod.GET, value = "/account/{accountName}/names")
List<String> getNames(
@PathVariable("accountName") String accountName,
Expand Down