feat(artifacts): Add ArtifactStore to Kork (#1069)

* feat(artifacts): Add ArtifactStore to Kork

This commit adds a new feature called ArtifactStore. This allows for
storage of embedded/base64 types to some storage, eg S3. So instead of
having duplication of some artifact in a context, it'll now be
referenced by a artifact URI.

Also, there's a nice README that explains various settings as well as
how to setup a local S3 for testing artifact storage

[README link](kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md )

Signed-off-by: benjamin-j-powell <[email protected]>

* Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: Nemesis Osorio <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: Nemesis Osorio <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java

Co-authored-by: Nemesis Osorio <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: David Byron <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: David Byron <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: David Byron <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java

Co-authored-by: Sergio Quintero <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: Sergio Quintero <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: Sergio Quintero <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: Sergio Quintero <[email protected]>

* Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java

Co-authored-by: David Byron <[email protected]>

* Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: David Byron <[email protected]>

Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

Co-authored-by: David Byron <[email protected]>

* fixup! feat(artifacts): Add ArtifactStore to Kork

Signed-off-by: benjamin-j-powell <[email protected]>

---------

Signed-off-by: benjamin-j-powell <[email protected]>
Co-authored-by: Benevolent Benjamin Powell <[email protected]>
Co-authored-by: Sergio Quintero <[email protected]>
Co-authored-by: David Byron <[email protected]>

kork-artifacts/kork-artifacts.gradle

@@ -5,11 +5,21 @@ dependencies {
   api(platform(project(":spinnaker-dependencies")))
 
   implementation project(":kork-annotations")
+  implementation project(":kork-exceptions")
+  implementation project(":kork-security")
   implementation "com.fasterxml.jackson.core:jackson-databind"
+  api 'software.amazon.awssdk:s3'
+  api 'software.amazon.awssdk:sts'
+  implementation "org.apache.httpcomponents:httpclient"
+  implementation "org.springframework.boot:spring-boot-autoconfigure"
+  implementation "org.springframework.security:spring-security-core"
+  implementation 'org.apache.logging.log4j:log4j-api'
+  implementation 'org.apache.logging.log4j:log4j-core'
   api "com.hubspot.jinjava:jinjava"
 
   testImplementation "org.assertj:assertj-core"
   testImplementation "org.junit.jupiter:junit-jupiter-api"
   testImplementation "org.junit.jupiter:junit-jupiter-params"
+  testImplementation "org.mockito:mockito-core"
   testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine"
 }

kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.kork.artifacts;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public enum ArtifactTypes {
+  EMBEDDED_BASE64("embedded/base64"),
+  REMOTE_BASE64("remote/base64"),
+  ;
+
+  @Getter private final String mimeType;
+}

kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md

@@ -0,0 +1,129 @@
+# Artifact Storage
+
+Artifact Storage is a feature that allows for embedded artifacts to be
+persisted to some storage, eg the S3ArtifactStore. Spinnaker keeps a history,
+which is called the pipeline context, that contains everything that an
+execution had done. Pipeline contexts can be very large, especially with
+complicated pipelines, and size can be further increased when users have large
+artifacts. Spinnaker will duplicate
+these artifacts whenever any stage uses any of those artifacts. Using an
+artifact store reduces this overhead by providing a reference link of,
+`ref://<spinnaker-application>/<content-hash>`. This reduces the context size
+tremendously, but will vary depending on the size of the pipeline, as well as
+how that artifact is used, but we've seen improvements of 80% for some
+pipelines.
+
+## Architecture
+
+                                 +-----------+
+                                 |           |
+                                 |   Orca    |
+                                 |           |
+                                 +-----------+
+                                   |        ^
+                                   |        | (outgoing artifact compressed)
+                                   |        +----------------------+
+                                   | (bake request)                |
+                                   +---------------------+         |
+                                                         v         |
+        +---------------+            (fetch)           +-------------+
+        |               |<-----------------------------|             |
+        |  Clouddriver  |                              |  Rosco      |
+        |               |                              |             |
+        |  s3 get       |   (full artifact returned)   |  s3 stores  |
+        |  artifacts    |----------------------------->|  artifacts  |
+        +---------------+                              +-------------+
+
+
+Artifact storage is divided into two operations of get and store, and there are
+primarily two services that utilize each of these operations. Further the
+artifact storage system relies on Spring's (de)serializing to call these
+operations to limit the amount of code changes needed within these services.
+
+When bootstrapping Spring we add in custom bean serializers and deserializers to
+handle storage or retrieval of an artifact.
+
+Rosco is primarily used for baking artifacts which will generate something
+deployable.  When Rosco responds to a bake request, the custom serializer
+injected in Rosco at startup stores the artifact and returns a `remote/base64`
+artifact instead of the usual `embedded/base64`.
+
+Clouddriver, for this document, handles mostly deployment and has some endpoints
+regarding artifacts. It does do a little more than this, but we only care about
+these two particular operations. When any request comes in, Spring will use its
+custom deserializers to expand any artifact in its payload since any request
+with artifacts, probably wants to do some operation on those artifacts. Further
+Clouddriver also allows for fetching of artifacts. Orca and Rosco both make
+calls to the `/artifact/fetch` endpoint. Where Rosco uses it to fetch an
+artifact to be baked, and Orca uses it primarily when dealing with deploy
+manifests. When a request is sent to the fetch endpoint, Clouddriver will always
+return the full `embedded/base64` artifact back to the service. It is up to the
+service receiving the artifact to compress it. Luckily, for Orca, we don't have
+to worry about compression, since this no longer becomes an artifact, but a
+manifest instead.
+
+Orca is a special case as it mostly does orchestration, but does cause some
+artifacts to be duplicated when handling the expected artifact logic. We inject
+some logic to handle the duplication along with ensuring that matching against
+expected artifacts still works properly. So if a `embedded/base64` type needs to
+match against a `remote/base64` type, Orca will use the artifact store to
+retrieve that artifact, and do the comparison. In addition, Orca will store any
+expected artifacts, to limit the context size.
+
+Orca also handles SpEL evaluation which means our new `remote/base64` type
+should be backwards compatible with existing pipelines. To ensure this, we
+utilized the Spring converters, and injected our own custom converter that will
+check if some `String` is a remote base64 URI, and if it is, retrieve it.
+
+## Configuration
+
+To enable artifact storage, simple add this to your `spinnaker-local.yml` file
+
+```yaml
+artifact-store:
+  enabled: true
+  s3:
+    enabled: true
+    bucket: some-artifact-store-bucket
+```
+
+## Storage Options
+
+### S3
+
+[S3](https://aws.amazon.com/s3/) is an object store provided by AWS. The
+current S3ArtifactStore implementation provides various ways to authenticate
+against AWS.
+
+```yaml
+artifact-store:
+  enabled: true
+  s3:
+    enabled: true
+    profile: dev # if you want to authenticate using a certain profile
+    region: us-west-2 # allows for specified regions
+    bucket: some-artifact-store-bucket
+```
+
+While the implementation is S3 specific, this does not limit usages of other S3
+compatible storage engines. For example, something like
+[SeaweedFS](https://github.com/seaweedfs/seaweedfs) can be used to test locally.
+with
+
+## Local Testing
+
+To test the artifact store locally, we will use SeaweedFS. To start the storage simply run
+`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
+
+Next enable the configuration
+
+```yaml
+artifact-store:
+  enabled: true
+  s3:
+    enabled: true
+    url: http://localhost:8333 # this URL will be used to make S3 API requests to
+    bucket: some-artifact-store-bucket
+```
+
+Start Spinnaker, and you should see reference links in your pipeline contexts.

kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.kork.artifacts.artifactstore;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+
+/** Primarily used to set any custom fields to an artifact */
+public interface ArtifactDecorator {
+  Artifact.ArtifactBuilder decorate(Artifact.ArtifactBuilder builder);
+}

kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2023 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.kork.artifacts.artifactstore;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.netflix.spinnaker.kork.artifacts.ArtifactTypes;
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import java.io.IOException;
+import org.springframework.beans.factory.annotation.Qualifier;
+
+/**
+ * ArtifactDeserializer is a custom deserializer that will allow for artifacts to be fetched from
+ * some artifact store as long as the referenceLink field is set and the reference field is null.
+ */
+public class ArtifactDeserializer extends StdDeserializer<Artifact> {
+  private final ObjectMapper defaultObjectMapper;
+  private final ArtifactStore storage;
+
+  public ArtifactDeserializer(
+      @Qualifier(value = "artifactObjectMapper") ObjectMapper defaultObjectMapper,
+      ArtifactStore storage) {
+    super(Artifact.class);
+    this.defaultObjectMapper = defaultObjectMapper;
+    this.storage = storage;
+  }
+
+  @Override
+  public Artifact deserialize(JsonParser parser, DeserializationContext ctx) throws IOException {
+    Artifact artifact = defaultObjectMapper.readValue(parser, Artifact.class);
+    if (ArtifactTypes.REMOTE_BASE64.getMimeType().equals(artifact.getType())) {
+      return storage.get(artifact.getReference(), new ArtifactMergeReferenceDecorator(artifact));
+    }
+
+    return artifact;
+  }
+
+  /**
+   * ArtifactMergeReferenceDecorator is used to take some artifact and replace its reference with
+   * the reference from another artifact.
+   */
+  public static class ArtifactMergeReferenceDecorator implements ArtifactDecorator {
+    private final Artifact artifactToCopy;
+
+    private ArtifactMergeReferenceDecorator(Artifact artifactToCopy) {
+      this.artifactToCopy = artifactToCopy;
+    }
+
+    @Override
+    public Artifact.ArtifactBuilder decorate(Artifact.ArtifactBuilder builder) {
+      Artifact retrieved = builder.build();
+      return artifactToCopy.toBuilder().reference(retrieved.getReference());
+    }
+  }
+}

kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Apple Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.spinnaker.kork.artifacts.artifactstore;
+
+import com.netflix.spinnaker.kork.artifacts.model.Artifact;
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.Getter;
+
+/**
+ * ArtifactStore is an interface that allows for different types of artifact storage to be used
+ * during runtime
+ */
+public abstract class ArtifactStore {
+  /** ensures the singleton has only been set once */
+  private static final AtomicBoolean singletonSet = new AtomicBoolean(false);
+
+  @Getter private static ArtifactStore instance = null;
+
+  public abstract Artifact store(Artifact artifact);
+  /**
+   * get is used to return an artifact with some id, while also decorating that artifact with any
+   * necessary fields needed which should be then be returned by the artifact store.
+   */
+  public abstract Artifact get(String id, ArtifactDecorator... decorators);
+
+  public static void setInstance(ArtifactStore storage) {
+    if (!singletonSet.compareAndSet(false, true)) {
+      throw new IllegalStateException("Multiple attempts at setting ArtifactStore's singleton");
+    }
+
+    ArtifactStore.instance = storage;
+  }
+
+  public boolean isArtifactURI(String value) {
+    return value.startsWith(ArtifactStoreURIBuilder.uriScheme + "://");
+  }
+}