storage migration to OCI artifacts

open-component-model · Feb 18, 2025 · b0a18be · b0a18be
1 parent 455055a
commit b0a18be
Show file tree

Hide file tree

Showing 68 changed files with 3,268 additions and 1,784 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -62,135 +62,135 @@ jobs:
             bin/k8s
           key: ${{ steps.cache-k8s-restore.outputs.cache-primary-key }}
 
-  e2e-tests:
-    runs-on: large_runner
-    steps:
-      - name: Self Hosted Runner Post Job Cleanup Action
-        uses: TooMuch4U/[email protected]
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: '${{ github.workspace }}/go.mod'
-          cache: false
-
-      - name: Get Go environment
-        run: |
-          echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
-          echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
-      - name: Set up cache
-        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
-        uses: actions/cache@v4
-        with:
-          path: |
-            ${{ env.go_cache }}
-            ${{ env.go_modcache }}
-            bin/k8s
-          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
-          restore-keys: |
-            ${{ env.cache_name }}-${{ runner.os }}-go-
-        env:
-          cache_name: integration-test
-
-      - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1
-        with:
-          config: test/e2e/config/kind-config.yaml
-
-      - name: Install internal image registry
-        run: |
-          kubectl apply -f test/e2e/config/image-registry.yaml
-          kubectl wait pod -l app=registry --for condition=Ready --timeout 5m
-          kubectl wait pod -l app=protected-registry1 --for condition=Ready --timeout 5m
-          kubectl wait pod -l app=protected-registry2 --for condition=Ready --timeout 5m
-
-      - name: Install external CRDs
-        run: kubectl apply --server-side -k https://github.com/openfluxcd/artifact//config/crd?ref=v0.1.1
-
-      - name: Checkout helm-controller
-        uses: actions/checkout@v4
-        with:
-          repository: openfluxcd/helm-controller
-          path: helm-controller
-
-      # TODO: Create helm-controller image in public repository to omit rebuilds
-      - name: Install helm-controller
-        env:
-          IMG: localhost:31000/helm-controller:latest
-        run: |
-          make -C helm-controller docker-build
-          make -C helm-controller docker-push
-          make -C helm-controller install
-          make -C helm-controller deploy
-          kubectl wait deployment.apps/helm-controller --for condition=Available --namespace helm-system --timeout 5m
-          kubectl logs --tail -1 -l app=helm-controller -n helm-system -f --ignore-errors &> helm-controller.log &
-
-      - name: Checkout kustomize-controller
-        uses: actions/checkout@v4
-        with:
-          repository: openfluxcd/kustomize-controller
-          path: kustomize-controller
-
-      # TODO: Create kustomize-controller image in public repository to omit rebuilds
-      - name: Install kustomize-controller
-        env:
-          IMG: localhost:31000/kustomize-controller:latest
-        run: |
-          make -C kustomize-controller docker-build
-          make -C kustomize-controller docker-push
-          make -C kustomize-controller install
-          make -C kustomize-controller deploy
-          kubectl wait deployment.apps/kustomize-controller --for condition=Available --namespace kustomize-system --timeout 5m
-          kubectl logs --tail -1 -l app=kustomize-controller -n kustomize-system -f --ignore-errors &> kustomize-controller.log &
-
-      # TODO: Replace once the release with the 'skipDigestGeneration' field in the component constructor is available
-      # uses: open-component-model/ocm-setup-action@main
-      # with:
-      #   version: v0.19.0-rc.1
-      - name: Set up cache for ocm (temporarily)
-        uses: actions/cache@v4
-        with:
-          path: |
-            ocm/bin
-          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
-          restore-keys: |
-            ${{ env.cache_name }}-${{ runner.os }}-go-
-        env:
-          cache_name: dummy-cache
-      - name: Checkout OCM (temporarily)
-        uses: actions/checkout@v4
-        with:
-          repository: open-component-model/ocm
-          path: ocm
-      - name: Build OCM (temporarily)
-        run: |
-          make -C ocm bin/ocm
-          echo "${{ github.workspace }}/ocm/bin" >> "$GITHUB_PATH"
-
-      - name: Run e2e test
-        env:
-          RESOURCE_TIMEOUT: 5m
-          HELM_CHART: ghcr.io/stefanprodan/charts/podinfo:6.7.1
-          IMAGE_REFERENCE: ghcr.io/stefanprodan/podinfo:6.7.1
-          CONTROLLER_LOG_PATH: ./ocm-k8s-toolkit-controller.log
-          IMAGE_REGISTRY_URL: http://localhost:31000
-          INTERNAL_IMAGE_REGISTRY_URL: http://registry-internal.default.svc.cluster.local:5000
-          PROTECTED_REGISTRY_URL: http://localhost:31001
-          INTERNAL_PROTECTED_REGISTRY_URL: http://protected-registry1-internal.default.svc.cluster.local:5001
-          PROTECTED_REGISTRY_URL2: http://localhost:31002
-          INTERNAL_PROTECTED_REGISTRY_URL2: http://protected-registry2-internal.default.svc.cluster.local:5002
-        run: make test-e2e
-
-      - name: Publish logs on failure
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: controller-logs
-          # Currently, it is planned that the integration tests runs on every commit on a PR. Therefore, we could
-          # produce a lot of logs. To note clutter the storage, the retention-days are reduced to 1.
-          retention-days: 1
-          path: |
-            helm-controller.log
-            kustomize-controller.log
-            ocm-k8s-toolkit-controller.log
+#  e2e-tests:
+#    runs-on: large_runner
+#    steps:
+#      - name: Self Hosted Runner Post Job Cleanup Action
+#        uses: TooMuch4U/[email protected]
+#      - name: Checkout
+#        uses: actions/checkout@v4
+#      - name: Setup Go
+#        uses: actions/setup-go@v5
+#        with:
+#          go-version-file: '${{ github.workspace }}/go.mod'
+#          cache: false
+#
+#      - name: Get Go environment
+#        run: |
+#          echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
+#          echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
+#      - name: Set up cache
+#        # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
+#        uses: actions/cache@v4
+#        with:
+#          path: |
+#            ${{ env.go_cache }}
+#            ${{ env.go_modcache }}
+#            bin/k8s
+#          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
+#          restore-keys: |
+#            ${{ env.cache_name }}-${{ runner.os }}-go-
+#        env:
+#          cache_name: integration-test
+#
+#      - name: Create k8s Kind Cluster
+#        uses: helm/kind-action@v1
+#        with:
+#          config: test/e2e/config/kind-config.yaml
+#
+#      - name: Install internal image registry
+#        run: |
+#          kubectl apply -f test/e2e/config/image-registry.yaml
+#          kubectl wait pod -l app=registry --for condition=Ready --timeout 5m
+#          kubectl wait pod -l app=protected-registry1 --for condition=Ready --timeout 5m
+#          kubectl wait pod -l app=protected-registry2 --for condition=Ready --timeout 5m
+#
+#      - name: Install external CRDs
+#        run: kubectl apply --server-side -k https://github.com/openfluxcd/artifact//config/crd?ref=v0.1.1
+#
+#      - name: Checkout helm-controller
+#        uses: actions/checkout@v4
+#        with:
+#          repository: openfluxcd/helm-controller
+#          path: helm-controller
+#
+#      # TODO: Create helm-controller image in public repository to omit rebuilds
+#      - name: Install helm-controller
+#        env:
+#          IMG: localhost:31000/helm-controller:latest
+#        run: |
+#          make -C helm-controller docker-build
+#          make -C helm-controller docker-push
+#          make -C helm-controller install
+#          make -C helm-controller deploy
+#          kubectl wait deployment.apps/helm-controller --for condition=Available --namespace helm-system --timeout 5m
+#          kubectl logs --tail -1 -l app=helm-controller -n helm-system -f --ignore-errors &> helm-controller.log &
+#
+#      - name: Checkout kustomize-controller
+#        uses: actions/checkout@v4
+#        with:
+#          repository: openfluxcd/kustomize-controller
+#          path: kustomize-controller
+#
+#      # TODO: Create kustomize-controller image in public repository to omit rebuilds
+#      - name: Install kustomize-controller
+#        env:
+#          IMG: localhost:31000/kustomize-controller:latest
+#        run: |
+#          make -C kustomize-controller docker-build
+#          make -C kustomize-controller docker-push
+#          make -C kustomize-controller install
+#          make -C kustomize-controller deploy
+#          kubectl wait deployment.apps/kustomize-controller --for condition=Available --namespace kustomize-system --timeout 5m
+#          kubectl logs --tail -1 -l app=kustomize-controller -n kustomize-system -f --ignore-errors &> kustomize-controller.log &
+#
+#      # TODO: Replace once the release with the 'skipDigestGeneration' field in the component constructor is available
+#      # uses: open-component-model/ocm-setup-action@main
+#      # with:
+#      #   version: v0.19.0-rc.1
+#      - name: Set up cache for ocm (temporarily)
+#        uses: actions/cache@v4
+#        with:
+#          path: |
+#            ocm/bin
+#          key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
+#          restore-keys: |
+#            ${{ env.cache_name }}-${{ runner.os }}-go-
+#        env:
+#          cache_name: dummy-cache
+#      - name: Checkout OCM (temporarily)
+#        uses: actions/checkout@v4
+#        with:
+#          repository: open-component-model/ocm
+#          path: ocm
+#      - name: Build OCM (temporarily)
+#        run: |
+#          make -C ocm bin/ocm
+#          echo "${{ github.workspace }}/ocm/bin" >> "$GITHUB_PATH"
+#
+#      - name: Run e2e test
+#        env:
+#          RESOURCE_TIMEOUT: 5m
+#          HELM_CHART: ghcr.io/stefanprodan/charts/podinfo:6.7.1
+#          IMAGE_REFERENCE: ghcr.io/stefanprodan/podinfo:6.7.1
+#          CONTROLLER_LOG_PATH: ./ocm-k8s-toolkit-controller.log
+#          IMAGE_REGISTRY_URL: http://localhost:31000
+#          INTERNAL_IMAGE_REGISTRY_URL: http://registry-internal.default.svc.cluster.local:5000
+#          PROTECTED_REGISTRY_URL: http://localhost:31001
+#          INTERNAL_PROTECTED_REGISTRY_URL: http://protected-registry1-internal.default.svc.cluster.local:5001
+#          PROTECTED_REGISTRY_URL2: http://localhost:31002
+#          INTERNAL_PROTECTED_REGISTRY_URL2: http://protected-registry2-internal.default.svc.cluster.local:5002
+#        run: make test-e2e
+#
+#      - name: Publish logs on failure
+#        if: failure()
+#        uses: actions/upload-artifact@v4
+#        with:
+#          name: controller-logs
+#          # Currently, it is planned that the integration tests runs on every commit on a PR. Therefore, we could
+#          # produce a lot of logs. To note clutter the storage, the retention-days are reduced to 1.
+#          retention-days: 1
+#          path: |
+#            helm-controller.log
+#            kustomize-controller.log
+#            ocm-k8s-toolkit-controller.log
diff --git a/Makefile b/Makefile
@@ -10,6 +10,10 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
+OS ?= $(shell go env GOOS)
+ARCH ?= $(shell go env GOARCH)
+
+
 # CONTAINER_TOOL defines the container tool to be used for building images.
 # Be aware that the target commands are only tested with Docker which is
 # scaffolded by default. However, you might want to replace it to use other
@@ -64,7 +68,7 @@ vet: ## Run go vet against code.
 	go vet ./...
 
 .PHONY: test
-test: manifests generate envtest ## Run tests.
+test: manifests generate envtest zot-registry ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
@@ -178,6 +182,7 @@ KUSTOMIZE_VERSION ?= v5.4.1
 CONTROLLER_TOOLS_VERSION ?= v0.16.0
 ENVTEST_VERSION ?= release-0.18
 GOLANGCI_LINT_VERSION ?= v1.61.0
+ZOT_VERSION ?= v2.1.2
 
 ## ZOT OCI Registry
 ZOT_VERSION ?= v2.1.2
@@ -213,6 +218,12 @@ deploy-cert-manager: ## Deploy cert-manager to the K8s cluster specified in ~/.k
 undeploy-cert-manager: ## Undeploy cert-manager from the K8s cluster specified in ~/.kube/config.
 	$(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f $(CERT-MANAGER_YAML)
 
+.PHONY: zot-registry
+zot-registry: $(LOCALBIN) # Download zot registry binary locally if necessary.
+	wget "https://github.com/project-zot/zot/releases/download/$(ZOT_VERSION)/zot-$(OS)-$(ARCH)-minimal" \
+		-O $(LOCALBIN)/zot-registry \
+		&& chmod u+x $(LOCALBIN)/zot-registry
+
 .PHONY: golangci-lint
 golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)

diff --git a/PROJECT b/PROJECT
@@ -71,4 +71,13 @@ resources:
   kind: Replication
   path: github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1
   version: v1alpha1
-version: "3"
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: ocm.software
+  group: delivery
+  kind: Snapshot
+  path: github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1
+  version: v1alpha1
+version: "3"
diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go
@@ -113,3 +113,15 @@ type ResourceInfo struct {
 	// +required
 	Digest string `json:"digest,omitempty"`
 }
+
+type BlobInfo struct {
+	// Digest is the digest of the blob in the form of '<algorithm>:<checksum>'.
+	Digest string `json:"digest"`
+
+	// Tag/Version of the blob
+	Tag string `json:"tag"`
+
+	// Size is the number of bytes of the blob.
+	// Can be used to determine how to file should be handled when downloaded (memory/disk)
+	Size int64 `json:"size"`
+}
diff --git a/api/v1alpha1/component_types.go b/api/v1alpha1/component_types.go
@@ -100,11 +100,11 @@ type ComponentStatus struct {
 	// +optional
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 
-	// ArtifactRef references the generated artifact containing a list of
+	// SnapshotRef references the generated snapshot containing a list of
 	// component descriptors. This list can be used by other controllers to
 	// avoid re-downloading (and potentially also re-verifying) the components.
 	// +optional
-	ArtifactRef corev1.LocalObjectReference `json:"artifactRef,omitempty"`
+	SnapshotRef corev1.LocalObjectReference `json:"snapshotRef,omitempty"`
 
 	// Component specifies the concrete version of the component that was
 	// fetched after based on the semver constraints during the last successful
@@ -180,6 +180,10 @@ func (in *Component) GetVerifications() []Verification {
 	return in.Spec.Verify
 }
 
+func (in *Component) GetSnapshotName() string {
+	return in.Status.SnapshotRef.Name
+}
+
 // +kubebuilder:object:root=true
 
 // ComponentList contains a list of Component.