Merge branch 'project-zot:main' into fix-index-header

.github/actions/clean-runner/action.yaml

@@ -0,0 +1,12 @@
+name: 'Clean runner'
+description: 'Remove unneeded tooling'
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        # To free up ~15 GB of disk space
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /usr/local/share/boost
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /usr/share/dotnet

.github/workflows/ci-cd.yml

@@ -32,21 +32,14 @@ jobs:
         os: [linux, darwin]
         arch: [amd64, arm64]
     steps:
-      - name: Remove unneeded tooling
-        run: |
-          # To free up ~15 GB of disk space
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /usr/local/share/boost
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /usr/share/dotnet
+      - name: Check out source code
+        uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
       - name: Install go
         uses: actions/setup-go@v4
         with:
           cache: false
           go-version: 1.20.x
-
-      - name: Check out source code
-        uses: actions/checkout@v3
       - name: Cache go dependencies
         id: cache-go-dependencies
         uses: actions/cache@v3

.github/workflows/nightly.yaml

@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20.x
@@ -59,7 +60,9 @@ jobs:
     name: Sync harness
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Check out source code
+        uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20.x

Makefile

@@ -129,6 +129,7 @@ $(TESTDATA): check-skopeo
 	skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TESTDATA}/zot-test:0.0.1; \
 	skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TESTDATA}/zot-cve-test:0.0.1; \
 	skopeo --insecure-policy copy -q docker://ghcr.io/project-zot/test-images/java:0.0.1 oci:${TESTDATA}/zot-cve-java-test:0.0.1; \
+	skopeo --insecure-policy copy -q docker://ghcr.io/project-zot/test-images/alpine:3.17.3 oci:${TESTDATA}/alpine:3.17.3; \
 	chmod -R a=rwx ${TESTDATA}
 
 .PHONY: run-bench

errors/errors.go

@@ -73,7 +73,7 @@ var (
 	ErrEmptyRepoName                  = errors.New("repodb: repo name can't be empty string")
 	ErrEmptyTag                       = errors.New("repodb: tag can't be empty string")
 	ErrEmptyDigest                    = errors.New("repodb: digest can't be empty string")
-	ErrInvalidRepoTagFormat           = errors.New("invalid format for tag search, not following repo:tag")
+	ErrInvalidRepoRefFormat           = errors.New("invalid image reference format")
 	ErrLimitIsNegative                = errors.New("pageturner: limit has negative value")
 	ErrOffsetIsNegative               = errors.New("pageturner: offset has negative value")
 	ErrSortCriteriaNotSupported       = errors.New("pageturner: the sort criteria is not supported")
@@ -96,4 +96,8 @@ var (
 	ErrSyncPingRegistry               = errors.New("sync: unable to ping any registry URLs")
 	ErrSyncImageNotSigned             = errors.New("sync: image is not signed")
 	ErrSyncImageFilteredOut           = errors.New("sync: image is filtered out by sync config")
+	ErrCallerInfo                     = errors.New("runtime: failed to get info regarding the current runtime")
+	ErrInvalidTruststoreType          = errors.New("signatures: invalid truststore type")
+	ErrInvalidTruststoreName          = errors.New("signatures: invalid truststore name")
+	ErrInvalidCertificateContent      = errors.New("signatures: invalid certificate content")
 )

pkg/api/controller.go

@@ -300,6 +300,8 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, config *config.Con
 		c.Config.Extensions = nil
 	}
 
+	c.InitCVEInfo()
+
 	c.StartBackgroundTasks(reloadCtx)
 
 	c.Log.Info().Interface("reloaded params", c.Config.Sanitize()).
@@ -359,6 +361,12 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 
 		c.SyncOnDemand = syncOnDemand
 	}
+
+	if c.Config.Extensions != nil {
+		if c.Config.Extensions.Mgmt != nil && *c.Config.Extensions.Mgmt.Enable {
+			ext.EnablePeriodicSignaturesVerification(c.Config, taskScheduler, c.RepoDB, c.Log) //nolint: contextcheck
+		}
+	}
 }
 
 type SyncOnDemand interface {

pkg/cli/config_reloader_test.go

@@ -1,3 +1,6 @@
+//go:build search
+// +build search
+
 package cli_test
 
 import (
@@ -322,7 +325,7 @@ func TestConfigReloader(t *testing.T) {
 						"cve": {
 							"updateInterval": "24h",
 							"trivy": {
-								"DBRepository": "ghcr.io/aquasecurity/trivy-db"
+								"DBRepository": "unreachable/trivy/url1"
 							}
 						}
 					},
@@ -367,7 +370,7 @@ func TestConfigReloader(t *testing.T) {
 					"cve": {
 						"updateInterval": "5h",
 						"trivy": {
-							"DBRepository": "ghcr.io/project-zot/trivy-db"
+							"DBRepository": "another/unreachable/trivy/url2"
 						}
 					}
 				}
@@ -387,7 +390,12 @@ func TestConfigReloader(t *testing.T) {
 		So(err, ShouldBeNil)
 
 		// wait for config reload
-		time.Sleep(2 * time.Second)
+		time.Sleep(5 * time.Second)
+
+		found, err := test.ReadLogFileAndSearchString(logFile.Name(),
+			"Error downloading Trivy DB to destination dir", 30*time.Second)
+		So(err, ShouldBeNil)
+		So(found, ShouldBeTrue)
 
 		data, err := os.ReadFile(logFile.Name())
 		So(err, ShouldBeNil)
@@ -397,7 +405,12 @@ func TestConfigReloader(t *testing.T) {
 		So(string(data), ShouldContainSubstring, "loaded new configuration settings")
 		So(string(data), ShouldContainSubstring, "\"UpdateInterval\":18000000000000")
 		So(string(data), ShouldContainSubstring, "\"Scrub\":null")
-		So(string(data), ShouldContainSubstring, "\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"")
+		So(string(data), ShouldContainSubstring, "\"DBRepository\":\"another/unreachable/trivy/url2\"")
+		// matching log message when it errors out, test that indeed the download will try the second url
+		found, err = test.ReadLogFileAndSearchString(logFile.Name(),
+			"\"dbRepository\":\"another/unreachable/trivy/url2\",\"goroutine", 1*time.Minute)
+		So(err, ShouldBeNil)
+		So(found, ShouldBeTrue)
 	})
 
 	Convey("reload bad config", t, func(c C) {

pkg/cli/cve_cmd_test.go

@@ -29,6 +29,7 @@ import (
 	zotErrors "zotregistry.io/zot/errors"
 	"zotregistry.io/zot/pkg/api"
 	"zotregistry.io/zot/pkg/api/config"
+	zcommon "zotregistry.io/zot/pkg/common"
 	extconf "zotregistry.io/zot/pkg/extensions/config"
 	"zotregistry.io/zot/pkg/extensions/monitoring"
 	cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
@@ -1035,7 +1036,7 @@ func TestServerCVEResponse(t *testing.T) {
 		space := regexp.MustCompile(`\s+`)
 		str := space.ReplaceAllString(buff.String(), " ")
 		So(err, ShouldBeNil)
-		So(strings.TrimSpace(str), ShouldEqual,
+		So(strings.TrimSpace(str), ShouldResemble,
 			"IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE zot-cve-test 0.0.1 linux/amd64 40d1f749 false 605B")
 	})
 
@@ -1172,7 +1173,8 @@ func getMockCveInfo(repoDB repodb.RepoDB, log log.Logger) cveinfo.CveInfo {
 	// Setup test CVE data in mock scanner
 	scanner := mocks.CveScannerMock{
 		ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) {
-			if image == "zot-cve-test:0.0.1" {
+			if image == "zot-cve-test@sha256:40d1f74918aefed733c590f798d7eafde8fc0a7ec63bb8bc52eaae133cf92495" ||
+				image == "zot-cve-test:0.0.1" {
 				return map[string]cvemodel.CVE{
 					"CVE-1": {
 						ID:          "CVE-1",
@@ -1223,12 +1225,20 @@ func getMockCveInfo(repoDB repodb.RepoDB, log log.Logger) cveinfo.CveInfo {
 				return false, err
 			}
 
-			manifestDigestStr, ok := repoMeta.Tags[inputTag]
-			if !ok {
-				return false, zotErrors.ErrTagMetaNotFound
+			manifestDigestStr := reference
+
+			if zcommon.IsTag(reference) {
+				var ok bool
+
+				descriptor, ok := repoMeta.Tags[inputTag]
+				if !ok {
+					return false, zotErrors.ErrTagMetaNotFound
+				}
+
+				manifestDigestStr = descriptor.Digest
 			}
 
-			manifestDigest, err := godigest.Parse(manifestDigestStr.Digest)
+			manifestDigest, err := godigest.Parse(manifestDigestStr)
 			if err != nil {
 				return false, err
 			}

pkg/cli/image_cmd_test.go

@@ -1891,8 +1891,8 @@ func (service mockService) getTagsForCVEGQL(ctx context.Context, config searchCo
 
 func (service mockService) getFixedTagsForCVEGQL(ctx context.Context, config searchConfig, username, password,
 	imageName, cveID string,
-) (*common.FixedTags, error) {
-	fixedTags := &common.FixedTags{
+) (*common.ImageListWithCVEFixedResponse, error) {
+	fixedTags := &common.ImageListWithCVEFixedResponse{
 		Errors: nil,
 		ImageListWithCVEFixed: struct {
 			common.PaginatedImagesResult `json:"ImageListWithCVEFixed"` //nolint:tagliatelle // graphQL schema

pkg/cli/service.go

@@ -43,7 +43,7 @@ type SearchService interface { //nolint:interfacebloat
 	getTagsForCVEGQL(ctx context.Context, config searchConfig, username, password, imageName,
 		cveID string) (*common.ImagesForCve, error)
 	getFixedTagsForCVEGQL(ctx context.Context, config searchConfig, username, password, imageName,
-		cveID string) (*common.FixedTags, error)
+		cveID string) (*common.ImageListWithCVEFixedResponse, error)
 	getDerivedImageListGQL(ctx context.Context, config searchConfig, username, password string,
 		derivedImage string) (*common.DerivedImageListResponse, error)
 	getBaseImageListGQL(ctx context.Context, config searchConfig, username, password string,
@@ -377,7 +377,7 @@ func (service searchService) getTagsForCVEGQL(ctx context.Context, config search
 
 func (service searchService) getFixedTagsForCVEGQL(ctx context.Context, config searchConfig,
 	username, password, imageName, cveID string,
-) (*common.FixedTags, error) {
+) (*common.ImageListWithCVEFixedResponse, error) {
 	query := fmt.Sprintf(`
 		{
 			ImageListWithCVEFixed(id: "%s", image: "%s") {
@@ -398,7 +398,7 @@ func (service searchService) getFixedTagsForCVEGQL(ctx context.Context, config s
 		}`,
 		cveID, imageName)
 
-	result := &common.FixedTags{}
+	result := &common.ImageListWithCVEFixedResponse{}
 
 	err := service.makeGraphQLQuery(ctx, config, username, password, query, result)
 
@@ -847,7 +847,7 @@ func (service searchService) getFixedTagsForCVE(ctx context.Context, config sear
 		}
 	}`, cvid, imageName)
 
-	result := &common.FixedTags{}
+	result := &common.ImageListWithCVEFixedResponse{}
 
 	err := service.makeGraphQLQuery(ctx, config, username, password, query, result)
 	if err != nil {

pkg/common/model.go

@@ -112,7 +112,7 @@ type Annotation struct {
 	Value string `json:"value"`
 }
 
-type FixedTags struct {
+type ImageListWithCVEFixedResponse struct {
 	Errors                []ErrorGQL `json:"errors"`
 	ImageListWithCVEFixed `json:"data"`
 }

pkg/common/oci.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	zerr "zotregistry.io/zot/errors"
@@ -101,11 +102,30 @@ func GetRepoRefference(repo string) (string, string, bool, error) {
 		repoName, tag, found := strings.Cut(repo, ":")
 
 		if !found {
-			return "", "", false, zerr.ErrInvalidRepoTagFormat
+			return "", "", false, zerr.ErrInvalidRepoRefFormat
 		}
 
 		return repoName, tag, true, nil
 	}
 
 	return repoName, digest, false, nil
 }
+
+// GetFullImageName returns the formated string for the given repo/tag or repo/digest.
+func GetFullImageName(repo, ref string) string {
+	if IsTag(ref) {
+		return repo + ":" + ref
+	}
+
+	return repo + "@" + ref
+}
+
+func IsDigest(ref string) bool {
+	_, err := digest.Parse(ref)
+
+	return err == nil
+}
+
+func IsTag(ref string) bool {
+	return !IsDigest(ref)
+}