Add the skip_caching_images flag to daisy.

Daisy fetches all the images as part of populating the image cache. When there are a lot of images in the GCP project, Daisy calls the compute API multiple times due to pagination calls thus slowing down the build as well as exhausting the Compute Engine `List Requests Per Minute` Quota

PiperOrigin-RevId: 713503325

cli/main.go

@@ -47,6 +47,9 @@ var (
 	gcsLogsDisabled    = flag.Bool("disable_gcs_logging", false, "do not stream logs to GCS")
 	cloudLogsDisabled  = flag.Bool("disable_cloud_logging", false, "do not stream logs to Cloud Logging")
 	stdoutLogsDisabled = flag.Bool("disable_stdout_logging", false, "do not display individual workflow logs on stdout")
+	// Projects with numerous images can improve performance by getting images directly rather than
+	// listing and caching them, since listing becomes slower than direct retrieval in such cases.
+	skipCachingImages = flag.Bool("skip_caching_images", false, "do not cache images from workflow project on startup")
 )
 
 const (
@@ -75,7 +78,7 @@ func populateVars(input string) map[string]string {
 	return varMap
 }
 
-func parseWorkflow(ctx context.Context, path string, varMap map[string]string, project, zone, gcsPath, oauth, dTimeout, cEndpoint string, disableGCSLogs, diableCloudLogs, disableStdoutLogs bool) (*daisy.Workflow, error) {
+func parseWorkflow(ctx context.Context, path string, varMap map[string]string, project, zone, gcsPath, oauth, dTimeout, cEndpoint string, disableGCSLogs, diableCloudLogs, disableStdoutLogs, skipCachingImages bool) (*daisy.Workflow, error) {
 	w, err := daisy.NewFromFile(path)
 	if err != nil {
 		return nil, err
@@ -131,6 +134,10 @@ Loop:
 		w.DisableStdoutLogging()
 	}
 
+	if skipCachingImages {
+		w.SkipCachingImages()
+	}
+
 	return w, nil
 }
 
@@ -245,7 +252,7 @@ func main() {
 	varMap := populateVars(*variables)
 
 	for _, path := range flag.Args() {
-		w, err := parseWorkflow(ctx, path, varMap, *project, *zone, *gcsPath, *oauth, *defaultTimeout, *ce, *gcsLogsDisabled, *cloudLogsDisabled, *stdoutLogsDisabled)
+		w, err := parseWorkflow(ctx, path, varMap, *project, *zone, *gcsPath, *oauth, *defaultTimeout, *ce, *gcsLogsDisabled, *cloudLogsDisabled, *stdoutLogsDisabled, *skipCachingImages)
 		if err != nil {
 			log.Fatalf("error parsing workflow %q: %v", path, err)
 		}

cli/main_test.go

@@ -81,7 +81,7 @@ func TestParseWorkflows(t *testing.T) {
 	oauth := "oauthpath"
 	dTimeout := "10m"
 	endpoint := "endpoint"
-	w, err := parseWorkflow(context.Background(), path, varMap, project, zone, gcsPath, oauth, dTimeout, endpoint, true, true, true)
+	w, err := parseWorkflow(context.Background(), path, varMap, project, zone, gcsPath, oauth, dTimeout, endpoint, true, true, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}

image.go

@@ -72,10 +72,13 @@ func (w *Workflow) imageExists(project, family, image string) (bool, DError) {
 	}
 	w.imageCache.mu.Lock()
 	defer w.imageCache.mu.Unlock()
-	err := w.imageCache.loadCache(func(project string, opts ...daisyCompute.ListCallOption) (interface{}, error) {
-		return w.ComputeClient.ListImages(project)
-	}, project, image)
-	if err != nil {
+	var err error
+	if !w.disableCachingImages {
+		err = w.imageCache.loadCache(func(project string, opts ...daisyCompute.ListCallOption) (interface{}, error) {
+			return w.ComputeClient.ListImages(project)
+		}, project, image)
+	}
+	if w.disableCachingImages || err != nil {
 		ic, err := w.ComputeClient.GetImage(project, image)
 		if err != nil {
 			return false, typedErr(apiError, "error getting resource for project", err)

workflow.go

@@ -130,6 +130,7 @@ type Workflow struct {
 	gcsLoggingDisabled    bool
 	cloudLoggingDisabled  bool
 	stdoutLoggingDisabled bool
+	disableCachingImages  bool
 	id                    string
 	Logger                Logger `json:"-"`
 	cleanupHooks          []func() DError
@@ -200,6 +201,11 @@ func (w *Workflow) DisableStdoutLogging() {
 	w.stdoutLoggingDisabled = true
 }
 
+// SkipCachingImages skips caching images for this workflow.
+func (w *Workflow) SkipCachingImages() {
+	w.disableCachingImages = true
+}
+
 // AddVar adds a variable set to the Workflow.
 func (w *Workflow) AddVar(k, v string) {
 	if w.Vars == nil {