Merge pull request #8 from devtron-labs/release-candidate-v0.20.0

misc: Release candidate v0.20.0

chart-sync/go.mod

@@ -4,7 +4,10 @@ go 1.22.4
 
 toolchain go1.22.6
 
-replace helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42
+replace (
+	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241010124815-ef03c0637ef4
+	helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42
+)
 
 require (
 	github.com/caarlos0/env v3.5.0+incompatible

chart-sync/go.sum

@@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/devtron-labs/common-lib v0.19.0 h1:fz+8Uaacv1bmvS+V5dH38/2UbZzebBDCSxvEMBVtBEc=
-github.com/devtron-labs/common-lib v0.19.0/go.mod h1:I+B+0ZeOV1Qv8dE/uNAFXOhw7lxfD6FqK6KzTBLBY7E=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241010124815-ef03c0637ef4 h1:AmmIR51ov3csUGBKIPpuPyC6B96kF1a2r/ossFj0m9U=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241010124815-ef03c0637ef4/go.mod h1:KpKnF4OSpQNDJmb4wVZq3Za88ePBw4xec2GOAGRm5UQ=
 github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY=
 github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=

chart-sync/vendor/modules.txt

@@ -93,7 +93,7 @@ github.com/containerd/platforms
 # github.com/davecgh/go-spew v1.1.1
 ## explicit
 github.com/davecgh/go-spew/spew
-# github.com/devtron-labs/common-lib v0.19.0
+# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241010124815-ef03c0637ef4
 ## explicit; go 1.21
 github.com/devtron-labs/common-lib/helmLib/registry
 github.com/devtron-labs/common-lib/utils/http
@@ -785,3 +785,4 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.3.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
+# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20241010124815-ef03c0637ef4

ci-runner/app/CiCdProcessor.go

@@ -25,6 +25,7 @@ import (
 	"github.com/devtron-labs/ci-runner/helper"
 	"github.com/devtron-labs/ci-runner/util"
 	blob_storage "github.com/devtron-labs/common-lib/blob-storage"
+	"github.com/devtron-labs/common-lib/utils/workFlow"
 	"log"
 	"os"
 	"os/signal"
@@ -80,15 +81,15 @@ func (impl *CiCdProcessor) ProcessCiCdEvent(ciCdRequest *helper.CiCdTriggerEvent
 	exitCode := 0
 	if ciCdRequestErr != nil {
 		log.Println(ciCdRequestErr)
-		exitCode = util.DefaultErrorCode
+		exitCode = workFlow.DefaultErrorCode
 		return
 	}
 	// Create a channel to receive the SIGTERM signal
 	sigTerm := make(chan os.Signal, 1)
 	signal.Notify(sigTerm, syscall.SIGTERM)
 
 	go func() {
-		var abortErrorCode = util.AbortErrorCode
+		var abortErrorCode = workFlow.AbortErrorCode
 		log.Println(util.DEVTRON, "SIGTERM listener started!")
 		receivedSignal := <-sigTerm
 		log.Println(util.DEVTRON, "signal received: ", receivedSignal)

ci-runner/executor/StageExecutor.go

@@ -23,6 +23,7 @@ import (
 	util2 "github.com/devtron-labs/ci-runner/executor/util"
 	"github.com/devtron-labs/ci-runner/helper"
 	"github.com/devtron-labs/ci-runner/util"
+	"github.com/devtron-labs/common-lib/utils/workFlow"
 	copylib "github.com/otiai10/copy"
 	"log"
 	"os"
@@ -36,7 +37,7 @@ type StageExecutorImpl struct {
 
 type StageExecutor interface {
 	RunCiCdSteps(stepType helper.StepType, ciCdRequest *helper.CommonWorkflowRequest, steps []*helper.StepObject, refStageMap map[int][]*helper.StepObject, globalEnvironmentVariables map[string]string, preCiStageVariable map[int]map[string]*helper.VariableObject) (pluginArtifacts *helper.PluginArtifacts, outVars map[int]map[string]*helper.VariableObject, failedStep *helper.StepObject, err error)
-	RunCdStageTasks(ciContext cictx.CiContext, tasks []*helper.Task, scriptEnvs map[string]string) error
+	RunCdStageTasks(ciContext cictx.CiContext, tasks []*helper.Task, scriptEnvs map[string]string, stageType helper.PipelineType) error
 }
 
 func NewStageExecutorImpl(cmdExecutor helper.CommandExecutor, scriptExecutor ScriptExecutor) *StageExecutorImpl {
@@ -384,7 +385,7 @@ func deduceVariables(desiredVars []*helper.VariableObject, globalVars map[string
 
 }
 
-func (impl *StageExecutorImpl) RunCdStageTasks(ciContext cictx.CiContext, tasks []*helper.Task, scriptEnvs map[string]string) error {
+func (impl *StageExecutorImpl) RunCdStageTasks(ciContext cictx.CiContext, tasks []*helper.Task, scriptEnvs map[string]string, stageType helper.PipelineType) error {
 	log.Println(util.DEVTRON, " cd-stage-processing")
 	//cleaning the directory
 	err := os.RemoveAll(util.Output_path)
@@ -408,7 +409,9 @@ func (impl *StageExecutorImpl) RunCdStageTasks(ciContext cictx.CiContext, tasks
 		log.Println(util.DEVTRON, "stage", task)
 		err := impl.scriptExecutor.RunScriptsV1(ciContext, util.Output_path, fmt.Sprintf("stage-%d", i), task.Script, scriptEnvs)
 		if err != nil {
-			return err
+			return helper.NewCdStageError(err).
+				WithFailureMessage(fmt.Sprintf(workFlow.CdStageTaskFailed.String(), stageType, task.Name)).
+				WithArtifactUploaded(false)
 		}
 	}
 	return nil

ci-runner/executor/stage/cdStages.go

@@ -18,6 +18,10 @@ package stage
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"github.com/devtron-labs/ci-runner/helper/adaptor"
+	"github.com/devtron-labs/common-lib/utils/workFlow"
 	"log"
 	"os"
 
@@ -42,21 +46,70 @@ func NewCdStage(gitManager helper.GitManager, dockerHelper helper.DockerHelper,
 	}
 }
 
-func (impl *CdStage) HandleCDEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) {
-	err := impl.runCDStages(ciCdRequest)
-	artifactUploadErr := collectAndUploadCDArtifacts(ciCdRequest.CommonWorkflowRequest)
-	if err != nil || artifactUploadErr != nil {
-		log.Println(err)
-		*exitCode = util.DefaultErrorCode
+func deferCDEvent(cdRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) {
+	log.Println(util.DEVTRON, "defer CD stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded)
+	if err != nil {
+		exitCode = workFlow.DefaultErrorCode
+		var stageError *helper.CdStageError
+		if errors.As(err, &stageError) {
+			// update artifact uploaded status
+			if !stageError.IsArtifactUploaded() {
+				stageError = stageError.WithArtifactUploaded(artifactUploaded)
+			}
+		} else {
+			stageError = helper.NewCdStageError(fmt.Errorf(workFlow.CdStageFailed.String(), cdRequest.GetCdStageType(), err)).
+				WithArtifactUploaded(artifactUploaded)
+		}
+		// send ci failure event, for ci failure notification
+		sendCDFailureEvent(cdRequest, stageError)
+		// populate stage error
+		util.PopulateStageError(stageError.ErrorMessage())
 	}
+	return exitCode
+}
 
+func (impl *CdStage) HandleCDEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) {
+	var artifactUploaded bool
+	var err error
+	var allPluginArtifacts *helper.PluginArtifacts
+	defer func() {
+		*exitCode = deferCDEvent(ciCdRequest.CommonWorkflowRequest, artifactUploaded, err)
+	}()
+	allPluginArtifacts, err = impl.runCDStages(ciCdRequest)
+	if err != nil {
+		log.Println("cd stage error: ", err)
+		// not returning error as we want to upload artifacts
+	}
+	var artifactUploadErr error
+	artifactUploaded, artifactUploadErr = collectAndUploadCDArtifacts(ciCdRequest.CommonWorkflowRequest)
+	if artifactUploadErr != nil {
+		log.Println("error in artifact upload: ", artifactUploadErr)
+		// if artifact upload fails, treat it as exit status code 1 and set err to artifact upload error
+		if err == nil {
+			err = artifactUploadErr
+		}
+	}
+	// IsVirtualExecution run flag indicates that cd stage is running in virtual mode.
+	// specifically for isolated environment type, for IsVirtualExecution we don't send success event.
+	// but failure event is sent in case of error.
+	if err == nil && !ciCdRequest.CommonWorkflowRequest.IsVirtualExecution {
+		log.Println(util.DEVTRON, " event")
+		event := adaptor.NewCdCompleteEvent(ciCdRequest.CommonWorkflowRequest).
+			WithPluginArtifacts(allPluginArtifacts).
+			WithIsArtifactUploaded(artifactUploaded)
+		err = helper.SendCDEvent(ciCdRequest.CommonWorkflowRequest, event)
+		if err != nil {
+			log.Println(err)
+		}
+		log.Println(util.DEVTRON, " /event")
+	}
+	return
 }
 
-func collectAndUploadCDArtifacts(cdRequest *helper.CommonWorkflowRequest) error {
+func collectAndUploadCDArtifacts(cdRequest *helper.CommonWorkflowRequest) (artifactUploaded bool, err error) {
 	cloudHelperBaseConfig := cdRequest.GetCloudHelperBaseConfig(util.BlobStorageObjectTypeArtifact)
 	if cdRequest.PrePostDeploySteps != nil && len(cdRequest.PrePostDeploySteps) > 0 {
-		err := helper.ZipAndUpload(cloudHelperBaseConfig, cdRequest.CiArtifactFileName)
-		return err
+		return helper.ZipAndUpload(cloudHelperBaseConfig, cdRequest.CiArtifactFileName)
 	}
 
 	// to support stage YAML outputs
@@ -82,73 +135,75 @@ func collectAndUploadCDArtifacts(cdRequest *helper.CommonWorkflowRequest) error
 	return helper.UploadArtifact(cloudHelperBaseConfig, artifactFiles, cdRequest.CiArtifactFileName)
 }
 
-func (impl *CdStage) runCDStages(cicdRequest *helper.CiCdTriggerEvent) error {
+func (impl *CdStage) runCDStages(ciCdRequest *helper.CiCdTriggerEvent) (*helper.PluginArtifacts, error) {
 	err := os.Chdir("/")
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if _, err := os.Stat(util.WORKINGDIR); os.IsNotExist(err) {
 		_ = os.Mkdir(util.WORKINGDIR, os.ModeDir)
 	}
 	err = os.Chdir(util.WORKINGDIR)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	// git handling
 	// we are skipping clone and checkout in case of ci job type poll cr images plugin does not require it.(ci-job)
-	skipCheckout := cicdRequest.CommonWorkflowRequest.CiPipelineType == helper.CI_JOB
+	skipCheckout := ciCdRequest.CommonWorkflowRequest.CiPipelineType == helper.CI_JOB
 	if !skipCheckout {
 		log.Println(util.DEVTRON, " git")
-		err = impl.gitManager.CloneAndCheckout(cicdRequest.CommonWorkflowRequest.CiProjectDetails)
+		err = impl.gitManager.CloneAndCheckout(ciCdRequest.CommonWorkflowRequest.CiProjectDetails)
 		if err != nil {
 			log.Println(util.DEVTRON, "clone err: ", err)
-			return err
+			return nil, err
 		}
 	}
 	log.Println(util.DEVTRON, " /git")
 	// Start docker daemon
 	log.Println(util.DEVTRON, " docker-start")
-	impl.dockerHelper.StartDockerDaemon(cicdRequest.CommonWorkflowRequest)
-	ciContext := cictx.BuildCiContext(context.Background(), cicdRequest.CommonWorkflowRequest.EnableSecretMasking)
+	impl.dockerHelper.StartDockerDaemon(ciCdRequest.CommonWorkflowRequest)
+	ciContext := cictx.BuildCiContext(context.Background(), ciCdRequest.CommonWorkflowRequest.EnableSecretMasking)
 	err = impl.dockerHelper.DockerLogin(ciContext, &helper.DockerCredentials{
-		DockerUsername:     cicdRequest.CommonWorkflowRequest.DockerUsername,
-		DockerPassword:     cicdRequest.CommonWorkflowRequest.DockerPassword,
-		AwsRegion:          cicdRequest.CommonWorkflowRequest.AwsRegion,
-		AccessKey:          cicdRequest.CommonWorkflowRequest.AccessKey,
-		SecretKey:          cicdRequest.CommonWorkflowRequest.SecretKey,
-		DockerRegistryURL:  cicdRequest.CommonWorkflowRequest.IntermediateDockerRegistryUrl,
-		DockerRegistryType: cicdRequest.CommonWorkflowRequest.DockerRegistryType,
+		DockerUsername:     ciCdRequest.CommonWorkflowRequest.DockerUsername,
+		DockerPassword:     ciCdRequest.CommonWorkflowRequest.DockerPassword,
+		AwsRegion:          ciCdRequest.CommonWorkflowRequest.AwsRegion,
+		AccessKey:          ciCdRequest.CommonWorkflowRequest.AccessKey,
+		SecretKey:          ciCdRequest.CommonWorkflowRequest.SecretKey,
+		DockerRegistryURL:  ciCdRequest.CommonWorkflowRequest.IntermediateDockerRegistryUrl,
+		DockerRegistryType: ciCdRequest.CommonWorkflowRequest.DockerRegistryType,
 	})
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	scriptEnvs, err := util2.GetGlobalEnvVariables(cicdRequest)
+	scriptEnvs, err := util2.GetGlobalEnvVariables(ciCdRequest)
 
 	allPluginArtifacts := helper.NewPluginArtifact()
-	if len(cicdRequest.CommonWorkflowRequest.PrePostDeploySteps) > 0 {
+	if len(ciCdRequest.CommonWorkflowRequest.PrePostDeploySteps) > 0 {
 		refStageMap := make(map[int][]*helper.StepObject)
-		for _, ref := range cicdRequest.CommonWorkflowRequest.RefPlugins {
+		for _, ref := range ciCdRequest.CommonWorkflowRequest.RefPlugins {
 			refStageMap[ref.Id] = ref.Steps
 		}
-		scriptEnvs["DEST"] = cicdRequest.CommonWorkflowRequest.CiArtifactDTO.Image
-		scriptEnvs["DIGEST"] = cicdRequest.CommonWorkflowRequest.CiArtifactDTO.ImageDigest
-		var stage = helper.StepType(cicdRequest.CommonWorkflowRequest.StageType)
-		pluginArtifacts, _, _, err := impl.stageExecutorManager.RunCiCdSteps(stage, cicdRequest.CommonWorkflowRequest, cicdRequest.CommonWorkflowRequest.PrePostDeploySteps, refStageMap, scriptEnvs, nil)
+		scriptEnvs["DEST"] = ciCdRequest.CommonWorkflowRequest.CiArtifactDTO.Image
+		scriptEnvs["DIGEST"] = ciCdRequest.CommonWorkflowRequest.CiArtifactDTO.ImageDigest
+		var stage = helper.StepType(ciCdRequest.CommonWorkflowRequest.StageType)
+		pluginArtifacts, _, step, err := impl.stageExecutorManager.RunCiCdSteps(stage, ciCdRequest.CommonWorkflowRequest, ciCdRequest.CommonWorkflowRequest.PrePostDeploySteps, refStageMap, scriptEnvs, nil)
 		if err != nil {
-			return err
+			return allPluginArtifacts, helper.NewCdStageError(err).
+				WithFailureMessage(fmt.Sprintf(workFlow.CdStageTaskFailed.String(), ciCdRequest.CommonWorkflowRequest.GetCdStageType(), step.Name)).
+				WithArtifactUploaded(false)
 		}
 		allPluginArtifacts.MergePluginArtifact(pluginArtifacts)
 	} else {
 
 		// Get devtron-cd yaml
-		taskYaml, err := helper.ToTaskYaml([]byte(cicdRequest.CommonWorkflowRequest.StageYaml))
+		taskYaml, err := helper.ToTaskYaml([]byte(ciCdRequest.CommonWorkflowRequest.StageYaml))
 		if err != nil {
 			log.Println(err)
-			return err
+			return allPluginArtifacts, err
 		}
-		cicdRequest.CommonWorkflowRequest.TaskYaml = taskYaml
+		ciCdRequest.CommonWorkflowRequest.TaskYaml = taskYaml
 
 		// run post artifact processing
 		log.Println(util.DEVTRON, " stage yaml", taskYaml)
@@ -158,28 +213,16 @@ func (impl *CdStage) runCDStages(cicdRequest *helper.CiCdTriggerEvent) error {
 			tasks = append(tasks, t.AfterTasks...)
 		}
 
+		err = impl.stageExecutorManager.RunCdStageTasks(ciContext, tasks, scriptEnvs, ciCdRequest.CommonWorkflowRequest.GetCdStageType())
 		if err != nil {
-			return err
-		}
-		err = impl.stageExecutorManager.RunCdStageTasks(ciContext, tasks, scriptEnvs)
-		if err != nil {
-			return err
-		}
-	}
-	// dry run flag indicates that ci runner image is being run from external helm chart
-	if !cicdRequest.CommonWorkflowRequest.IsDryRun {
-		log.Println(util.DEVTRON, " event")
-		err = helper.SendCDEvent(cicdRequest.CommonWorkflowRequest, allPluginArtifacts)
-		if err != nil {
-			log.Println(err)
-			return err
+			return allPluginArtifacts, err
 		}
-		log.Println(util.DEVTRON, " /event")
 	}
+
 	err = impl.dockerHelper.StopDocker(ciContext)
 	if err != nil {
 		log.Println("error while stopping docker", err)
-		return err
+		return allPluginArtifacts, err
 	}
-	return nil
+	return allPluginArtifacts, nil
 }