diff --git a/chart-sync/Dockerfile b/chart-sync/Dockerfile index af25043cf..cf447d61a 100644 --- a/chart-sync/Dockerfile +++ b/chart-sync/Dockerfile @@ -1,22 +1,25 @@ FROM golang:1.22 AS build-env -RUN apt update -RUN apt install git gcc musl-dev make -y -RUN go install github.com/google/wire/cmd/wire@latest +RUN apt update && \ + apt install git gcc musl-dev make -y && \ + go install github.com/google/wire/cmd/wire@latest WORKDIR /go/src/github.com/devtron-labs/chart-sync + ADD . /go/src/github.com/devtron-labs/chart-sync + RUN GOOS=linux make FROM ubuntu -RUN apt update -RUN apt install ca-certificates -y -RUN apt clean autoclean -RUN apt autoremove -y && rm -rf /var/lib/apt/lists/* -COPY --from=build-env /go/src/github.com/devtron-labs/chart-sync/chart-sync . - -RUN useradd -ms /bin/bash devtron -RUN chown -R devtron:devtron ./chart-sync + +RUN apt update && \ + apt install ca-certificates -y && \ + apt clean autoclean && \ + apt autoremove -y && rm -rf /var/lib/apt/lists/* && \ + useradd -ms /bin/bash devtron + +COPY --chown=devtron:devtron --from=build-env /go/src/github.com/devtron-labs/chart-sync/chart-sync . + USER devtron CMD ["./chart-sync"] diff --git a/chart-sync/go.mod b/chart-sync/go.mod index e81069f67..c5a766737 100644 --- a/chart-sync/go.mod +++ b/chart-sync/go.mod @@ -5,7 +5,7 @@ go 1.22.4 toolchain go1.22.6 replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f helm.sh/helm/v3 v3.14.3 => github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 ) diff --git a/chart-sync/go.sum b/chart-sync/go.sum index ad3c58aa7..cda7489c3 100644 --- a/chart-sync/go.sum +++ b/chart-sync/go.sum @@ -54,8 +54,8 @@ github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42 h1:pJmK44QaSztOiZe0iQHNf0sdy5KwkAeceydyhOG4RaY= github.com/devtron-labs/helm/v3 v3.14.1-0.20240401080259-90238cf69e42/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/chart-sync/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/chart-sync/vendor/modules.txt b/chart-sync/vendor/modules.txt index d2853f8f1..89461210e 100644 --- a/chart-sync/vendor/modules.txt +++ b/chart-sync/vendor/modules.txt @@ -93,7 +93,7 @@ github.com/containerd/platforms # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager/util @@ -790,4 +790,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/ci-runner/Dockerfile b/ci-runner/Dockerfile index 559d28da9..d516493ec 100644 --- a/ci-runner/Dockerfile +++ b/ci-runner/Dockerfile @@ -1,38 +1,46 @@ -####-------------- FROM golang:1.21-alpine3.18 AS build-env -RUN apk add --no-cache git gcc musl-dev -RUN apk add --update make +RUN apk add --no-cache git gcc musl-dev && \ + apk add --update make WORKDIR /go/src/github.com/devtron-labs/cirunner -ADD . /go/src/github.com/devtron-labs/cirunner/ + +# ADD . /go/src/github.com/devtron-labs/cirunner/ COPY . . + # Build the binary RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -a -installsuffix cgo -o /go/bin/cirunner FROM docker:20.10.24-dind # All these steps will be cached #RUN apk add --no-cache ca-certificates -RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq -RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN apk -Uuv add groff less python3 py3-pip -RUN pip3 install awscli -RUN apk --purge -v del py-pip -RUN rm /var/cache/apk/* + +RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && \ + ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + apk -Uuv add groff less python3 py3-pip && \ + pip3 install awscli && \ + apk --purge -v del py-pip && \ + rm /var/cache/apk/* + COPY --from=docker/compose:latest /usr/local/bin/docker-compose /usr/bin/docker-compose -COPY ./buildpack.json /buildpack.json -COPY ./git-ask-pass.sh /git-ask-pass.sh -RUN chmod +x /git-ask-pass.sh +COPY ./buildpack.json ./git-ask-pass.sh / +# COPY ./git-ask-pass.sh /git-ask-pass.sh -RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) +RUN chmod +x /git-ask-pass.sh && \ + (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) COPY --from=build-env /go/bin/cirunner . + COPY ./ssh-config /root/.ssh/config -RUN chmod 644 /root/.ssh/config -RUN mkdir -p /etc/docker + +RUN chmod 644 /root/.ssh/config && \ + mkdir -p /etc/docker && \ + mkdir -p /root/.docker/buildx + COPY docker-daemon.json /etc/docker/daemon.json -RUN mkdir -p /root/.docker/buildx + + COPY buildkitd.default.toml /root/.docker/buildx/buildkitd.default.toml # passing PARENT_MODE as argument to cirunner as default behavior diff --git a/ci-runner/Dockerfile-v27 b/ci-runner/Dockerfile-v27 index d7c49cfaf..7a337b48c 100644 --- a/ci-runner/Dockerfile-v27 +++ b/ci-runner/Dockerfile-v27 @@ -1,11 +1,12 @@ -####-------------- FROM golang:1.21-alpine3.18 AS build-env -RUN apk add --no-cache git gcc musl-dev -RUN apk add --update make +RUN apk add --no-cache git gcc musl-dev && \ + apk add --update make WORKDIR /go/src/github.com/devtron-labs/cirunner + ADD . /go/src/github.com/devtron-labs/cirunner/ + COPY . . # Build the binary RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -a -installsuffix cgo -o /go/bin/cirunner @@ -14,24 +15,26 @@ RUN CGO_ENABLED=0 GOOS=linux go build -buildvcs=false -a -installsuffix cgo -o / FROM docker:27.3-dind # All these steps will be cached #RUN apk add --no-cache ca-certificates -RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq -RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN apk -Uuv add groff less python3 py3-pip pipx -RUN pipx ensurepath && pipx install awscli -RUN cp /root/.local/bin/aws* /usr/local/bin -RUN apk --purge -v del py-pip -RUN rm /var/cache/apk/* + +RUN apk update && apk add --no-cache --virtual .build-deps && apk add bash && apk add make && apk add curl && apk add git && apk add zip && apk add jq && \ + ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + apk -Uuv add groff less python3 py3-pip pipx && \ + pipx ensurepath && pipx install awscli && \ + cp /root/.local/bin/aws* /usr/local/bin && \ + apk --purge -v del py-pip && \ + rm /var/cache/apk/* + COPY --from=docker/compose:1.29.2 /usr/local/bin/docker-compose /usr/bin/docker-compose -COPY ./buildpack.json /buildpack.json -COPY ./git-ask-pass.sh /git-ask-pass.sh -RUN chmod +x /git-ask-pass.sh +COPY ./buildpack.json /git-ask-pass.sh / -RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) +# COPY ./git-ask-pass.sh /git-ask-pass.sh +RUN chmod +x /git-ask-pass.sh && \ + (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) COPY --from=build-env /go/bin/cirunner . COPY ./ssh-config /root/.ssh/config RUN chmod 644 /root/.ssh/config # passing PARENT_MODE as argument to cirunner as default behavior -ENTRYPOINT ["./cirunner", "PARENT_MODE"] \ No newline at end of file +ENTRYPOINT ["./cirunner", "PARENT_MODE"] diff --git a/ci-runner/executor/StageExecutor.go b/ci-runner/executor/StageExecutor.go index 3b959e4e0..c3fb6c7bc 100644 --- a/ci-runner/executor/StageExecutor.go +++ b/ci-runner/executor/StageExecutor.go @@ -75,7 +75,7 @@ func (impl *StageExecutorImpl) RunCiCdSteps(stepType helper.StepType, ciCdReques if stageInfoLoggingRequired { log.Println(util.DEVTRON, "stage logging required") - err = util.ExecuteWithStageInfoLog(step.Name, executeStep) + err = util.ExecuteWithStageInfoLog(helper.GetPrePostStageDisplayName(step.Name, stepType), executeStep) } else { log.Println(util.DEVTRON, "stage logging not required") err = executeStep() @@ -144,6 +144,9 @@ func (impl *StageExecutorImpl) RunCiCdStep(stepType helper.StepType, ciCdRequest var vars []*commonBean.VariableObject if stepType == helper.STEP_TYPE_REF_PLUGIN { vars, err = deduceVariables(step.InputVars, scriptEnvVariables, nil, nil, stageVariable) + } else if stepType == helper.STEP_TYPE_SCANNING { + // only global variables are supported here in image scanning step + vars, err = deduceVariables(step.InputVars, scriptEnvVariables, nil, nil, nil) } else { log.Printf("running step : %s\n", step.Name) if stepType == helper.STEP_TYPE_PRE { diff --git a/ci-runner/executor/adaptor/ImageScanAdaptor.go b/ci-runner/executor/adaptor/ImageScanAdaptor.go index e363e9ef9..6eeb6fdd6 100644 --- a/ci-runner/executor/adaptor/ImageScanAdaptor.go +++ b/ci-runner/executor/adaptor/ImageScanAdaptor.go @@ -17,6 +17,8 @@ package adaptor import ( + bean2 "github.com/devtron-labs/ci-runner/executor/stage/bean" + util2 "github.com/devtron-labs/ci-runner/executor/util" "github.com/devtron-labs/ci-runner/helper" "github.com/devtron-labs/common-lib/constants" "github.com/devtron-labs/common-lib/imageScan/bean" @@ -42,3 +44,14 @@ func GetImageScanEvent(dest, digest string, commonWorkflowRequest *helper.Common ImageScanRetryDelay: commonWorkflowRequest.ImageScanRetryDelay, } } +func GetImageScannerExecutorBean(ciCdRequest *helper.CiCdTriggerEvent, scriptEnvs *util2.ScriptEnvVariables, refStageMap map[int][]*helper.StepObject, metrics *helper.CIMetrics, artifactUploaded bool, dest string, digest string) *bean2.ImageScanningExecutorBean { + return &bean2.ImageScanningExecutorBean{ + CiCdRequest: ciCdRequest, + ScriptEnvs: scriptEnvs, + RefStageMap: refStageMap, + Metrics: metrics, + ArtifactUploaded: artifactUploaded, + Dest: dest, + Digest: digest, + } +} diff --git a/ci-runner/executor/helper/envVariableHelper.go b/ci-runner/executor/helper/envVariableHelper.go new file mode 100644 index 000000000..bdda8bff7 --- /dev/null +++ b/ci-runner/executor/helper/envVariableHelper.go @@ -0,0 +1,7 @@ +package helper + +import util2 "github.com/devtron-labs/ci-runner/executor/util" + +func SetKeyValueInGlobalSystemEnv(scriptEnvs *util2.ScriptEnvVariables, key, value string) { + scriptEnvs.SystemEnv[key] = value +} diff --git a/ci-runner/executor/stage/bean/bean.go b/ci-runner/executor/stage/bean/bean.go index f66364d82..203011d6d 100644 --- a/ci-runner/executor/stage/bean/bean.go +++ b/ci-runner/executor/stage/bean/bean.go @@ -16,9 +16,28 @@ package bean +import ( + util2 "github.com/devtron-labs/ci-runner/executor/util" + "github.com/devtron-labs/ci-runner/helper" +) + const ( ExternalCiArtifact = "externalCiArtifact" ImageDigest = "imageDigest" UseAppDockerConfig = "useAppDockerConfig" CiProjectDetails = "ciProjectDetails" ) +const ( + DigestGlobalEnvKey = "DIGEST" + ScanToolIdGlobalEnvKey = "SCAN_TOOL_ID" +) + +type ImageScanningExecutorBean struct { + CiCdRequest *helper.CiCdTriggerEvent + ScriptEnvs *util2.ScriptEnvVariables + RefStageMap map[int][]*helper.StepObject + Metrics *helper.CIMetrics + ArtifactUploaded bool + Dest string + Digest string +} diff --git a/ci-runner/executor/stage/ciStages.go b/ci-runner/executor/stage/ciStages.go index 324fee828..fa06ae702 100644 --- a/ci-runner/executor/stage/ciStages.go +++ b/ci-runner/executor/stage/ciStages.go @@ -24,6 +24,7 @@ import ( "github.com/devtron-labs/ci-runner/executor" adaptor2 "github.com/devtron-labs/ci-runner/executor/adaptor" cicxt "github.com/devtron-labs/ci-runner/executor/context" + helper2 "github.com/devtron-labs/ci-runner/executor/helper" bean2 "github.com/devtron-labs/ci-runner/executor/stage/bean" util2 "github.com/devtron-labs/ci-runner/executor/util" "github.com/devtron-labs/ci-runner/helper" @@ -257,6 +258,8 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper. return artifactUploaded, err } } + // setting digest in global env + helper2.SetKeyValueInGlobalSystemEnv(scriptEnvs, bean2.DigestGlobalEnvKey, digest) var postCiDuration float64 start = time.Now() metrics.PostCiStartTime = start @@ -299,7 +302,7 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper. // scan only if ci scan enabled if helper.IsEventTypeEligibleToScanImage(ciCdRequest.Type) && ciCdRequest.CommonWorkflowRequest.ScanEnabled { - err = runImageScanning(dest, digest, ciCdRequest, metrics, artifactUploaded) + err = impl.runImageScanning(adaptor2.GetImageScannerExecutorBean(ciCdRequest, scriptEnvs, refStageMap, metrics, artifactUploaded, dest, digest)) if err != nil { return artifactUploaded, err } @@ -310,7 +313,8 @@ func (impl *CiStage) runCIStages(ciContext cicxt.CiContext, ciCdRequest *helper. event := adaptor.NewCiCompleteEvent(ciCdRequest.CommonWorkflowRequest).WithMetrics(*metrics). WithDockerImage(dest).WithDigest(digest).WithIsArtifactUploaded(artifactUploaded). - WithImageDetailsFromCR(resultsFromPlugin).WithPluginArtifacts(pluginArtifacts) + WithImageDetailsFromCR(resultsFromPlugin).WithPluginArtifacts(pluginArtifacts). + WithTargetPlatforms(GetTargetPlatformFromCiBuildConfig(ciCdRequest.CommonWorkflowRequest.CiBuildConfig)) err = helper.SendCiCompleteEvent(ciCdRequest.CommonWorkflowRequest, event) if err != nil { log.Println(err) @@ -438,7 +442,12 @@ func (impl *CiStage) runPostCiSteps(ciCdRequest *helper.CiCdTriggerEvent, script return pluginArtifactsFromFile, resultsFromPlugin, nil } -func runImageScanning(dest string, digest string, ciCdRequest *helper.CiCdTriggerEvent, metrics *helper.CIMetrics, artifactUploaded bool) error { +func (impl *CiStage) runImageScanning(imageScannerExecutor *bean2.ImageScanningExecutorBean) error { + ciCdRequest := imageScannerExecutor.CiCdRequest + dest, digest := imageScannerExecutor.Dest, imageScannerExecutor.Digest + metrics, artifactUploaded := imageScannerExecutor.Metrics, imageScannerExecutor.ArtifactUploaded + scriptEnvs, refStageMap := imageScannerExecutor.ScriptEnvs, imageScannerExecutor.RefStageMap + imageScanningStage := func() error { log.Println("Image Scanning Started for digest", digest) scanEvent := adaptor2.GetImageScanEvent(dest, digest, ciCdRequest.CommonWorkflowRequest) @@ -453,7 +462,31 @@ func runImageScanning(dest string, digest string, ciCdRequest *helper.CiCdTrigge log.Println("Image scanning completed with scanEvent", scanEvent) return nil } - + imageScanningTaskExecution := func() error { + log.Println("Image Scanning Started") + for _, allSteps := range ciCdRequest.CommonWorkflowRequest.ImageScanningSteps { + scanToolId := allSteps.ScanToolId + tasks := allSteps.Steps + //setting scan tool id in script env + scriptEnvs.SystemEnv[bean2.ScanToolIdGlobalEnvKey] = strconv.Itoa(scanToolId) + // run image scanning steps + _, _, _, err := impl.stageExecutorManager.RunCiCdSteps(helper.STEP_TYPE_SCANNING, ciCdRequest.CommonWorkflowRequest, tasks, refStageMap, scriptEnvs, nil, true) + if err != nil { + log.Println("error in running pre Ci Steps", "err", err) + return helper.NewCiStageError(err). + WithMetrics(metrics). + WithFailureMessage(workFlow.ScanFailed.String()). + WithArtifactUploaded(artifactUploaded) + } + } + //unset scan tool id in script env + delete(scriptEnvs.SystemEnv, bean2.ScanToolIdGlobalEnvKey) + log.Println("Image scanning completed") + return nil + } + if ciCdRequest.CommonWorkflowRequest.ExecuteImageScanningVia.IsScanMediumExternal() { + return util.ExecuteWithStageInfoLog(util.IMAGE_SCAN, imageScanningTaskExecution) + } return util.ExecuteWithStageInfoLog(util.IMAGE_SCAN, imageScanningStage) } @@ -659,3 +692,13 @@ func (impl *CiStage) extractDigestForCiJob(workflowRequest *helper.CommonWorkflo log.Println(fmt.Sprintf("time since extract digest from image process:- %s", time.Since(startTime).String())) return imgDigest, nil } + +func GetTargetPlatformFromCiBuildConfig(ciBuildConfig *helper.CiBuildConfigBean) []string { + if ciBuildConfig == nil { + return []string{} + } else if ciBuildConfig.DockerBuildConfig == nil { + return []string{} + } else { + return utils.ConvertTargetPlatformStringToList(ciBuildConfig.DockerBuildConfig.TargetPlatform) + } +} diff --git a/ci-runner/executor/util/envUtils.go b/ci-runner/executor/util/envUtils.go index 753b6cb2a..d3e6ed10d 100644 --- a/ci-runner/executor/util/envUtils.go +++ b/ci-runner/executor/util/envUtils.go @@ -115,6 +115,7 @@ func GetGlobalEnvVariables(ciCdRequest *helper.CiCdTriggerEvent) (*ScriptEnvVari RegistryCredentials, _ := json.Marshal(ciCdRequest.CommonWorkflowRequest.RegistryCredentialMap) envs["REGISTRY_DESTINATION_IMAGE_MAP"] = string(RegistryDestinationImage) envs["REGISTRY_CREDENTIALS"] = string(RegistryCredentials) + envs["AWS_INSPECTOR_CONFIG"] = ciCdRequest.CommonWorkflowRequest.AwsInspectorConfig } else { envs["DOCKER_IMAGE"] = ciCdRequest.CommonWorkflowRequest.CiArtifactDTO.Image envs["DOCKER_IMAGE_TAG"] = ciCdRequest.CommonWorkflowRequest.DockerImageTag diff --git a/ci-runner/go.mod b/ci-runner/go.mod index 2f51329e4..b70c95156 100644 --- a/ci-runner/go.mod +++ b/ci-runner/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.21.8 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f require ( github.com/Knetic/govaluate v3.0.0+incompatible diff --git a/ci-runner/go.sum b/ci-runner/go.sum index 00b2eebe3..58c9b3dc2 100644 --- a/ci-runner/go.sum +++ b/ci-runner/go.sum @@ -57,8 +57,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/ci-runner/helper/DockerHelper.go b/ci-runner/helper/DockerHelper.go index de4081849..3241d4b09 100644 --- a/ci-runner/helper/DockerHelper.go +++ b/ci-runner/helper/DockerHelper.go @@ -30,6 +30,8 @@ import ( "github.com/aws/aws-sdk-go/service/ecr" "github.com/caarlos0/env" cicxt "github.com/devtron-labs/ci-runner/executor/context" + bean2 "github.com/devtron-labs/ci-runner/helper/bean" + "github.com/devtron-labs/ci-runner/util" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/bean" @@ -344,7 +346,7 @@ func (impl *DockerHelperImpl) BuildArtifact(ciRequest *CommonWorkflowRequest) (s } useBuildxK8sDriver, eligibleK8sDriverNodes = dockerBuildConfig.CheckForBuildXK8sDriver() if useBuildxK8sDriver { - err = impl.createBuildxBuilderWithK8sDriver(ciContext, ciRequest.DockerConnection, dockerBuildConfig.BuildxDriverImage, eligibleK8sDriverNodes, ciRequest.PipelineId, ciRequest.WorkflowId) + err = impl.createBuildxBuilderWithK8sDriver(ciContext, ciRequest, ciRequest.DockerConnection, dockerBuildConfig.BuildxDriverImage, eligibleK8sDriverNodes, ciRequest.PipelineId, ciRequest.WorkflowId) if err != nil { log.Println(util.DEVTRON, " error in creating buildxDriver , err : ", err.Error()) return err @@ -403,7 +405,7 @@ func (impl *DockerHelperImpl) BuildArtifact(ciRequest *CommonWorkflowRequest) (s return nil } - if err = util.ExecuteWithStageInfoLog(util.DOCKER_BUILD, buildImageStage); err != nil { + if err = util.ExecuteWithStageInfoLogWithMetadata(util.DOCKER_BUILD, bean2.DockerBuildStageMetadata{TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciBuildConfig.DockerBuildConfig.TargetPlatform)}, buildImageStage); err != nil { return "", nil } @@ -557,7 +559,7 @@ func getExportCacheCmds(targetPlatforms, dockerBuild, localCachePath string, use } cacheCmd := "%s --platform=%s --cache-to=type=local,dest=%s,mode=" + cacheMode - platforms := strings.Split(targetPlatforms, ",") + platforms := utils.ConvertTargetPlatformStringToList(targetPlatforms) exportCacheCmds := make(map[string]string) for _, platform := range platforms { @@ -938,14 +940,17 @@ func (impl *DockerHelperImpl) createBuildxBuilderForMultiArchBuild(ciContext cic return nil } -func (impl *DockerHelperImpl) createBuildxBuilderWithK8sDriver(ciContext cicxt.CiContext, dockerConnection, buildxDriverImage string, builderNodes []map[string]string, ciPipelineId, ciWorkflowId int) error { +func (impl *DockerHelperImpl) createBuildxBuilderWithK8sDriver(ciContext cicxt.CiContext, ciRequest *CommonWorkflowRequest, dockerConnection, buildxDriverImage string, builderNodes []map[string]string, ciPipelineId, ciWorkflowId int) error { if len(builderNodes) == 0 { return errors.New("atleast one node is expected for builder with kubernetes driver") } deploymentNames := make([]string, 0) for i := 0; i < len(builderNodes); i++ { nodeOpts := builderNodes[i] - builderCmd, deploymentName := getBuildxK8sDriverCmd(dockerConnection, buildxDriverImage, nodeOpts, ciPipelineId, ciWorkflowId) + builderCmd, deploymentName, err := getBuildxK8sDriverCmd(ciRequest, dockerConnection, buildxDriverImage, nodeOpts, ciPipelineId, ciWorkflowId) + if err != nil { + return err + } deploymentNames = append(deploymentNames, deploymentName) // first node is used as default node, we create builder with --use flag, then we append other nodes if i == 0 { @@ -957,7 +962,7 @@ func (impl *DockerHelperImpl) createBuildxBuilderWithK8sDriver(ciContext cicxt.C fmt.Println(util.DEVTRON, " cmd : ", builderCmd) builderExecCmd := impl.GetCommandToExecute(builderCmd) - err := impl.cmdExecutor.RunCommand(ciContext, builderExecCmd) + err = impl.cmdExecutor.RunCommand(ciContext, builderExecCmd) if err != nil { fmt.Println(util.DEVTRON, " builderCmd : ", builderCmd, " err : ", err, " error : ") return err @@ -1017,7 +1022,7 @@ func (impl *DockerHelperImpl) runCmd(cmd string) (error, *bytes.Buffer) { return err, errBuf } -func getBuildxK8sDriverCmd(dockerConnection, buildxDriverImage string, driverOpts map[string]string, ciPipelineId, ciWorkflowId int) (string, string) { +func getBuildxK8sDriverCmd(ciRequest *CommonWorkflowRequest, dockerConnection, buildxDriverImage string, driverOpts map[string]string, ciPipelineId, ciWorkflowId int) (string, string, error) { buildxCreate := "docker buildx create --buildkitd-flags '--allow-insecure-entitlement network.host --allow-insecure-entitlement security.insecure' --name=%s --driver=kubernetes --node=%s --bootstrap " nodeName := driverOpts["node"] if nodeName == "" { @@ -1029,6 +1034,15 @@ func getBuildxK8sDriverCmd(dockerConnection, buildxDriverImage string, driverOpt buildxCreate += " --platform=%s " buildxCreate = fmt.Sprintf(buildxCreate, platforms) } + // add driver options for app labels and annotations + var err error + if ciRequest.PropagateLabelsInBuildxPod { + driverOpts["driverOptions"], err = getBuildXDriverOptionsWithLabelsAndAnnotations(driverOpts["driverOptions"]) + if err != nil { + return "", "", err + } + } + driverOpts["driverOptions"] = getBuildXDriverOptionsWithImage(buildxDriverImage, driverOpts["driverOptions"]) if len(driverOpts["driverOptions"]) > 0 { buildxCreate += " '--driver-opt=%s' " @@ -1039,7 +1053,7 @@ func getBuildxK8sDriverCmd(dockerConnection, buildxDriverImage string, driverOpt buildkitToml = fmt.Sprintf("--config %s", BuildkitdConfigPath) } buildxCreate = fmt.Sprintf("%s %s", buildxCreate, buildkitToml) - return buildxCreate, nodeName + return buildxCreate, nodeName, nil } func getBuildXDriverOptionsWithImage(buildxDriverImage, driverOptions string) string { @@ -1054,6 +1068,88 @@ func getBuildXDriverOptionsWithImage(buildxDriverImage, driverOptions string) st return driverOptions } +func getBuildXDriverOptionsWithLabelsAndAnnotations(driverOptions string) (string, error) { + // not passing annotation as of now because --driver-opt=annotations is not supported by buildx if contains quotes + labels := make(map[string]string) + + // Read labels from file + labelsPath := utils.DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH + "/" + utils.POD_LABELS + labelsOut, err := readFileAndLogErrors(labelsPath) + if err != nil { + return "", err + } + + // Parse labels + if len(labelsOut) > 0 { + labels = parseKeyValuePairs(string(labelsOut)) + } + + // Combine driver options + driverOptions = getBuildXDriverOptions(utils.POD_LABELS, labels, driverOptions) + + //annotations := make(map[string]string) + //annotationsPath := utils.DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH + "/" + utils.POD_ANNOTATIONS + //annotationsOut, err := readFileAndLogErrors(annotationsPath) + //if err != nil { + // return "", err + //} + //if len(annotationsOut) > 0 { + // annotations = parseKeyValuePairs(string(annotationsOut)) + //} + //driverOptions = getBuildXDriverOptions(utils.POD_ANNOTATIONS, annotations, driverOptions) + + return driverOptions, nil +} + +func readFileAndLogErrors(filePath string) ([]byte, error) { + content, err := os.ReadFile(filePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + log.Println(util.DEVTRON, "file not found at path:", filePath) + return content, nil + } else { + log.Println(util.DEVTRON, "IO error while reading file at path:", filePath, "err:", err) + return nil, err + } + } + return content, nil +} + +func parseKeyValuePairs(input string) map[string]string { + keyValuePairs := make(map[string]string) + lines := strings.Split(input, "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if len(line) > 0 { + kv := strings.SplitN(line, "=", 2) + if len(kv) == 2 { + key := strings.TrimSpace(kv[0]) + value := strings.Trim(strings.TrimSpace(kv[1]), `"`) + keyValuePairs[key] = value + } + } + } + return keyValuePairs +} + +func getBuildXDriverOptions(optionType string, options map[string]string, driverOptions string) string { + if len(options) > 0 { + optionStr := fmt.Sprintf("\"%s=", optionType) + for k, v := range options { + optionStr += fmt.Sprintf("%s=%s,", k, v) + } + optionStr = strings.TrimSuffix(optionStr, ",") + optionStr += "\"" + + if len(driverOptions) > 0 { + driverOptions += fmt.Sprintf(",%s", optionStr) + } else { + driverOptions = optionStr + } + } + return driverOptions +} + func (impl *DockerHelperImpl) StopDocker(ciContext cicxt.CiContext) error { cmd := exec.Command("docker", "ps", "-a", "-q") out, err := cmd.Output() diff --git a/ci-runner/helper/EventHelper.go b/ci-runner/helper/EventHelper.go index 06c8ebfea..c162437ef 100644 --- a/ci-runner/helper/EventHelper.go +++ b/ci-runner/helper/EventHelper.go @@ -162,6 +162,7 @@ type CommonWorkflowRequest struct { ImageScanRetryDelay int `json:"imageScanRetryDelay,omitempty"` ShouldPullDigest bool `json:"shouldPullDigest,omitempty"` EnableSecretMasking bool `json:"enableSecretMasking"` + PropagateLabelsInBuildxPod bool `json:"propagateLabelsInBuildxPod"` // Data from CD Workflow service WorkflowRunnerId int `json:"workflowRunnerId"` CdPipelineId int `json:"cdPipelineId"` @@ -190,6 +191,9 @@ type CommonWorkflowRequest struct { AsyncBuildxCacheExport bool `json:"asyncBuildxCacheExport"` UseDockerApiToGetDigest bool `json:"useDockerApiToGetDigest"` HostUrl string `json:"hostUrl"` + ImageScanningSteps []*ImageScanningSteps `json:"imageScanningSteps,omitempty"` + ExecuteImageScanningVia bean2.ScanExecutionMedium `json:"executeImageScanningVia,omitempty"` + AwsInspectorConfig string `json:"awsInspectorConfig,omitempty"` } func (c *CommonWorkflowRequest) IsPreCdStage() bool { @@ -208,7 +212,6 @@ func (c *CommonWorkflowRequest) GetCdStageType() PipelineType { } return "" } - func (c *CommonWorkflowRequest) GetCloudHelperBaseConfig(blobStorageObjectType string) *util.CloudHelperBaseConfig { return &util.CloudHelperBaseConfig{ StorageModuleConfigured: c.BlobStorageConfigured, @@ -373,6 +376,12 @@ type CiCompleteEvent struct { IsScanEnabled bool `json:"isScanEnabled"` PluginArtifacts *PluginArtifacts `json:"pluginArtifacts"` DockerRegistryId string `json:"dockerRegistryId"` + TargetPlatforms []string `json:"targetPlatforms"` +} + +func (event *CiCompleteEvent) WithTargetPlatforms(targetPlatforms []string) *CiCompleteEvent { + event.TargetPlatforms = targetPlatforms + return event } func (event *CiCompleteEvent) WithMetrics(metrics CIMetrics) *CiCompleteEvent { @@ -764,3 +773,19 @@ func GetImageScanningEvent(ciCdRequest CommonWorkflowRequest) ImageScanningEvent event.PipelineType = stage return event } + +type ImageScanningSteps struct { + Steps []*StepObject `json:"steps"` + ScanToolId int `json:"scanToolId"` +} + +func GetPrePostStageDisplayName(stageName string, stepType StepType) string { + if stepType == STEP_TYPE_PRE { + return fmt.Sprintf("%s (Pre-Build Task)", stageName) + } else if stepType == STEP_TYPE_POST { + return fmt.Sprintf("%s (Post-Build Task)", stageName) + } else { + + return fmt.Sprintf("%s", stageName) + } +} diff --git a/ci-runner/helper/bean/bean.go b/ci-runner/helper/bean/bean.go new file mode 100644 index 000000000..060d7307d --- /dev/null +++ b/ci-runner/helper/bean/bean.go @@ -0,0 +1,7 @@ +package bean + +import "github.com/devtron-labs/common-lib/utils/bean" + +type DockerBuildStageMetadata struct { + TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` +} diff --git a/ci-runner/helper/pluginBean.go b/ci-runner/helper/pluginBean.go index bbf151907..1d7aee0af 100644 --- a/ci-runner/helper/pluginBean.go +++ b/ci-runner/helper/pluginBean.go @@ -39,6 +39,7 @@ const ( STEP_TYPE_REF_PLUGIN StepType = "REF_PLUGIN" STEP_TYPE_PRE StepType = "PRE" STEP_TYPE_POST StepType = "POST" + STEP_TYPE_SCANNING StepType = "SCANNING" ) type StepObject struct { diff --git a/ci-runner/util/Util.go b/ci-runner/util/Util.go index fe65cf76d..9f7fb06ed 100644 --- a/ci-runner/util/Util.go +++ b/ci-runner/util/Util.go @@ -51,7 +51,7 @@ const ( DOCKER_STOP = "Docker Stop" BUILD_ARTIFACT = "Build Artifact" UPLOAD_ARTIFACT = "Uploading Artifact" - PUSH_CACHE = "Pushing Cache" + PUSH_CACHE = "Pushing Cache (Cleanup)" DOCKER_PUSH_AND_EXTRACT_IMAGE_DIGEST = "Docker Push And Extract Image Digest" IMAGE_SCAN = "Image Scanning" SETUP_BUILDX_BUILDER = "Setting Up Buildx Builder" @@ -180,10 +180,11 @@ const ( type StageLogData struct { //eg : 'STAGE_INFO|{"stage":"Resource availability","startTime":"2021-01-01T00:00:00Z"}' - Stage string `json:"stage,omitempty"` - StartTime *time.Time `json:"startTime,omitempty"` - EndTime *time.Time `json:"endTime,omitempty"` - Status Status `json:"status,omitempty"` + Stage string `json:"stage,omitempty"` + StartTime *time.Time `json:"startTime,omitempty"` + EndTime *time.Time `json:"endTime,omitempty"` + Status Status `json:"status,omitempty"` + Metadata interface{} `json:"metadata,omitempty"` } func (stageLogData *StageLogData) withStatus(status Status) *StageLogData { @@ -203,6 +204,11 @@ func (stageLogData *StageLogData) withCurrentEndTime() *StageLogData { return stageLogData } +func (stageLogData *StageLogData) withMetadata(metadata interface{}) *StageLogData { + stageLogData.Metadata = metadata + return stageLogData +} + func (stageLogData *StageLogData) log() { infoLog := fmt.Sprintf("STAGE_INFO|%s\n", stageLogData.string()) log.SetFlags(0) @@ -219,7 +225,14 @@ func (stageLogData *StageLogData) string() string { // it will log info for pre stage execution and post the stage execution // return the error returned by the stageExecutor func func ExecuteWithStageInfoLog(stageName string, stageExecutor func() error) (err error) { - startDockerStageInfo := newStageInfo(stageName).withCurrentStartTime() + return ExecuteWithStageInfoLogWithMetadata(stageName, nil, stageExecutor) +} + +// ExecuteWithStageInfoLogWithMetadata logs the stage info. +// it will log info for pre stage execution and post the stage execution +// return the error returned by the stageExecutor func +func ExecuteWithStageInfoLogWithMetadata(stageName string, metadata interface{}, stageExecutor func() error) (err error) { + startDockerStageInfo := newStageInfo(stageName).withCurrentStartTime().withMetadata(metadata) startDockerStageInfo.log() defer func() { status := success diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go index b18d694ea..79decb1db 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go @@ -62,3 +62,22 @@ func (r *ImageScanEvent) IsImageFromManifest() bool { func (r *ImageScanEvent) IsBuiltImage() bool { return r.SourceType == constants.SourceTypeImage && r.SourceSubType == constants.SourceSubTypeCi } + +type ScanResultPayload struct { + ImageScanEvent *ImageScanEvent + ScanToolId int `json:"scanToolId"` + SourceScanningResult string `json:"sourceScanningResult"` + Sbom string `json:"sbom"` + ImageScanOutput []*ImageScanOutputObject `json:"imageScanOutput"` +} + +type ImageScanOutputObject struct { + TargetName string `json:"targetName"` + Class string `json:"class"` + Type string `json:"type"` + Name string `json:"name"` + Package string `json:"package"` + PackageVersion string `json:"packageVersion"` + FixedInVersion string `json:"fixedInVersion"` + Severity string `json:"severity"` +} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/executionBean.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/executionBean.go new file mode 100644 index 000000000..d1313aba5 --- /dev/null +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/executionBean.go @@ -0,0 +1,16 @@ +package bean + +type ScanExecutionMedium string + +const ( + InHouse ScanExecutionMedium = "in-house" // this contains all methods of hitting image scanner directly via rest or rpc + External ScanExecutionMedium = "external" // if a scan tool is registered via api and execution is via plugin steps +) + +func (e ScanExecutionMedium) IsScanExecutionMediumInHouse() bool { + return e == InHouse +} + +func (e ScanExecutionMedium) IsScanMediumExternal() bool { + return e == External +} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/ci-runner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/ci-runner/vendor/modules.txt b/ci-runner/vendor/modules.txt index 813bc2b33..463aa22ef 100644 --- a/ci-runner/vendor/modules.txt +++ b/ci-runner/vendor/modules.txt @@ -118,7 +118,7 @@ github.com/cespare/xxhash/v2 # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/constants @@ -861,4 +861,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/common-lib/imageScan/bean/bean.go b/common-lib/imageScan/bean/bean.go index b18d694ea..79decb1db 100644 --- a/common-lib/imageScan/bean/bean.go +++ b/common-lib/imageScan/bean/bean.go @@ -62,3 +62,22 @@ func (r *ImageScanEvent) IsImageFromManifest() bool { func (r *ImageScanEvent) IsBuiltImage() bool { return r.SourceType == constants.SourceTypeImage && r.SourceSubType == constants.SourceSubTypeCi } + +type ScanResultPayload struct { + ImageScanEvent *ImageScanEvent + ScanToolId int `json:"scanToolId"` + SourceScanningResult string `json:"sourceScanningResult"` + Sbom string `json:"sbom"` + ImageScanOutput []*ImageScanOutputObject `json:"imageScanOutput"` +} + +type ImageScanOutputObject struct { + TargetName string `json:"targetName"` + Class string `json:"class"` + Type string `json:"type"` + Name string `json:"name"` + Package string `json:"package"` + PackageVersion string `json:"packageVersion"` + FixedInVersion string `json:"fixedInVersion"` + Severity string `json:"severity"` +} diff --git a/common-lib/imageScan/bean/executionBean.go b/common-lib/imageScan/bean/executionBean.go new file mode 100644 index 000000000..d1313aba5 --- /dev/null +++ b/common-lib/imageScan/bean/executionBean.go @@ -0,0 +1,16 @@ +package bean + +type ScanExecutionMedium string + +const ( + InHouse ScanExecutionMedium = "in-house" // this contains all methods of hitting image scanner directly via rest or rpc + External ScanExecutionMedium = "external" // if a scan tool is registered via api and execution is via plugin steps +) + +func (e ScanExecutionMedium) IsScanExecutionMediumInHouse() bool { + return e == InHouse +} + +func (e ScanExecutionMedium) IsScanMediumExternal() bool { + return e == External +} diff --git a/common-lib/utils/CommonUtils.go b/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/common-lib/utils/CommonUtils.go +++ b/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/common-lib/utils/FileUtil.go b/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/common-lib/utils/FileUtil.go +++ b/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/common-lib/utils/bean/bean.go b/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/common-lib/utils/bean/bean.go +++ b/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/common-lib/utils/registry/bean.go b/common-lib/utils/registry/bean.go index 457453268..d6796fe3b 100644 --- a/common-lib/utils/registry/bean.go +++ b/common-lib/utils/registry/bean.go @@ -1,18 +1,28 @@ package registry -type registry string +type Registry string -func (r registry) String() string { +func (r Registry) String() string { return string(r) } const ( - DOCKER_REGISTRY_TYPE_ECR registry = "ecr" - DOCKER_REGISTRY_TYPE_ACR registry = "acr" - DOCKER_REGISTRY_TYPE_DOCKERHUB registry = "docker-hub" - DOCKER_REGISTRY_TYPE_OTHER registry = "other" - REGISTRY_TYPE_ARTIFACT_REGISTRY registry = "artifact-registry" - REGISTRY_TYPE_GCR registry = "gcr" + DOCKER_REGISTRY_TYPE_ECR Registry = "ecr" + DOCKER_REGISTRY_TYPE_ACR Registry = "acr" + DOCKER_REGISTRY_TYPE_DOCKERHUB Registry = "docker-hub" + DOCKER_REGISTRY_TYPE_OTHER Registry = "other" + REGISTRY_TYPE_ARTIFACT_REGISTRY Registry = "artifact-registry" + REGISTRY_TYPE_GCR Registry = "gcr" ) const JSON_KEY_USERNAME = "_json_key" + +type RegistryCredential struct { + RegistryType Registry `json:"registryType"` + RegistryURL string `json:"registryURL"` + Username string `json:"username"` + Password string `json:"password"` + AWSAccessKeyId string `json:"awsAccessKeyId,omitempty"` + AWSSecretAccessKey string `json:"awsSecretAccessKey,omitempty"` + AWSRegion string `json:"awsRegion,omitempty"` +} diff --git a/common-lib/utils/registry/extractCredentials.go b/common-lib/utils/registry/extractCredentials.go new file mode 100644 index 000000000..b0017670f --- /dev/null +++ b/common-lib/utils/registry/extractCredentials.go @@ -0,0 +1,72 @@ +package registry + +import ( + "encoding/base64" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "strings" +) + +func ExtractCredentialsForRegistry(registryCredential *RegistryCredential) (string, string, error) { + username := registryCredential.Username + pwd := registryCredential.Password + if (registryCredential.RegistryType == REGISTRY_TYPE_GCR || registryCredential.RegistryType == REGISTRY_TYPE_ARTIFACT_REGISTRY) && username == JSON_KEY_USERNAME { + if strings.HasPrefix(pwd, "'") { + pwd = pwd[1:] + } + if strings.HasSuffix(pwd, "'") { + pwd = pwd[:len(pwd)-1] + } + } + if registryCredential.RegistryType == DOCKER_REGISTRY_TYPE_ECR { + accessKey, secretKey := registryCredential.AWSAccessKeyId, registryCredential.AWSSecretAccessKey + var creds *credentials.Credentials + + if len(registryCredential.AWSAccessKeyId) == 0 || len(registryCredential.AWSSecretAccessKey) == 0 { + sess, err := session.NewSession(&aws.Config{ + Region: ®istryCredential.AWSRegion, + }) + if err != nil { + fmt.Printf("Error in creating AWS client", "err", err) + return "", "", err + } + creds = ec2rolecreds.NewCredentials(sess) + } else { + creds = credentials.NewStaticCredentials(accessKey, secretKey, "") + } + sess, err := session.NewSession(&aws.Config{ + Region: ®istryCredential.AWSRegion, + Credentials: creds, + }) + if err != nil { + fmt.Println("Error in creating AWS client session", "err", err) + return "", "", err + } + svc := ecr.New(sess) + input := &ecr.GetAuthorizationTokenInput{} + authData, err := svc.GetAuthorizationToken(input) + if err != nil { + fmt.Println("Error fetching authData", "err", err) + return "", "", err + } + // decode token + token := authData.AuthorizationData[0].AuthorizationToken + decodedToken, err := base64.StdEncoding.DecodeString(*token) + if err != nil { + fmt.Println("Error in decoding auth token", "err", err) + return "", "", err + } + credsSlice := strings.Split(string(decodedToken), ":") + if len(credsSlice) < 2 { + fmt.Println("Error in decoding auth token", "err", err) + return "", "", fmt.Errorf("error in decoding auth token for docker Registry") + } + username = credsSlice[0] + pwd = credsSlice[1] + } + return username, pwd, nil +} diff --git a/git-sensor/Dockerfile b/git-sensor/Dockerfile index 9ddb1b7b2..5204dd0bb 100644 --- a/git-sensor/Dockerfile +++ b/git-sensor/Dockerfile @@ -1,23 +1,27 @@ FROM golang:1.21-alpine3.17 AS build-env -RUN apk add --no-cache git gcc musl-dev -RUN apk add --update make -RUN go install github.com/google/wire/cmd/wire@latest +RUN apk add --no-cache git gcc musl-dev && \ + apk add --update make && \ + go install github.com/google/wire/cmd/wire@latest + WORKDIR /go/src/github.com/devtron-labs/git-sensor + ADD . /go/src/github.com/devtron-labs/git-sensor/ + RUN GOOS=linux make FROM alpine:3.17 + COPY ./git-ask-pass.sh /git-ask-pass.sh -RUN chmod +x /git-ask-pass.sh -RUN apk add --no-cache ca-certificates -RUN apk add git --no-cache -RUN apk add openssh --no-cache -COPY --from=build-env /go/src/github.com/devtron-labs/git-sensor/git-sensor . -COPY --from=build-env /go/src/github.com/devtron-labs/git-sensor/scripts/ . - -RUN adduser -D devtron -RUN chown -R devtron:devtron ./git-sensor + +RUN chmod +x /git-ask-pass.sh && \ + apk add --no-cache ca-certificates && \ + apk add git --no-cache && \ + apk add openssh --no-cache && \ + adduser -D devtron + +COPY --chown=devtron:devtron --from=build-env /go/src/github.com/devtron-labs/git-sensor/scripts/ /go/src/github.com/devtron-labs/git-sensor/git-sensor . + USER devtron CMD ["./git-sensor"] diff --git a/git-sensor/go.mod b/git-sensor/go.mod index e80110fbb..79b6284c5 100644 --- a/git-sensor/go.mod +++ b/git-sensor/go.mod @@ -4,7 +4,7 @@ go 1.21 toolchain go1.22.4 -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f require ( github.com/caarlos0/env v3.5.0+incompatible diff --git a/git-sensor/go.sum b/git-sensor/go.sum index 1c50ffcfc..c25cf547b 100644 --- a/git-sensor/go.sum +++ b/git-sensor/go.sum @@ -29,8 +29,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/devtron-labs/protos v0.0.3-0.20240809072909-83171af34169 h1:9OMZv0/fOWKK9s9BLTofFL/BO79TdyvC1Sc1HsC4esQ= github.com/devtron-labs/protos v0.0.3-0.20240809072909-83171af34169/go.mod h1:1TqULGlTey+VNhAu/ag7NJuUvByJemkqodsc9L5PHJk= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/git-sensor/internals/middleware/instrument.go b/git-sensor/internals/middleware/instrument.go index 90ee82576..86b3aa33c 100644 --- a/git-sensor/internals/middleware/instrument.go +++ b/git-sensor/internals/middleware/instrument.go @@ -110,6 +110,14 @@ var GitMaterialPollCounter = promauto.NewCounterVec( }, []string{}) +var GitFetchTimeoutCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "git_fetch_timeout", + Help: "no of timeouts while running git fetch command", + ConstLabels: constLabels, + }, + []string{}) + var PanicCounter = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "panic", diff --git a/git-sensor/pkg/git/GitBaseManager.go b/git-sensor/pkg/git/GitBaseManager.go index 93d276368..d9ffe4b7a 100644 --- a/git-sensor/pkg/git/GitBaseManager.go +++ b/git-sensor/pkg/git/GitBaseManager.go @@ -23,6 +23,7 @@ import ( "fmt" commonLibGitManager "github.com/devtron-labs/common-lib/git-manager" "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/middleware" "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/util" "go.uber.org/zap" @@ -111,7 +112,7 @@ func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (response, errMsg string, err error) { impl.logger.Debugw("git fetch ", "location", rootDir) - cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") defer cancel() tlsPathInfo, err := commonLibGitManager.CreateFilesForTlsData(commonLibGitManager.BuildTlsData(gitCtx.TLSKey, gitCtx.TLSCertificate, gitCtx.CACert, gitCtx.TLSVerificationEnabled), TLS_FILES_DIR) if err != nil { @@ -119,22 +120,22 @@ func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (respon impl.logger.Errorw("error encountered in createFilesForTlsData", "err", err) } defer commonLibGitManager.DeleteTlsFiles(tlsPathInfo) - output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password, tlsPathInfo) + output, errMsg, err := impl.runCommandWithCred(cmd, newGitCtx, gitCtx.Username, gitCtx.Password, tlsPathInfo) if strings.Contains(output, util.LOCK_REF_MESSAGE) { impl.logger.Info("error in fetch, pruning local refs and retrying", "rootDir", rootDir) // running git remote prune origin and retrying fetch. gitHub issue - https://github.com/devtron-labs/devtron/issues/4605 - pruneCmd, pruneCmdCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "prune", "origin") - pruneOutput, pruneMsg, pruneErr := impl.runCommandWithCred(pruneCmd, gitCtx.Username, gitCtx.Password, tlsPathInfo) + pruneCmd, newGitCtx, pruneCmdCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "prune", "origin") + pruneOutput, pruneMsg, pruneErr := impl.runCommandWithCred(pruneCmd, newGitCtx, gitCtx.Username, gitCtx.Password, tlsPathInfo) defer pruneCmdCancel() if pruneErr != nil { impl.logger.Errorw("error in pruning local refs that do not exist at remote") return pruneOutput, pruneMsg, pruneErr } - retryFetchCmd, retryFetchCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") + retryFetchCmd, newGitCtx, retryFetchCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") defer retryFetchCancel() - output, errMsg, err = impl.runCommandWithCred(retryFetchCmd, gitCtx.Username, gitCtx.Password, tlsPathInfo) + output, errMsg, err = impl.runCommandWithCred(retryFetchCmd, newGitCtx, gitCtx.Username, gitCtx.Password, tlsPathInfo) } impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err @@ -142,9 +143,9 @@ func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (respon func (impl *GitManagerBaseImpl) Checkout(gitCtx GitContext, rootDir, branch string) (response, errMsg string, err error) { impl.logger.Debugw("git checkout ", "location", rootDir) - cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force") + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force") defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.runCommand(newGitCtx, cmd) impl.logger.Debugw("checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } @@ -159,9 +160,9 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st } cmdArgs := []string{"-C", rootDir, "log", from + "..." + toCommitHash, "--date=iso-strict", GITFORMAT} impl.logger.Debugw("git", cmdArgs) - cmd, cancel := impl.createCmdWithContext(gitCtx, "git", cmdArgs...) + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitCtx, "git", cmdArgs...) defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.runCommand(newGitCtx, cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -174,7 +175,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st return commits, nil } -func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string, tlsPathInfo *commonLibGitManager.TlsPathInfo) (response, errMsg string, err error) { +func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, gitCtx GitContext, userName, password string, tlsPathInfo *commonLibGitManager.TlsPathInfo) (response, errMsg string, err error) { cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_ASKPASS=%s", util.GIT_ASK_PASS), fmt.Sprintf("GIT_USERNAME=%s", userName), @@ -190,10 +191,10 @@ func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, pass cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_SSL_CAINFO=%s", tlsPathInfo.CaCertPath)) } } - return impl.runCommand(cmd) + return impl.runCommand(gitCtx, cmd) } -func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg string, err error) { +func (impl *GitManagerBaseImpl) runCommand(gitCtx GitContext, cmd *exec.Cmd) (response, errMsg string, err error) { cmd.Env = append(cmd.Env, "HOME=/dev/null") outBytes, err := cmd.CombinedOutput() output := string(outBytes) @@ -201,6 +202,13 @@ func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg stri if err != nil { impl.logger.Errorw("error in git cli operation", "msg", string(outBytes), "err", err) errMsg = output + if errors.Is(gitCtx.Err(), context.DeadlineExceeded) { + errMsg = "command timed out" + impl.logger.Errorw("command timed out", "cmd", cmd.String()) + // prometheus event count for timeout + middleware.GitFetchTimeoutCounter.WithLabelValues().Inc() + return output, errMsg, err + } exErr, ok := err.(*exec.ExitError) if !ok { return output, errMsg, err @@ -222,9 +230,9 @@ func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg stri func (impl *GitManagerBaseImpl) ConfigureSshCommand(gitCtx GitContext, rootDir string, sshPrivateKeyPath string) (response, errMsg string, err error) { impl.logger.Debugw("configuring ssh command on ", "location", rootDir) coreSshCommand := fmt.Sprintf("ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", sshPrivateKeyPath) - cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand) + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand) defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.runCommand(newGitCtx, cmd) impl.logger.Debugw("configure ssh command output ", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } @@ -318,7 +326,7 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommitsNameOnly(gitCtx GitCo newHash = oldHash oldHash = oldHash + "^" } - cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--name-only", oldHash, newHash) + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--name-only", oldHash, newHash) tlsPathInfo, err := commonLibGitManager.CreateFilesForTlsData(commonLibGitManager.BuildTlsData(gitCtx.TLSKey, gitCtx.TLSCertificate, gitCtx.CACert, gitCtx.TLSVerificationEnabled), TLS_FILES_DIR) if err != nil { @@ -331,7 +339,7 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommitsNameOnly(gitCtx GitCo commonLibGitManager.DeleteTlsFiles(tlsPathInfo) }() - output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password, tlsPathInfo) + output, errMsg, err := impl.runCommandWithCred(cmd, newGitCtx, gitCtx.Username, gitCtx.Password, tlsPathInfo) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil || len(errMsg) > 0 { impl.logger.Errorw("error in fetching fileStat diff btw commits: ", "oldHash", oldHash, "newHash", newHash, "checkoutPath", rootDir, "errorMsg", errMsg, "err", err) @@ -347,7 +355,7 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommitsWithNumstat(gitCtx Gi newHash = oldHash oldHash = oldHash + "^" } - cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash) + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash) defer cancel() tlsPathInfo, err := commonLibGitManager.CreateFilesForTlsData(commonLibGitManager.BuildTlsData(gitCtx.TLSKey, gitCtx.TLSCertificate, gitCtx.CACert, gitCtx.TLSVerificationEnabled), TLS_FILES_DIR) if err != nil { @@ -355,7 +363,7 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommitsWithNumstat(gitCtx Gi impl.logger.Errorw("error encountered in createFilesForTlsData", "err", err) } defer commonLibGitManager.DeleteTlsFiles(tlsPathInfo) - output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password, tlsPathInfo) + output, errMsg, err := impl.runCommandWithCred(cmd, newGitCtx, gitCtx.Username, gitCtx.Password, tlsPathInfo) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil || len(errMsg) > 0 { impl.logger.Errorw("error in fetching fileStat diff btw commits: ", "oldHash", oldHash, "newHash", newHash, "checkoutPath", rootDir, "errorMsg", errMsg, "err", err) @@ -364,20 +372,21 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommitsWithNumstat(gitCtx Gi return processFileStatOutputWithNumstat(output) } -func (impl *GitManagerBaseImpl) createCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) { +func (impl *GitManagerBaseImpl) createCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, GitContext, context.CancelFunc) { newCtx := ctx cancel := func() {} //TODO: how to make it generic, currently works because the // git command is placed at index 2 for current implementations - timeout := impl.getCommandTimeout(arg[2]) - + timeout := 0 + if len(arg) > 2 { + timeout = impl.getCommandTimeout(arg[2]) + } if timeout > 0 { - newCtx, cancel = ctx.WithTimeout(timeout) //context.WithTimeout(ctx.Context, timeout*time.Second) } cmd := exec.CommandContext(newCtx, name, arg...) - return cmd, cancel + return cmd, newCtx, cancel } func (impl *GitManagerBaseImpl) getCommandTimeout(command string) int { @@ -389,7 +398,7 @@ func (impl *GitManagerBaseImpl) getCommandTimeout(command string) int { } func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name string, arg ...string) (response, errMsg string, err error) { - cmd, cancel := impl.createCmdWithContext(gitContext, name, arg...) + cmd, newGitCtx, cancel := impl.createCmdWithContext(gitContext, name, arg...) defer cancel() tlsPathInfo, err := commonLibGitManager.CreateFilesForTlsData(commonLibGitManager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), TLS_FILES_DIR) if err != nil { @@ -397,6 +406,6 @@ func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name impl.logger.Errorw("error encountered in createFilesForTlsData", "err", err) } defer commonLibGitManager.DeleteTlsFiles(tlsPathInfo) - output, errMsg, err := impl.runCommandWithCred(cmd, gitContext.Username, gitContext.Password, tlsPathInfo) + output, errMsg, err := impl.runCommandWithCred(cmd, newGitCtx, gitContext.Username, gitContext.Password, tlsPathInfo) return output, errMsg, err } diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/git-sensor/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/git-sensor/vendor/modules.txt b/git-sensor/vendor/modules.txt index f3c0e9677..81623963f 100644 --- a/git-sensor/vendor/modules.txt +++ b/git-sensor/vendor/modules.txt @@ -62,7 +62,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -482,4 +482,4 @@ gopkg.in/yaml.v3 # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/image-scanner/Dockerfile b/image-scanner/Dockerfile index 2d0e1fa3b..66c29078f 100644 --- a/image-scanner/Dockerfile +++ b/image-scanner/Dockerfile @@ -1,20 +1,29 @@ FROM golang:1.21-alpine3.18 AS build-env -RUN apk add --no-cache git gcc musl-dev -RUN apk add --update make -RUN go install github.com/google/wire/cmd/wire@latest + +RUN apk add --no-cache git gcc musl-dev && \ + apk add --update make && \ + go install github.com/google/wire/cmd/wire@latest + WORKDIR /go/src/github.com/devtron-labs/image-scanner + ADD . /go/src/github.com/devtron-labs/image-scanner + RUN GOOS=linux make FROM alpine:3.17 + COPY --from=aquasec/trivy:0.46.1 /usr/local/bin/trivy /usr/local/bin/trivy -RUN apk add --no-cache ca-certificates -RUN mkdir -p /security -RUN adduser -D devtron -COPY --from=build-env /go/src/github.com/devtron-labs/image-scanner/image-scanner . -RUN chown -R devtron:devtron ./image-scanner -RUN chmod +x ./image-scanner -RUN chown -R devtron:devtron ./security -RUN chmod +x ./security + +RUN apk add --no-cache ca-certificates && \ + mkdir -p /security && \ + adduser -D devtron + +COPY --chown=devtron:devtron --from=build-env /go/src/github.com/devtron-labs/image-scanner/image-scanner . + +RUN chmod +x ./image-scanner && \ + chown -R devtron:devtron ./security && \ + chmod +x ./security + USER devtron + CMD ["./image-scanner"] diff --git a/image-scanner/api/RestHandler.go b/image-scanner/api/RestHandler.go index 045dd3e76..c2dc4771c 100644 --- a/image-scanner/api/RestHandler.go +++ b/image-scanner/api/RestHandler.go @@ -18,12 +18,14 @@ package api import ( "encoding/json" + "fmt" bean2 "github.com/devtron-labs/common-lib/imageScan/bean" "github.com/devtron-labs/image-scanner/common" "github.com/devtron-labs/image-scanner/pkg/clairService" "github.com/devtron-labs/image-scanner/pkg/grafeasService" "github.com/devtron-labs/image-scanner/pkg/klarService" "github.com/devtron-labs/image-scanner/pkg/security" + "github.com/devtron-labs/image-scanner/pkg/sql/adaptor" "github.com/devtron-labs/image-scanner/pkg/sql/bean" "github.com/devtron-labs/image-scanner/pkg/sql/repository" "github.com/devtron-labs/image-scanner/pkg/user" @@ -36,6 +38,7 @@ import ( type RestHandler interface { ScanForVulnerability(w http.ResponseWriter, r *http.Request) ScanForVulnerabilityEvent(scanConfig *bean2.ImageScanEvent) (*common.ScanEventResponse, error) + RegisterAndSaveScannedResult(w http.ResponseWriter, r *http.Request) } func NewRestHandlerImpl(logger *zap.SugaredLogger, @@ -119,14 +122,9 @@ func (impl *RestHandlerImpl) ScanForVulnerabilityEvent(scanConfig *bean2.ImageSc impl.Logger.Errorw("error in marshalling scanEvent", "event", scanConfig, "err", err) return nil, err } - executionHistoryModel := &repository.ImageScanExecutionHistory{ - Image: scanConfig.Image, - ImageHash: scanConfig.ImageDigest, - ExecutionTime: time.Now(), - ExecutedBy: scanConfig.UserId, - SourceMetadataJson: string(scanEventJson), - } - executionHistory, executionHistoryDirPath, err := impl.ImageScanService.RegisterScanExecutionHistoryAndState(executionHistoryModel, tool) + executionHistoryModel := adaptor.GetImageScanExecutionHistory(scanConfig, scanEventJson, time.Now()) + + executionHistory, executionHistoryDirPath, err := impl.ImageScanService.RegisterScanExecutionHistoryAndState(executionHistoryModel, tool.Id) if err != nil { impl.Logger.Errorw("service err, RegisterScanExecutionHistoryAndState", "err", err) return nil, err @@ -166,12 +164,65 @@ func (impl *RestHandlerImpl) ScanImageAsPerTool(scanConfig *bean2.ImageScanEvent impl.Logger.Errorw("err in process msg", "err", err) return nil, err } - } else { + } else if tool.Name == bean.ScannerTypeTrivy && tool.Version == bean.ScanToolVersion1 { err = impl.ImageScanService.ScanImage(scanConfig, tool, executionHistory, executionHistoryDirPath) if err != nil { impl.Logger.Errorw("err in process msg", "err", err) return nil, err } + } else { + err = fmt.Errorf("no tool found for scanning") + impl.Logger.Errorw("err in process msg", "err", err) + return nil, err } return result, nil } + +func (impl *RestHandlerImpl) RegisterAndSaveScannedResult(w http.ResponseWriter, r *http.Request) { + decoder := json.NewDecoder(r.Body) + var scanResultPayload bean2.ScanResultPayload + err := decoder.Decode(&scanResultPayload) + if err != nil { + impl.Logger.Errorw("error in decoding scanResultPayload", "error", err) + WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + err = ValidateScanResultPayload(&scanResultPayload) + if err != nil { + impl.Logger.Errorw("validation failed", "error", err) + WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + impl.Logger.Debugw("register and save scan result payload", "saveScanResultPayload", scanResultPayload) + _, err = impl.ImageScanService.RegisterAndSaveScannedResult(&scanResultPayload) + if err != nil { + impl.Logger.Errorw("service err, RegisterAndSaveScannedResult", "err", err) + WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + WriteJsonResp(w, nil, nil, http.StatusOK) +} + +func ValidateScanResultPayload(scanResultPayload *bean2.ScanResultPayload) error { + if scanResultPayload.ScanToolId == 0 { + return fmt.Errorf("scan tool id not found: required") + } + if scanResultPayload.ImageScanEvent == nil { + return fmt.Errorf("image and digest not found: required") + } + if scanResultPayload.ImageScanEvent != nil && len(scanResultPayload.ImageScanEvent.Image) == 0 { + return fmt.Errorf("image not found: required") + } + if scanResultPayload.ImageScanEvent != nil && len(scanResultPayload.ImageScanEvent.ImageDigest) == 0 { + return fmt.Errorf("image digest not found: required") + } + if len(scanResultPayload.Sbom) == 0 { + return fmt.Errorf("sbom not found: required") + } + if len(scanResultPayload.SourceScanningResult) == 0 { + return fmt.Errorf("source Scanning result not found: required") + } + return nil +} diff --git a/image-scanner/api/Router.go b/image-scanner/api/Router.go index b311f51b7..55fb94bbb 100644 --- a/image-scanner/api/Router.go +++ b/image-scanner/api/Router.go @@ -57,4 +57,6 @@ func (r Router) Init() { }) r.Router.Path("/scanner/image").HandlerFunc(r.restHandler.ScanForVulnerability).Methods("POST") + r.Router.Path("/scanner/save-result").HandlerFunc(r.restHandler.RegisterAndSaveScannedResult).Methods("POST") + } diff --git a/image-scanner/go.mod b/image-scanner/go.mod index a9d06b779..e3758cffa 100644 --- a/image-scanner/go.mod +++ b/image-scanner/go.mod @@ -74,4 +74,4 @@ require ( mellium.im/sasl v0.3.2 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/image-scanner/go.sum b/image-scanner/go.sum index 8bdee1597..3765fe5b8 100644 --- a/image-scanner/go.sum +++ b/image-scanner/go.sum @@ -280,8 +280,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= diff --git a/image-scanner/pkg/security/ImageScanService.go b/image-scanner/pkg/security/ImageScanService.go index bff7f78c7..51c9dac08 100644 --- a/image-scanner/pkg/security/ImageScanService.go +++ b/image-scanner/pkg/security/ImageScanService.go @@ -31,6 +31,7 @@ import ( httpUtil "github.com/devtron-labs/image-scanner/internals/step-lib/util/http-util" "github.com/devtron-labs/image-scanner/internals/util" "github.com/devtron-labs/image-scanner/pkg/helper" + "github.com/devtron-labs/image-scanner/pkg/sql/adaptor" "github.com/devtron-labs/image-scanner/pkg/sql/bean" "github.com/devtron-labs/image-scanner/pkg/sql/repository" "github.com/go-pg/pg" @@ -58,11 +59,12 @@ type ImageScanService interface { CreateFolderForOutputData(executionHistoryModelId int) string HandleProgressingScans() GetActiveTool() (*repository.ScanToolMetadata, error) - RegisterScanExecutionHistoryAndState(executionHistoryModel *repository.ImageScanExecutionHistory, tool *repository.ScanToolMetadata) (*repository.ImageScanExecutionHistory, string, error) + RegisterScanExecutionHistoryAndState(executionHistoryModel *repository.ImageScanExecutionHistory, toolId int) (*repository.ImageScanExecutionHistory, string, error) GetImageScanRenderDto(registryId string, scanEvent *bean2.ImageScanEvent) (*common.ImageScanRenderDto, error) GetImageToBeScannedAndFetchCliEnv(scanEvent *bean2.ImageScanEvent) (string, error) FetchProxyUrl(scanEvent *bean2.ImageScanEvent) (string, []name.Option, error) - SaveCvesAndImageScanExecutionResults(vulnerabilities []*bean.ImageScanOutputObject, executionHistoryId int, toolId int, userId int32) error + SaveCvesAndImageScanExecutionResults(vulnerabilities []*bean2.ImageScanOutputObject, executionHistoryId int, toolId int, userId int32) error + RegisterAndSaveScannedResult(scanResultPayload *bean2.ScanResultPayload) (int, error) } type ImageScanServiceImpl struct { @@ -113,6 +115,22 @@ func NewImageScanServiceImpl(logger *zap.SugaredLogger, scanHistoryRepository re return imageScanService } +func GetImageScannerConfig() (*ImageScanConfig, error) { + scannerConfig := &ImageScanConfig{} + err := env.Parse(scannerConfig) + if err != nil { + return nil, errors.New(fmt.Sprintf("could not get scanner config from environment :%v", err)) + } + return scannerConfig, err +} + +type ImageScanConfig struct { + ScannerType string `env:"SCANNER_TYPE" envDefault:""` + ScanTryCount int `env:"IMAGE_SCAN_TRY_COUNT" envDefault:"1"` + ScanImageTimeout int `env:"IMAGE_SCAN_TIMEOUT" envDefault:"10"` // Time is considered in minutes + ScanImageAsyncTimeout int `env:"IMAGE_SCAN_ASYNC_TIMEOUT" envDefault:"3"` // Time is considered in minutes +} + func (impl *ImageScanServiceImpl) GetImageToBeScannedAndFetchCliEnv(scanEvent *bean2.ImageScanEvent) (string, error) { impl.CliCommandEnv = append(os.Environ(), impl.CliCommandEnv...) return scanEvent.Image, nil @@ -252,23 +270,42 @@ func (impl *ImageScanServiceImpl) CreateFolderForOutputData(executionHistoryMode return executionHistoryDirPath } -func (impl *ImageScanServiceImpl) RegisterScanExecutionHistoryAndState(executionHistoryModel *repository.ImageScanExecutionHistory, - tool *repository.ScanToolMetadata) (*repository.ImageScanExecutionHistory, string, error) { - executionHistoryDirPath := "" +func (impl *ImageScanServiceImpl) saveImageScanExecutionHistoryAndState(executionHistoryModel *repository.ImageScanExecutionHistory, toolId int) error { //creating execution history tx, err := impl.ScanHistoryRepository.GetConnection().Begin() if err != nil { impl.Logger.Errorw("error in initiating db transaction", "err", err) - return nil, executionHistoryDirPath, err + return err } // Rollback tx on error. defer tx.Rollback() err = impl.ScanHistoryRepository.Save(tx, executionHistoryModel) if err != nil { impl.Logger.Errorw("Failed to save executionHistory", "model", executionHistoryModel, "err", err) - return nil, executionHistoryDirPath, err + return err + } + executionHistoryMappingModel := adaptor.GetScanToolExecutionHistoryMapping(executionHistoryModel, bean.ScanExecutionProcessStateCompleted, "", toolId) + err = impl.ScanToolExecutionHistoryMappingRepository.Save(tx, executionHistoryMappingModel) + if err != nil { + impl.Logger.Errorw("Failed to save executionHistoryMappingModel", "err", err) + return err + } + err = tx.Commit() + if err != nil { + impl.Logger.Errorw("error in committing transaction", "err", err) + return err } + return nil +} +func (impl *ImageScanServiceImpl) RegisterScanExecutionHistoryAndState(executionHistoryModel *repository.ImageScanExecutionHistory, + toolId int) (*repository.ImageScanExecutionHistory, string, error) { + executionHistoryDirPath := "" + err := impl.saveImageScanExecutionHistoryAndState(executionHistoryModel, toolId) + if err != nil { + impl.Logger.Errorw("error in saving image scan exec history and it's state", "executionHistoryModel", executionHistoryModel, "err", err) + return nil, executionHistoryDirPath, err + } // creating folder for storing all details if not exist isExist, err := helper.DoesFileExist(bean.ScanOutputDirectory) if err != nil { @@ -278,7 +315,7 @@ func (impl *ImageScanServiceImpl) RegisterScanExecutionHistoryAndState(execution if !isExist { err = os.Mkdir(bean.ScanOutputDirectory, commonUtil.DefaultFileCreatePermission) if err != nil && !os.IsExist(err) { - impl.Logger.Errorw("error in creating Output directory", "toolId", tool.Id, "executionHistoryDir", executionHistoryDirPath, "err", err) + impl.Logger.Errorw("error in creating Output directory", "toolId", toolId, "executionHistoryDir", executionHistoryDirPath, "err", err) return nil, executionHistoryDirPath, err } } @@ -289,28 +326,6 @@ func (impl *ImageScanServiceImpl) RegisterScanExecutionHistoryAndState(execution impl.Logger.Errorw("error in creating executionHistory directory", "executionHistoryId", executionHistoryModel.Id, "err", err) return nil, executionHistoryDirPath, err } - executionHistoryMappingModel := &repository.ScanToolExecutionHistoryMapping{ - ImageScanExecutionHistoryId: executionHistoryModel.Id, - ScanToolId: tool.Id, - ExecutionStartTime: executionHistoryModel.ExecutionTime, - State: bean.ScanExecutionProcessStateRunning, - AuditLog: repository.AuditLog{ - CreatedOn: executionHistoryModel.ExecutionTime, - CreatedBy: int32(executionHistoryModel.ExecutedBy), - UpdatedOn: executionHistoryModel.ExecutionTime, - UpdatedBy: int32(executionHistoryModel.ExecutedBy), - }, - } - err = impl.ScanToolExecutionHistoryMappingRepository.Save(tx, executionHistoryMappingModel) - if err != nil { - impl.Logger.Errorw("Failed to save executionHistoryMappingModel", "err", err) - return nil, executionHistoryDirPath, err - } - err = tx.Commit() - if err != nil { - impl.Logger.Errorw("error in committing transaction", "err", err) - return nil, executionHistoryDirPath, err - } return executionHistoryModel, executionHistoryDirPath, nil } @@ -455,9 +470,9 @@ func (impl *ImageScanServiceImpl) ProcessScanStep(step repository.ScanToolStep, return output, nil } -func (impl *ImageScanServiceImpl) SaveCvesAndImageScanExecutionResults(vulnerabilities []*bean.ImageScanOutputObject, executionHistoryId int, toolId int, userId int32) error { +func (impl *ImageScanServiceImpl) SaveCvesAndImageScanExecutionResults(vulnerabilities []*bean2.ImageScanOutputObject, executionHistoryId int, toolId int, userId int32) error { cvesToBeSaved := make([]*repository.CveStore, 0, len(vulnerabilities)) - uniqueVulnerabilityMap := make(map[string]*bean.ImageScanOutputObject) + uniqueVulnerabilityMap := make(map[string]*bean2.ImageScanOutputObject) allCvesNames := make([]string, 0, len(vulnerabilities)) for _, vul := range vulnerabilities { if _, ok := uniqueVulnerabilityMap[vul.Name]; !ok { @@ -539,7 +554,7 @@ func (impl *ImageScanServiceImpl) SaveCvesAndImageScanExecutionResults(vulnerabi } func (impl *ImageScanServiceImpl) ConvertEndStepOutputAndSaveVulnerabilities(stepOutput []byte, executionHistoryId int, tool repository.ScanToolMetadata, step repository.ScanToolStep, userId int32) error { - var vulnerabilities []*bean.ImageScanOutputObject + var vulnerabilities []*bean2.ImageScanOutputObject var err error if isV1Template(tool.ResultDescriptorTemplate) { // result descriptor template is go template, go with v1 logic vulnerabilities, err = impl.getImageScanOutputObjectsV1(stepOutput, tool.ResultDescriptorTemplate) @@ -575,7 +590,7 @@ func isValidGoTemplate(templateStr string) bool { return err == nil } -func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV1(stepOutput []byte, resultDescriptorTemplate string) ([]*bean.ImageScanOutputObject, error) { +func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV1(stepOutput []byte, resultDescriptorTemplate string) ([]*bean2.ImageScanOutputObject, error) { //rendering image descriptor template with output json to get vulnerabilities updated renderedTemplate, err := commonUtil.ParseJsonTemplate(resultDescriptorTemplate, stepOutput) if err != nil { @@ -583,7 +598,7 @@ func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV1(stepOutput []byte, return nil, err } renderedTemplate = common.RemoveTrailingComma(renderedTemplate) - var vulnerabilities []*bean.ImageScanOutputObject + var vulnerabilities []*bean2.ImageScanOutputObject err = json.Unmarshal([]byte(renderedTemplate), &vulnerabilities) if err != nil { impl.Logger.Errorw("error in unmarshalling rendered template", "err", err) @@ -592,8 +607,8 @@ func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV1(stepOutput []byte, return vulnerabilities, nil } -func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV2(stepOutput []byte, resultDescriptorTemplate string) ([]*bean.ImageScanOutputObject, error) { - var vulnerabilities []*bean.ImageScanOutputObject +func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV2(stepOutput []byte, resultDescriptorTemplate string) ([]*bean2.ImageScanOutputObject, error) { + var vulnerabilities []*bean2.ImageScanOutputObject var mappings []map[string]interface{} err := json.Unmarshal([]byte(resultDescriptorTemplate), &mappings) if err != nil { @@ -613,7 +628,7 @@ func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV2(stepOutput []byte, if nestedValue.Get(vulnerabilitiesPath).IsArray() { nestedValue.Get(vulnerabilitiesPath).ForEach(func(_, vul gjson.Result) bool { - vulnerability := &bean.ImageScanOutputObject{ + vulnerability := &bean2.ImageScanOutputObject{ Name: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyName].(string)).String(), Package: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyPackage].(string)).String(), PackageVersion: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyPackageVersion].(string)).String(), @@ -1008,18 +1023,22 @@ func (impl *ImageScanServiceImpl) FetchProxyUrl(scanEvent *bean2.ImageScanEvent) return "", []name.Option{}, nil } -func GetImageScannerConfig() (*ImageScanConfig, error) { - scannerConfig := &ImageScanConfig{} - err := env.Parse(scannerConfig) +func (impl *ImageScanServiceImpl) RegisterAndSaveScannedResult(scanResultPayload *bean2.ScanResultPayload) (int, error) { + scanEventJson, err := json.Marshal(scanResultPayload.ImageScanEvent) if err != nil { - return nil, errors.New(fmt.Sprintf("could not get scanner config from environment :%v", err)) + impl.Logger.Errorw("error in marshalling scanEvent", "imageScanEvent", scanResultPayload.ImageScanEvent, "err", err) + return 0, err } - return scannerConfig, err -} - -type ImageScanConfig struct { - ScannerType string `env:"SCANNER_TYPE" envDefault:""` - ScanTryCount int `env:"IMAGE_SCAN_TRY_COUNT" envDefault:"1"` - ScanImageTimeout int `env:"IMAGE_SCAN_TIMEOUT" envDefault:"10"` // Time is considered in minutes - ScanImageAsyncTimeout int `env:"IMAGE_SCAN_ASYNC_TIMEOUT" envDefault:"3"` // Time is considered in minutes + executionHistoryModel := adaptor.GetImageScanExecutionHistory(scanResultPayload.ImageScanEvent, scanEventJson, time.Now()) + err = impl.saveImageScanExecutionHistoryAndState(executionHistoryModel, scanResultPayload.ScanToolId) + if err != nil { + impl.Logger.Errorw("error in saving scan execution history and state mapping", "executionHistoryModel", executionHistoryModel, "toolId", scanResultPayload.ScanToolId, "err", err) + return 0, err + } + err = impl.SaveCvesAndImageScanExecutionResults(scanResultPayload.ImageScanOutput, executionHistoryModel.Id, scanResultPayload.ScanToolId, int32(scanResultPayload.ImageScanEvent.UserId)) + if err != nil { + impl.Logger.Errorw("error in saving cves and image scan exec result ", "executionHistoryId", executionHistoryModel.Id, "err", err) + return 0, err + } + return executionHistoryModel.Id, nil } diff --git a/image-scanner/pkg/sql/adaptor/adaptor.go b/image-scanner/pkg/sql/adaptor/adaptor.go new file mode 100644 index 000000000..9e3584488 --- /dev/null +++ b/image-scanner/pkg/sql/adaptor/adaptor.go @@ -0,0 +1,57 @@ +package adaptor + +import ( + "github.com/devtron-labs/common-lib/imageScan/bean" + bean2 "github.com/devtron-labs/image-scanner/pkg/sql/bean" + "github.com/devtron-labs/image-scanner/pkg/sql/repository" + "time" +) + +func GetImageScanExecutionHistory(imageScanEvent *bean.ImageScanEvent, scanEventJson []byte, time time.Time) *repository.ImageScanExecutionHistory { + return &repository.ImageScanExecutionHistory{ + Image: imageScanEvent.Image, + ImageHash: imageScanEvent.ImageDigest, + ExecutionTime: time, + ExecutedBy: imageScanEvent.UserId, + SourceMetadataJson: string(scanEventJson), + SourceType: imageScanEvent.SourceType, + SourceSubType: imageScanEvent.SourceSubType, + } +} + +func GetScanToolExecutionHistoryMapping(executionHistoryModel *repository.ImageScanExecutionHistory, state bean2.ScanExecutionProcessState, + errorMsg string, toolId int) *repository.ScanToolExecutionHistoryMapping { + return &repository.ScanToolExecutionHistoryMapping{ + ImageScanExecutionHistoryId: executionHistoryModel.Id, + ScanToolId: toolId, + ExecutionStartTime: executionHistoryModel.ExecutionTime, + State: state, + ErrorMessage: errorMsg, + AuditLog: repository.AuditLog{ + CreatedOn: executionHistoryModel.ExecutionTime, + CreatedBy: int32(executionHistoryModel.ExecutedBy), + UpdatedOn: executionHistoryModel.ExecutionTime, + UpdatedBy: int32(executionHistoryModel.ExecutedBy), + }, + } +} + +func GetResourceScanExecutionResult(executionHistoryId, scanToolId int, scanDataJson string, format repository.ResourceScanFormat) *repository.ResourceScanResult { + var resultTypes []int + switch format { + case repository.SbomResultSource: + resultTypes = []int{repository.Vulnerabilities.ToInt()} + case repository.CycloneDxSbom: + resultTypes = []int{repository.Vulnerabilities.ToInt(), repository.License.ToInt()} + default: + resultTypes = []int{repository.Vulnerabilities.ToInt(), repository.License.ToInt(), repository.Config.ToInt(), repository.Secrets.ToInt()} + + } + return &repository.ResourceScanResult{ + ImageScanExecutionHistoryId: executionHistoryId, + ScanDataJson: scanDataJson, + Format: format, + Types: resultTypes, + ScanToolId: scanToolId, + } +} diff --git a/image-scanner/pkg/sql/bean/bean.go b/image-scanner/pkg/sql/bean/bean.go index ff89d2365..cae798985 100644 --- a/image-scanner/pkg/sql/bean/bean.go +++ b/image-scanner/pkg/sql/bean/bean.go @@ -32,8 +32,10 @@ const ( ScannerTypeClairV2 = "CLAIRV2" ScannerTypeTrivy = "TRIVY" ScanToolClair = "CLAIR" + ScanToolAwsInspector = "AWS INSPECTOR" ScanToolVersion2 = "V2" ScanToolVersion4 = "V4" + ScanToolVersion1 = "V1" SbomOutputFileNameSuffix = "_out.json" ) @@ -52,17 +54,6 @@ const ( ScanExecutionProcessStateCompleted //resolved value = 1 ) -type ImageScanOutputObject struct { - TargetName string `json:"targetName"` - Class string `json:"class"` - Type string `json:"type"` - Name string `json:"name"` - Package string `json:"package"` - PackageVersion string `json:"packageVersion"` - FixedInVersion string `json:"fixedInVersion"` - Severity string `json:"severity"` -} - // Mapping is the used to store mappings of fields in ImageScanOutputObject and the path at which they are present in stepOutput type Mapping map[string]string diff --git a/image-scanner/pkg/sql/repository/ResourceScanExecutionResultRepository.go b/image-scanner/pkg/sql/repository/ResourceScanExecutionResultRepository.go index c79adadbc..4d3ea8c73 100644 --- a/image-scanner/pkg/sql/repository/ResourceScanExecutionResultRepository.go +++ b/image-scanner/pkg/sql/repository/ResourceScanExecutionResultRepository.go @@ -34,9 +34,10 @@ type ResourceScanResult struct { type ResourceScanFormat int const ( - CycloneDxSbom ResourceScanFormat = 1 //SBOM - TrivyJson = 2 - Json = 3 + CycloneDxSbom ResourceScanFormat = 1 //SBOM + TrivyJson = 2 + Json = 3 + SbomResultSource = 4 ) type ResourceScanType int diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go index b18d694ea..79decb1db 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/bean.go @@ -62,3 +62,22 @@ func (r *ImageScanEvent) IsImageFromManifest() bool { func (r *ImageScanEvent) IsBuiltImage() bool { return r.SourceType == constants.SourceTypeImage && r.SourceSubType == constants.SourceSubTypeCi } + +type ScanResultPayload struct { + ImageScanEvent *ImageScanEvent + ScanToolId int `json:"scanToolId"` + SourceScanningResult string `json:"sourceScanningResult"` + Sbom string `json:"sbom"` + ImageScanOutput []*ImageScanOutputObject `json:"imageScanOutput"` +} + +type ImageScanOutputObject struct { + TargetName string `json:"targetName"` + Class string `json:"class"` + Type string `json:"type"` + Name string `json:"name"` + Package string `json:"package"` + PackageVersion string `json:"packageVersion"` + FixedInVersion string `json:"fixedInVersion"` + Severity string `json:"severity"` +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/executionBean.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/executionBean.go new file mode 100644 index 000000000..d1313aba5 --- /dev/null +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/imageScan/bean/executionBean.go @@ -0,0 +1,16 @@ +package bean + +type ScanExecutionMedium string + +const ( + InHouse ScanExecutionMedium = "in-house" // this contains all methods of hitting image scanner directly via rest or rpc + External ScanExecutionMedium = "external" // if a scan tool is registered via api and execution is via plugin steps +) + +func (e ScanExecutionMedium) IsScanExecutionMediumInHouse() bool { + return e == InHouse +} + +func (e ScanExecutionMedium) IsScanMediumExternal() bool { + return e == External +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/image-scanner/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/image-scanner/vendor/modules.txt b/image-scanner/vendor/modules.txt index b1def1af5..154af5072 100644 --- a/image-scanner/vendor/modules.txt +++ b/image-scanner/vendor/modules.txt @@ -72,7 +72,7 @@ github.com/cespare/xxhash/v2 github.com/coreos/clair/api/v3/clairpb github.com/coreos/clair/database github.com/coreos/clair/ext/versionfmt -# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.19.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -440,4 +440,4 @@ google.golang.org/protobuf/types/known/wrapperspb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/image-scanner/wire_gen.go b/image-scanner/wire_gen.go index b668a2a40..d3a947ce0 100644 --- a/image-scanner/wire_gen.go +++ b/image-scanner/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/kubelink/Dockerfile b/kubelink/Dockerfile index e049e081f..5da88236f 100644 --- a/kubelink/Dockerfile +++ b/kubelink/Dockerfile @@ -1,22 +1,22 @@ FROM golang:1.22 AS build-env -RUN apt update -RUN apt install git gcc musl-dev make -y -RUN go install github.com/google/wire/cmd/wire@latest +RUN apt update && \ + apt install git gcc musl-dev make -y && \ + go install github.com/google/wire/cmd/wire@latest WORKDIR /go/src/github.com/devtron-labs/kubelink ADD . /go/src/github.com/devtron-labs/kubelink/ RUN GOOS=linux make FROM ubuntu:22.04@sha256:1b8d8ff4777f36f19bfe73ee4df61e3a0b789caeff29caa019539ec7c9a57f95 -RUN apt update -RUN apt install ca-certificates -y -RUN apt clean autoclean -RUN apt autoremove -y && rm -rf /var/lib/apt/lists/* -COPY --from=build-env /go/src/github.com/devtron-labs/kubelink/kubelink . +RUN apt update && \ + apt install ca-certificates -y && \ + apt clean autoclean && \ + apt autoremove -y && rm -rf /var/lib/apt/lists/* && \ + useradd -ms /bin/bash devtron + +COPY --chown=devtron:devtron --from=build-env /go/src/github.com/devtron-labs/kubelink/kubelink . -RUN useradd -ms /bin/bash devtron -RUN chown -R devtron:devtron ./kubelink USER devtron CMD ["./kubelink"] diff --git a/kubelink/go.mod b/kubelink/go.mod index 63b91768a..4024d8316 100644 --- a/kubelink/go.mod +++ b/kubelink/go.mod @@ -177,7 +177,7 @@ require ( ) replace ( - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 // https://github.com/kubernetes/kubernetes/issues/79384#issuecomment-505627280 k8s.io/api => k8s.io/api v0.29.0 diff --git a/kubelink/go.sum b/kubelink/go.sum index 0434a469f..c959a860e 100644 --- a/kubelink/go.sum +++ b/kubelink/go.sum @@ -79,8 +79,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/distribution/v3 v3.0.0-beta.1 h1:X+ELTxPuZ1Xe5MsD3kp2wfGUhc8I+MPfRis8dZ818Ic= diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/kubelink/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/kubelink/vendor/modules.txt b/kubelink/vendor/modules.txt index e6e76e811..e4e907c89 100644 --- a/kubelink/vendor/modules.txt +++ b/kubelink/vendor/modules.txt @@ -127,7 +127,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/constants @@ -1354,7 +1354,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f # go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.18.0 # k8s.io/api => k8s.io/api v0.29.0 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.0 diff --git a/kubelink/wire_gen.go b/kubelink/wire_gen.go index c8075adb9..2ded63c44 100644 --- a/kubelink/wire_gen.go +++ b/kubelink/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/kubewatch/Dockerfile b/kubewatch/Dockerfile index b4a281f5a..e750188e1 100644 --- a/kubewatch/Dockerfile +++ b/kubewatch/Dockerfile @@ -1,25 +1,22 @@ FROM golang:1.21-alpine3.19 AS build-env -RUN echo $GOPATH +RUN echo $GOPATH && \ + apk add --no-cache git gcc musl-dev && \ + apk add --update make -RUN apk add --no-cache git gcc musl-dev -RUN apk add --update make WORKDIR /go/src/github.com/devtron-labs/kubewatch ADD . /go/src/github.com/devtron-labs/kubewatch RUN GOOS=linux make FROM alpine:3.19 -RUN apk add --update ca-certificates +RUN apk add --update ca-certificates && \ + adduser -D devtron -RUN adduser -D devtron - -COPY --from=build-env /go/src/github.com/devtron-labs/kubewatch/kubewatch . - -RUN chown devtron:devtron ./kubewatch +COPY --chown=devtron:devtron --from=build-env /go/src/github.com/devtron-labs/kubewatch/kubewatch . RUN chmod +x ./kubewatch USER devtron -ENTRYPOINT ["./kubewatch"] \ No newline at end of file +ENTRYPOINT ["./kubewatch"] diff --git a/kubewatch/go.mod b/kubewatch/go.mod index 5b9de5098..1d248f39e 100644 --- a/kubewatch/go.mod +++ b/kubewatch/go.mod @@ -245,4 +245,4 @@ replace ( k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/kubewatch/go.sum b/kubewatch/go.sum index 347db2488..4fb4348ff 100644 --- a/kubewatch/go.sum +++ b/kubewatch/go.sum @@ -719,8 +719,8 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/kubewatch/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/kubewatch/vendor/modules.txt b/kubewatch/vendor/modules.txt index fb18baab2..b3e9e2190 100644 --- a/kubewatch/vendor/modules.txt +++ b/kubewatch/vendor/modules.txt @@ -211,7 +211,7 @@ github.com/cyphar/filepath-securejoin # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -1754,4 +1754,4 @@ upper.io/db.v3/postgresql # k8s.io/mount-utils => k8s.io/mount-utils v0.29.7 # k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.7 # k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.7 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/lens/Dockerfile b/lens/Dockerfile index 5d4c54a4a..90f74a7ad 100644 --- a/lens/Dockerfile +++ b/lens/Dockerfile @@ -1,17 +1,22 @@ FROM golang:1.21-alpine3.18 AS build-env -RUN apk add --no-cache git gcc musl-dev -RUN apk add --update make +RUN apk add --no-cache git gcc musl-dev && \ + apk add --update make + WORKDIR /go/src/github.com/devtron-labs/lens + ADD . /go/src/github.com/devtron-labs/lens -RUN go install github.com/google/wire/cmd/wire@latest -RUN GOOS=linux make + +RUN go install github.com/google/wire/cmd/wire@latest && \ + GOOS=linux make FROM alpine:3.17 -RUN apk add --no-cache ca-certificates -COPY --from=build-env /go/src/github.com/devtron-labs/lens/lens . -COPY --from=build-env /go/src/github.com/devtron-labs/lens/scripts/ . -RUN adduser -D devtron -RUN chown -R devtron:devtron ./lens + +RUN apk add --no-cache ca-certificates && \ + adduser -D devtron + +COPY --chown=devtron:devtron --from=build-env /go/src/github.com/devtron-labs/lens/lens /go/src/github.com/devtron-labs/lens/scripts/ . + USER devtron + CMD ["./lens"] diff --git a/lens/go.mod b/lens/go.mod index 23c243db7..4b7a86b5e 100644 --- a/lens/go.mod +++ b/lens/go.mod @@ -59,4 +59,4 @@ require ( github.com/onsi/gomega v1.18.1 // indirect ) -replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +replace github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f diff --git a/lens/go.sum b/lens/go.sum index 9bb520cd2..e78edac3b 100644 --- a/lens/go.sum +++ b/lens/go.sum @@ -22,8 +22,8 @@ github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWH github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a h1:KKPEVBfa4sXptJc/0ZVWcxePI/JNtzQl0ugnwTePvgk= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f h1:4wUbt+83DmpZFqYS69CJxNtBpSuCb58UwqOrWsZG82s= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f/go.mod h1:1QJJLpgJSkb5Jm9xPeKAk+kXb0QgBOOOgJj0cgYhAVA= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb h1:CkfQQgZc950/hTPqtQSiHV2RmZgkBLGCzwR02FZYjAU= github.com/devtron-labs/protos v0.0.3-0.20240130061723-7b2e12ab0abb/go.mod h1:pjLjgoa1GzbkOkvbMyP4SAKsaiK7eG6GoQCNauG03JA= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index 4f92471ed..ad3cbbda0 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -36,9 +36,13 @@ import ( var chars = []rune("abcdefghijklmnopqrstuvwxyz0123456789") const ( - DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" - DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" - DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" + DEVTRON_SELF_POD_UID = "DEVTRON_SELF_POD_UID" + DEVTRON_SELF_POD_NAME = "DEVTRON_SELF_POD_NAME" + DEVTRON_SELF_DOWNWARD_API_VOLUME = "devtron-pod-info" + DEVTRON_SELF_DOWNWARD_API_VOLUME_PATH = "/etc/devtron-pod-info" + POD_LABELS = "labels" + POD_ANNOTATIONS = "annotations" ) // Generates random string @@ -151,3 +155,22 @@ var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pg_query_duration_seconds", Help: "Duration of PG queries", }, []string{"status", "serviceName"}) + +func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { + targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) + targetPlatformObject := []*bean.TargetPlatform{} + for _, targetPlatform := range targetPlatforms { + if len(targetPlatform) > 0 { + targetPlatformObject = append(targetPlatformObject, &bean.TargetPlatform{Name: targetPlatform}) + } + } + return targetPlatformObject +} + +func ConvertTargetPlatformStringToList(targetPlatform string) []string { + return strings.Split(targetPlatform, ",") +} + +func ConvertTargetPlatformListToString(targetPlatforms []string) string { + return strings.Join(targetPlatforms, ",") +} diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go index a26977c6b..61d8c5fff 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/FileUtil.go @@ -1,6 +1,8 @@ package utils import ( + "errors" + "fmt" "io/ioutil" "os" "path" @@ -34,3 +36,39 @@ func DeleteAFileIfExists(path string) error { } return nil } + +const ( + PermissionMode = 0644 +) + +func CreateDirectory(path string) error { + err := os.MkdirAll(path, PermissionMode) + if err != nil { + fmt.Println("error in creating directory", "err", err) + return err + } + return nil +} + +func CheckFileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err == nil { + // exists + return true, nil + + } else if errors.Is(err, os.ErrNotExist) { + // not exists + return false, nil + } else { + // Some other error + return false, err + } +} + +func WriteToFile(file string, fileName string) error { + err := os.WriteFile(fileName, []byte(file), PermissionMode) + if err != nil { + fmt.Println("error in writing results to json file", "err", err) + return err + } + return nil +} diff --git a/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 56d485090..50b122e49 100644 --- a/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/lens/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -84,3 +84,7 @@ type PgQueryEvent struct { Error error Query string } + +type TargetPlatform struct { + Name string `json:"name"` +} diff --git a/lens/vendor/modules.txt b/lens/vendor/modules.txt index bbc16246a..6417e4a1b 100644 --- a/lens/vendor/modules.txt +++ b/lens/vendor/modules.txt @@ -7,7 +7,7 @@ github.com/caarlos0/env # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib v0.0.0 => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f ## explicit; go 1.21 github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/fetchAllEnv @@ -289,4 +289,4 @@ google.golang.org/protobuf/types/known/timestamppb # mellium.im/sasl v0.3.2 ## explicit; go 1.20 mellium.im/sasl -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250122120743-f9dc0038c36a +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250127104410-85d6bfe0b45f