From a4bcbd606105a255faf0638d90726b70cb135e73 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 11 Sep 2024 12:50:58 +0530 Subject: [PATCH 01/32] support for CdRollback and DeploymentHistory in configData API --- .../DeploymentConfigurationService.go | 183 ++++++++++++++++-- pkg/configDiff/bean/bean.go | 33 +++- pkg/pipeline/bean/ConfigMapBean.go | 1 + .../DeployedConfigurationHistoryService.go | 2 + wire_gen.go | 4 +- 5 files changed, 203 insertions(+), 20 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index f64de5cd2f..1862af6535 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -3,9 +3,11 @@ package configDiff import ( "context" "encoding/json" + "errors" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" + bean3 "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/configDiff/adaptor" @@ -15,6 +17,8 @@ import ( "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/go-pg/pg" "go.uber.org/zap" "net/http" "strconv" @@ -26,12 +30,15 @@ type DeploymentConfigurationService interface { } type DeploymentConfigurationServiceImpl struct { - logger *zap.SugaredLogger - configMapService pipeline.ConfigMapService - appRepository appRepository.AppRepository - environmentRepository repository.EnvironmentRepository - chartService chartService.ChartService - deploymentTemplateService generateManifest.DeploymentTemplateService + logger *zap.SugaredLogger + configMapService pipeline.ConfigMapService + appRepository appRepository.AppRepository + environmentRepository repository.EnvironmentRepository + chartService chartService.ChartService + deploymentTemplateService generateManifest.DeploymentTemplateService + deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository + pipelineStrategyHistoryRepository repository3.PipelineStrategyHistoryRepository + configMapHistoryRepository repository3.ConfigMapHistoryRepository } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, @@ -40,14 +47,20 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, environmentRepository repository.EnvironmentRepository, chartService chartService.ChartService, deploymentTemplateService generateManifest.DeploymentTemplateService, + deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, + pipelineStrategyHistoryRepository repository3.PipelineStrategyHistoryRepository, + configMapHistoryRepository repository3.ConfigMapHistoryRepository, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ - logger: logger, - configMapService: configMapService, - appRepository: appRepository, - environmentRepository: environmentRepository, - chartService: chartService, - deploymentTemplateService: deploymentTemplateService, + logger: logger, + configMapService: configMapService, + appRepository: appRepository, + environmentRepository: environmentRepository, + chartService: chartService, + deploymentTemplateService: deploymentTemplateService, + deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, + pipelineStrategyHistoryRepository: pipelineStrategyHistoryRepository, + configMapHistoryRepository: configMapHistoryRepository, } return deploymentConfigurationService, nil @@ -102,7 +115,153 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con return nil, err } + switch configDataQueryParams.ConfigArea { + case bean2.CdRollback.ToString(): + return impl.getConfigDataForCdRollback(ctx, configDataQueryParams, appId, envId) + case bean2.DeploymentHistory.ToString(): + return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, appId, envId) + } + // this would be the default case + return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId) +} + +func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, + appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + // we would be expecting wfrId in case of getting data for cdRollback + +} + +func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*json.RawMessage, error) { + deploymentJson := &json.RawMessage{} + deploymentHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(configDataQueryParams.PipelineId, configDataQueryParams.WfrId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting deployment template history for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) + return nil, err + } else if err == pg.ErrNoRows { + //history not created yet + return deploymentJson, nil + } + err = deploymentJson.UnmarshalJSON([]byte(deploymentHistory.Template)) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string deploymentTemplateResponse data into json Raw message", "data", deploymentHistory.Template, "err", err) + return nil, err + } + return deploymentJson, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getPipelineStrategyConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*json.RawMessage, error) { + pipelineStrategyJson := &json.RawMessage{} + pipelineStrategyHistory, err := impl.pipelineStrategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(ctx, configDataQueryParams.PipelineId, configDataQueryParams.WfrId) + if err != nil && !errors.Is(err, pg.ErrNoRows) { + impl.logger.Errorw("error in checking if history exists for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) + return nil, err + } else if errors.Is(err, pg.ErrNoRows) { + return pipelineStrategyJson, nil + } + + err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategyHistory.Config)) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "pipelineStrategyHistoryConfig", pipelineStrategyHistory.Config, "err", err) + return nil, err + } + return pipelineStrategyJson, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getConfigDataForDeploymentHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, + appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + // we would be expecting wfrId in case of getting data for Deployment history configDataDto := &bean2.DeploymentAndCmCsConfigDto{} + var err error + //fetching history for deployment config starts + deploymentConfigJson, err := impl.getDeploymentHistoryConfig(ctx, configDataQueryParams) + if err != nil { + impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getDeploymentHistoryConfig", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + if deploymentConfigJson != nil { + deploymentConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*deploymentConfigJson).WithResourceType(bean.DeploymentTemplate) + configDataDto.WithDeploymentTemplateData(deploymentConfig) + } + // fetching for deployment config ends + + // fetching for pipeline strategy config starts + pipelineConfigJson, err := impl.getPipelineStrategyConfigHistory(ctx, configDataQueryParams) + if err != nil { + impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getPipelineStrategyConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + if pipelineConfigJson != nil { + pipelineConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*pipelineConfigJson).WithResourceType(bean.PipelineStrategy) + configDataDto.WithPipelineConfigData(pipelineConfig) + } + // fetching for pipeline strategy config ends + + // fetching for cm config starts + cmConfigJson, err := impl.getCmCsConfigHistory(ctx, configDataQueryParams, repository3.CONFIGMAP_TYPE) + if err != nil { + impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getCmConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + if cmConfigJson != nil { + cmConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*cmConfigJson).WithResourceType(bean.CM) + configDataDto.WithConfigMapData(cmConfigData) + } + // fetching for cm config ends + + // fetching for cs config starts + secretConfigJson, err := impl.getCmCsConfigHistory(ctx, configDataQueryParams, repository3.SECRET_TYPE) + if err != nil { + impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getSecretConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + if secretConfigJson != nil { + secretConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*secretConfigJson).WithResourceType(bean.CS) + configDataDto.WithSecretData(secretConfigData) + } + // fetching for cs config ends + + return configDataDto, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, configType repository3.ConfigType) (*json.RawMessage, error) { + cmJson := &json.RawMessage{} + history, err := impl.configMapHistoryRepository.GetHistoryByPipelineIdAndWfrId(configDataQueryParams.PipelineId, configDataQueryParams.WfrId, configType) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in checking if cm cs history exists for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) + return nil, err + } else if err == pg.ErrNoRows { + return cmJson, nil + } + //var configData []*bean3.ConfigData + if configType == repository3.CONFIGMAP_TYPE { + configList := bean3.ConfigList{} + if len(history.Data) > 0 { + err = json.Unmarshal([]byte(history.Data), &configList) + if err != nil { + impl.logger.Debugw("error while Unmarshal", "err", err) + return nil, err + } + } + //configData = configList.ConfigData + } else if configType == repository3.SECRET_TYPE { + secretList := bean3.SecretList{} + if len(history.Data) > 0 { + err = json.Unmarshal([]byte(history.Data), &secretList) + if err != nil { + impl.logger.Debugw("error while Unmarshal", "err", err) + return nil, err + } + } + //configData = secretList.ConfigData + } + + return nil, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getConfigDataForAppConfiguration(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, + appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + configDataDto := &bean2.DeploymentAndCmCsConfigDto{} + var err error switch configDataQueryParams.ConfigType { default: // keeping default as PublishedOnly configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 2113ea81a6..2e13b2aea4 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -25,6 +25,18 @@ const ( Overridden ConfigStage = "Overridden" ) +type ConfigArea string + +const ( + AppConfiguration ConfigArea = "AppConfiguration" + DeploymentHistory ConfigArea = "DeploymentHistory" + CdRollback ConfigArea = "CdRollback" +) + +func (r ConfigArea) ToString() string { + return string(r) +} + type ConfigProperty struct { Id int `json:"id"` Name string `json:"name"` @@ -71,8 +83,10 @@ func (r *ConfigProperty) GetIdentifier() ConfigPropertyIdentifier { } type DeploymentAndCmCsConfig struct { - ResourceType bean.ResourceType `json:"resourceType"` - Data json.RawMessage `json:"data"` + ResourceType bean.ResourceType `json:"resourceType"` + Data json.RawMessage `json:"data"` + VariableSnapshot string `json:"variableSnapshot"` + ResolvedValue string `json:"resolvedValue"` } func NewDeploymentAndCmCsConfig() *DeploymentAndCmCsConfig { @@ -93,6 +107,7 @@ type DeploymentAndCmCsConfigDto struct { DeploymentTemplate *DeploymentAndCmCsConfig `json:"deploymentTemplate"` ConfigMapsData *DeploymentAndCmCsConfig `json:"configMapData"` SecretsData *DeploymentAndCmCsConfig `json:"secretsData"` + PipelineConfigData *DeploymentAndCmCsConfig `json:"pipelineConfigData,omitempty"` IsAppAdmin bool `json:"isAppAdmin"` } @@ -112,17 +127,23 @@ func (r *DeploymentAndCmCsConfigDto) WithSecretData(data *DeploymentAndCmCsConfi r.SecretsData = data return r } +func (r *DeploymentAndCmCsConfigDto) WithPipelineConfigData(data *DeploymentAndCmCsConfig) *DeploymentAndCmCsConfigDto { + r.PipelineConfigData = data + return r +} type ConfigDataQueryParams struct { AppName string `schema:"appName"` EnvName string `schema:"envName"` ConfigType string `schema:"configType"` IdentifierId int `schema:"identifierId"` - PipelineId int `schema:"pipelineId"` // req for fetching previous deployments data - ResourceName string `schema:"resourceName"` - ResourceType string `schema:"resourceType"` - ResourceId int `schema:"resourceId"` + PipelineId int `schema:"pipelineId"` // req for fetching previous deployments data + ResourceName string `schema:"resourceName"` // used in case of cm and cs + ResourceType string `schema:"resourceType"` // used in case of cm and cs + ResourceId int `schema:"resourceId"` // used in case of cm and cs UserId int32 `schema:"-"` + WfrId int `schema:"wfrId"` + ConfigArea string `schema:"configArea"` } // FilterCriteria []string `schema:"filterCriteria"` diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index 2f572bd605..3630a3aa7e 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -120,6 +120,7 @@ const ( CM ResourceType = "ConfigMap" CS ResourceType = "Secret" DeploymentTemplate ResourceType = "Deployment Template" + PipelineStrategy ResourceType = "Pipeline Strategy" ) func (r ResourceType) ToString() string { diff --git a/pkg/pipeline/history/DeployedConfigurationHistoryService.go b/pkg/pipeline/history/DeployedConfigurationHistoryService.go index 26124a6df7..241574a2f1 100644 --- a/pkg/pipeline/history/DeployedConfigurationHistoryService.go +++ b/pkg/pipeline/history/DeployedConfigurationHistoryService.go @@ -152,6 +152,7 @@ func (impl *DeployedConfigurationHistoryServiceImpl) GetDeployedConfigurationByW impl.logger.Errorw("error in checking if history exists for deployment template", "err", err, "pipelineId", pipelineId, "wfrId", wfrId) return nil, err } + deploymentTemplateConfiguration := &DeploymentConfigurationDto{ Name: DEPLOYMENT_TEMPLATE_TYPE_HISTORY_COMPONENT, } @@ -161,6 +162,7 @@ func (impl *DeployedConfigurationHistoryServiceImpl) GetDeployedConfigurationByW deployedConfigurations = append(deployedConfigurations, deploymentTemplateConfiguration) //checking if pipeline strategy configuration for this pipelineId and wfrId exists or not + strategyHistoryId, exists, err := impl.strategyHistoryService.CheckIfHistoryExistsForPipelineIdAndWfrId(newCtx, pipelineId, wfrId) if err != nil { impl.logger.Errorw("error in checking if history exists for pipeline strategy", "err", err, "pipelineId", pipelineId, "wfrId", wfrId) diff --git a/wire_gen.go b/wire_gen.go index 9ad1f1575a..c236db97e6 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -949,7 +949,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl) if err != nil { return nil, err } From e2ee5a6b9139f743dbf4d729469ff1ac9439a362 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 12 Sep 2024 16:55:56 +0530 Subject: [PATCH 02/32] support for getConfigDataForDeploymentHistory and getConfigDataForCdRollback in configData --- .../DeploymentConfigurationRestHandler.go | 3 +- pkg/bean/configSecretData.go | 3 + .../DeploymentConfigurationService.go | 178 ++++++++++++------ pkg/configDiff/adaptor/adaptor.go | 35 ++++ pkg/configDiff/bean/bean.go | 18 +- pkg/configDiff/utils/utils.go | 8 + pkg/pipeline/adapter/adapter.go | 119 ++++++++++++ pkg/pipeline/bean/ConfigMapBean.go | 2 + wire_gen.go | 2 +- 9 files changed, 304 insertions(+), 64 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 144838da01..f5a2551f81 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -97,8 +97,9 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp return } //RBAC END + userHasAdminAccess := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, object) - res, err := handler.deploymentConfigurationService.GetAllConfigData(r.Context(), configDataQueryParams) + res, err := handler.deploymentConfigurationService.GetAllConfigData(r.Context(), configDataQueryParams, userHasAdminAccess) if err != nil { handler.logger.Errorw("service err, GetAllConfigData ", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/pkg/bean/configSecretData.go b/pkg/bean/configSecretData.go index 61e55db588..42803600f9 100644 --- a/pkg/bean/configSecretData.go +++ b/pkg/bean/configSecretData.go @@ -29,6 +29,8 @@ type SecretList struct { ConfigData []*ConfigData `json:"secrets"` } +// there is an adapter written in pkg/bean folder to convert below ConfigData struct to pipeline/bean's ConfigData + type ConfigData struct { Name string `json:"name"` Type string `json:"type"` @@ -46,6 +48,7 @@ type ConfigData struct { RoleARN string `json:"roleARN"` SubPath bool `json:"subPath"` FilePermission string `json:"filePermission"` + Overridden bool `json:"overridden"` } type ExternalSecret struct { diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 1862af6535..46eb911249 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -3,11 +3,9 @@ package configDiff import ( "context" "encoding/json" - "errors" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" - bean3 "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/configDiff/adaptor" @@ -16,9 +14,13 @@ import ( "github.com/devtron-labs/devtron/pkg/configDiff/utils" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" - "github.com/go-pg/pg" + "github.com/devtron-labs/devtron/pkg/variables" + "github.com/devtron-labs/devtron/pkg/variables/parsers" + repository6 "github.com/devtron-labs/devtron/pkg/variables/repository" + util2 "github.com/devtron-labs/devtron/util" "go.uber.org/zap" "net/http" "strconv" @@ -26,7 +28,7 @@ import ( type DeploymentConfigurationService interface { ConfigAutoComplete(appId int, envId int) (*bean2.ConfigDataResponse, error) - GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfigDto, error) + GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) } type DeploymentConfigurationServiceImpl struct { @@ -39,6 +41,7 @@ type DeploymentConfigurationServiceImpl struct { deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository pipelineStrategyHistoryRepository repository3.PipelineStrategyHistoryRepository configMapHistoryRepository repository3.ConfigMapHistoryRepository + scopedVariableManager variables.ScopedVariableCMCSManager } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, @@ -50,6 +53,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, pipelineStrategyHistoryRepository repository3.PipelineStrategyHistoryRepository, configMapHistoryRepository repository3.ConfigMapHistoryRepository, + scopedVariableManager variables.ScopedVariableCMCSManager, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ logger: logger, @@ -61,6 +65,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, pipelineStrategyHistoryRepository: pipelineStrategyHistoryRepository, configMapHistoryRepository: configMapHistoryRepository, + scopedVariableManager: scopedVariableManager, } return deploymentConfigurationService, nil @@ -95,7 +100,7 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en return configDataResp, nil } -func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfigDto, error) { +func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { if !configDataQueryParams.IsValidConfigType() { return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.InvalidConfigTypeErr, UserMessage: bean2.InvalidConfigTypeErr} } @@ -117,124 +122,121 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con switch configDataQueryParams.ConfigArea { case bean2.CdRollback.ToString(): - return impl.getConfigDataForCdRollback(ctx, configDataQueryParams, appId, envId) + return impl.getConfigDataForCdRollback(ctx, configDataQueryParams, userHasAdminAccess) case bean2.DeploymentHistory.ToString(): - return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, appId, envId) + return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) } // this would be the default case return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId) } -func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, - appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { - // we would be expecting wfrId in case of getting data for cdRollback - +func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { + // wfrId is expected in this case to return the expected data + return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) } -func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*json.RawMessage, error) { - deploymentJson := &json.RawMessage{} +func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfig, error) { + deploymentJson := json.RawMessage{} deploymentHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(configDataQueryParams.PipelineId, configDataQueryParams.WfrId) - if err != nil && err != pg.ErrNoRows { + if err != nil { impl.logger.Errorw("error in getting deployment template history for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err - } else if err == pg.ErrNoRows { - //history not created yet - return deploymentJson, nil } err = deploymentJson.UnmarshalJSON([]byte(deploymentHistory.Template)) if err != nil { impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string deploymentTemplateResponse data into json Raw message", "data", deploymentHistory.Template, "err", err) return nil, err } - return deploymentJson, nil + isSuperAdmin, err := util2.GetIsSuperAdminFromContext(ctx) + if err != nil { + return nil, err + } + reference := repository6.HistoryReference{ + HistoryReferenceId: deploymentHistory.Id, + HistoryReferenceType: repository6.HistoryReferenceTypeDeploymentTemplate, + } + variableSnapshotMap, resolvedTemplate, err := impl.scopedVariableManager.GetVariableSnapshotAndResolveTemplate(deploymentHistory.Template, parsers.JsonVariableTemplate, reference, isSuperAdmin, false) + if err != nil { + impl.logger.Errorw("error while resolving template from history", "deploymentHistoryId", deploymentHistory.Id, "pipelineId", configDataQueryParams.PipelineId, "err", err) + } + deploymentConfig := bean2.NewDeploymentAndCmCsConfig(). + WithConfigData(deploymentJson). + WithResourceType(bean.DeploymentTemplate). + WithVariableSnapshot(map[string]map[string]string{bean.DeploymentTemplate.ToString(): variableSnapshotMap}). + WithResolvedValue(resolvedTemplate) + return deploymentConfig, nil } -func (impl *DeploymentConfigurationServiceImpl) getPipelineStrategyConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*json.RawMessage, error) { - pipelineStrategyJson := &json.RawMessage{} +func (impl *DeploymentConfigurationServiceImpl) getPipelineStrategyConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfig, error) { + pipelineStrategyJson := json.RawMessage{} pipelineStrategyHistory, err := impl.pipelineStrategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(ctx, configDataQueryParams.PipelineId, configDataQueryParams.WfrId) - if err != nil && !errors.Is(err, pg.ErrNoRows) { + if err != nil { impl.logger.Errorw("error in checking if history exists for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err - } else if errors.Is(err, pg.ErrNoRows) { - return pipelineStrategyJson, nil } - err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategyHistory.Config)) if err != nil { impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "pipelineStrategyHistoryConfig", pipelineStrategyHistory.Config, "err", err) return nil, err } - return pipelineStrategyJson, nil + pipelineConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(pipelineStrategyJson).WithResourceType(bean.PipelineStrategy) + return pipelineConfig, nil } -func (impl *DeploymentConfigurationServiceImpl) getConfigDataForDeploymentHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, - appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { +func (impl *DeploymentConfigurationServiceImpl) getConfigDataForDeploymentHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { // we would be expecting wfrId in case of getting data for Deployment history configDataDto := &bean2.DeploymentAndCmCsConfigDto{} var err error //fetching history for deployment config starts - deploymentConfigJson, err := impl.getDeploymentHistoryConfig(ctx, configDataQueryParams) + deploymentConfig, err := impl.getDeploymentHistoryConfig(ctx, configDataQueryParams) if err != nil { impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getDeploymentHistoryConfig", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - if deploymentConfigJson != nil { - deploymentConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*deploymentConfigJson).WithResourceType(bean.DeploymentTemplate) - configDataDto.WithDeploymentTemplateData(deploymentConfig) - } + configDataDto.WithDeploymentTemplateData(deploymentConfig) // fetching for deployment config ends // fetching for pipeline strategy config starts - pipelineConfigJson, err := impl.getPipelineStrategyConfigHistory(ctx, configDataQueryParams) + pipelineConfig, err := impl.getPipelineStrategyConfigHistory(ctx, configDataQueryParams) if err != nil { impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getPipelineStrategyConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - if pipelineConfigJson != nil { - pipelineConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*pipelineConfigJson).WithResourceType(bean.PipelineStrategy) - configDataDto.WithPipelineConfigData(pipelineConfig) - } + configDataDto.WithPipelineConfigData(pipelineConfig) // fetching for pipeline strategy config ends // fetching for cm config starts - cmConfigJson, err := impl.getCmCsConfigHistory(ctx, configDataQueryParams, repository3.CONFIGMAP_TYPE) + cmConfigData, err := impl.getCmCsConfigHistory(ctx, configDataQueryParams, repository3.CONFIGMAP_TYPE, userHasAdminAccess) if err != nil { impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getCmConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - if cmConfigJson != nil { - cmConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*cmConfigJson).WithResourceType(bean.CM) - configDataDto.WithConfigMapData(cmConfigData) - } + configDataDto.WithConfigMapData(cmConfigData) // fetching for cm config ends // fetching for cs config starts - secretConfigJson, err := impl.getCmCsConfigHistory(ctx, configDataQueryParams, repository3.SECRET_TYPE) + secretConfigDto, err := impl.getCmCsConfigHistory(ctx, configDataQueryParams, repository3.SECRET_TYPE, userHasAdminAccess) if err != nil { impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getSecretConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - if secretConfigJson != nil { - secretConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(*secretConfigJson).WithResourceType(bean.CS) - configDataDto.WithSecretData(secretConfigData) - } + configDataDto.WithSecretData(secretConfigDto) // fetching for cs config ends return configDataDto, nil } -func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, configType repository3.ConfigType) (*json.RawMessage, error) { - cmJson := &json.RawMessage{} +func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, configType repository3.ConfigType, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfig, error) { + var resourceType bean.ResourceType history, err := impl.configMapHistoryRepository.GetHistoryByPipelineIdAndWfrId(configDataQueryParams.PipelineId, configDataQueryParams.WfrId, configType) - if err != nil && err != pg.ErrNoRows { + if err != nil { impl.logger.Errorw("error in checking if cm cs history exists for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err - } else if err == pg.ErrNoRows { - return cmJson, nil } - //var configData []*bean3.ConfigData + var configData []*bean.ConfigData + configList := pipeline.ConfigsList{} + secretList := bean.SecretsList{} if configType == repository3.CONFIGMAP_TYPE { - configList := bean3.ConfigList{} if len(history.Data) > 0 { err = json.Unmarshal([]byte(history.Data), &configList) if err != nil { @@ -242,9 +244,9 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context return nil, err } } - //configData = configList.ConfigData + resourceType = bean.CM + configData = configList.ConfigData } else if configType == repository3.SECRET_TYPE { - secretList := bean3.SecretList{} if len(history.Data) > 0 { err = json.Unmarshal([]byte(history.Data), &secretList) if err != nil { @@ -252,10 +254,70 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context return nil, err } } - //configData = secretList.ConfigData + resourceType = bean.CS + configData = secretList.ConfigData } - return nil, nil + resolvedDataMap, variableSnapshotMap, err := impl.scopedVariableManager.GetResolvedCMCSHistoryDtos(ctx, configType, adaptor.ReverseConfigListConvertor(configList), history, adaptor.ReverseSecretListConvertor(secretList)) + if err != nil { + return nil, err + } + resolvedConfigDataList := make([]*bean.ConfigData, 0, len(resolvedDataMap)) + for _, resolvedConfigData := range resolvedDataMap { + resolvedConfigDataList = append(resolvedConfigDataList, adapter.ConvertConfigDataToPipelineConfigData(&resolvedConfigData)) + } + + if configType == repository3.SECRET_TYPE { + impl.encodeSecretDataFromNonAdminUsers(configData, userHasAdminAccess) + impl.encodeSecretDataFromNonAdminUsers(resolvedConfigDataList, userHasAdminAccess) + + } + + configDataReq := &bean.ConfigDataRequest{ConfigData: configData} + configDataJson, err := utils.ConvertToJsonRawMessage(configDataReq) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) + return nil, err + } + resolvedConfigDataReq := &bean.ConfigDataRequest{ConfigData: resolvedConfigDataList} + resolvedConfigDataString, err := utils.ConvertToString(resolvedConfigDataReq) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) + return nil, err + } + + cmConfigData := bean2.NewDeploymentAndCmCsConfig(). + WithConfigData(configDataJson). + WithResourceType(resourceType). + WithVariableSnapshot(variableSnapshotMap). + WithResolvedValue(resolvedConfigDataString) + return cmConfigData, nil +} + +func (impl *DeploymentConfigurationServiceImpl) encodeSecretDataFromNonAdminUsers(configDataList []*bean.ConfigData, userHasAdminAccess bool) { + for _, config := range configDataList { + if config.Data != nil { + if !userHasAdminAccess { + //removing keys and sending + resultMap := make(map[string]string) + resultMapFinal := make(map[string]string) + err := json.Unmarshal(config.Data, &resultMap) + if err != nil { + impl.logger.Errorw("unmarshal failed", "error", err) + return + } + for key, _ := range resultMap { + //hard-coding values to show them as hidden to user + resultMapFinal[key] = "*****" + } + config.Data, err = utils.ConvertToJsonRawMessage(resultMapFinal) + if err != nil { + impl.logger.Errorw("error while marshaling request", "err", err) + return + } + } + } + } } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForAppConfiguration(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, diff --git a/pkg/configDiff/adaptor/adaptor.go b/pkg/configDiff/adaptor/adaptor.go index 4ab81eb2d1..6b93442ff4 100644 --- a/pkg/configDiff/adaptor/adaptor.go +++ b/pkg/configDiff/adaptor/adaptor.go @@ -1,7 +1,10 @@ package adaptor import ( + bean3 "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) @@ -27,3 +30,35 @@ func GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel []bean.Config } return cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap } + +func ConfigListConvertor(r bean3.ConfigList) pipeline.ConfigsList { + pipelineConfigData := make([]*bean.ConfigData, 0, len(r.ConfigData)) + for _, item := range r.ConfigData { + pipelineConfigData = append(pipelineConfigData, adapter.ConvertConfigDataToPipelineConfigData(item)) + } + return pipeline.ConfigsList{ConfigData: pipelineConfigData} +} + +func SecretListConvertor(r bean3.SecretList) bean.SecretsList { + pipelineConfigData := make([]*bean.ConfigData, 0, len(r.ConfigData)) + for _, item := range r.ConfigData { + pipelineConfigData = append(pipelineConfigData, adapter.ConvertConfigDataToPipelineConfigData(item)) + } + return bean.SecretsList{ConfigData: pipelineConfigData} +} + +func ReverseConfigListConvertor(r pipeline.ConfigsList) bean3.ConfigList { + configData := make([]*bean3.ConfigData, 0, len(r.ConfigData)) + for _, item := range r.ConfigData { + configData = append(configData, adapter.ConvertPipelineConfigDataToConfigData(item)) + } + return bean3.ConfigList{ConfigData: configData} +} + +func ReverseSecretListConvertor(r bean.SecretsList) bean3.SecretList { + configData := make([]*bean3.ConfigData, 0, len(r.ConfigData)) + for _, item := range r.ConfigData { + configData = append(configData, adapter.ConvertPipelineConfigDataToConfigData(item)) + } + return bean3.SecretList{ConfigData: configData} +} diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 2e13b2aea4..8642903bda 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -83,10 +83,10 @@ func (r *ConfigProperty) GetIdentifier() ConfigPropertyIdentifier { } type DeploymentAndCmCsConfig struct { - ResourceType bean.ResourceType `json:"resourceType"` - Data json.RawMessage `json:"data"` - VariableSnapshot string `json:"variableSnapshot"` - ResolvedValue string `json:"resolvedValue"` + ResourceType bean.ResourceType `json:"resourceType"` + Data json.RawMessage `json:"data"` + VariableSnapshot map[string]map[string]string `json:"variableSnapshot"` // for deployment->{Deployment Template: resolvedValuesMap}, for cm->{cmComponentName: resolvedValuesMap} + ResolvedValue string `json:"resolvedValue"` } func NewDeploymentAndCmCsConfig() *DeploymentAndCmCsConfig { @@ -103,6 +103,16 @@ func (r *DeploymentAndCmCsConfig) WithConfigData(data json.RawMessage) *Deployme return r } +func (r *DeploymentAndCmCsConfig) WithVariableSnapshot(snapshot map[string]map[string]string) *DeploymentAndCmCsConfig { + r.VariableSnapshot = snapshot + return r +} + +func (r *DeploymentAndCmCsConfig) WithResolvedValue(resolvedValue string) *DeploymentAndCmCsConfig { + r.ResolvedValue = resolvedValue + return r +} + type DeploymentAndCmCsConfigDto struct { DeploymentTemplate *DeploymentAndCmCsConfig `json:"deploymentTemplate"` ConfigMapsData *DeploymentAndCmCsConfig `json:"configMapData"` diff --git a/pkg/configDiff/utils/utils.go b/pkg/configDiff/utils/utils.go index 8185993775..62d1272c31 100644 --- a/pkg/configDiff/utils/utils.go +++ b/pkg/configDiff/utils/utils.go @@ -14,3 +14,11 @@ func ConvertToJsonRawMessage(request interface{}) (json.RawMessage, error) { } return r, nil } + +func ConvertToString(req interface{}) (string, error) { + reqByte, err := json.Marshal(req) + if err != nil { + return "", err + } + return string(reqByte), nil +} diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index ed179e1458..bd1ef81821 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -21,6 +21,7 @@ import ( dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" + "github.com/devtron-labs/devtron/pkg/bean" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -225,3 +226,121 @@ func GetSourceCiDownStreamResponse(linkedCIDetails []ciPipeline.LinkedCIDetails, } return response } + +func ConvertConfigDataToPipelineConfigData(r *bean.ConfigData) *pipelineConfigBean.ConfigData { + return &pipelineConfigBean.ConfigData{ + Name: r.Name, + Type: r.Type, + External: r.External, + MountPath: r.MountPath, + Data: r.Data, + DefaultData: r.DefaultData, + DefaultMountPath: r.DefaultMountPath, + Global: r.Global, + ExternalSecretType: r.ExternalSecretType, + ESOSecretData: ConvertESOSecretDataToPipelineESOSecretData(r.ESOSecretData), + DefaultESOSecretData: ConvertESOSecretDataToPipelineESOSecretData(r.DefaultESOSecretData), + ExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.ExternalSecret), + DefaultExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.DefaultExternalSecret), + RoleARN: r.RoleARN, + SubPath: r.SubPath, + FilePermission: r.FilePermission, + Overridden: r.Overridden, + } +} + +func ConvertESOSecretDataToPipelineESOSecretData(r bean.ESOSecretData) pipelineConfigBean.ESOSecretData { + return pipelineConfigBean.ESOSecretData{ + SecretStore: r.SecretStore, + SecretStoreRef: r.SecretStoreRef, + EsoData: ConvertEsoDataToPipelineEsoData(r.EsoData), + RefreshInterval: r.RefreshInterval, + } +} + +func ConvertExternalSecretToPipelineExternalSecret(r []bean.ExternalSecret) []pipelineConfigBean.ExternalSecret { + extSec := make([]pipelineConfigBean.ExternalSecret, 0, len(r)) + for _, item := range r { + newItem := pipelineConfigBean.ExternalSecret{ + Key: item.Key, + Name: item.Name, + Property: item.Property, + IsBinary: item.IsBinary, + } + extSec = append(extSec, newItem) + } + return extSec +} + +func ConvertEsoDataToPipelineEsoData(r []bean.ESOData) []pipelineConfigBean.ESOData { + newEsoData := make([]pipelineConfigBean.ESOData, 0, len(r)) + for _, item := range r { + newItem := pipelineConfigBean.ESOData{ + SecretKey: item.SecretKey, + Key: item.Key, + Property: item.Property, + } + newEsoData = append(newEsoData, newItem) + } + return newEsoData +} + +// reverse adapter for the above adapters + +func ConvertPipelineConfigDataToConfigData(r *pipelineConfigBean.ConfigData) *bean.ConfigData { + return &bean.ConfigData{ + Name: r.Name, + Type: r.Type, + External: r.External, + MountPath: r.MountPath, + Data: r.Data, + DefaultData: r.DefaultData, + DefaultMountPath: r.DefaultMountPath, + Global: r.Global, + ExternalSecretType: r.ExternalSecretType, + ESOSecretData: ConvertPipelineESOSecretDataToESOSecretData(r.ESOSecretData), + DefaultESOSecretData: ConvertPipelineESOSecretDataToESOSecretData(r.DefaultESOSecretData), + ExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.ExternalSecret), + DefaultExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.DefaultExternalSecret), + RoleARN: r.RoleARN, + SubPath: r.SubPath, + FilePermission: r.FilePermission, + Overridden: r.Overridden, + } +} + +func ConvertPipelineESOSecretDataToESOSecretData(r pipelineConfigBean.ESOSecretData) bean.ESOSecretData { + return bean.ESOSecretData{ + SecretStore: r.SecretStore, + SecretStoreRef: r.SecretStoreRef, + EsoData: ConvertPipelineEsoDataToEsoData(r.EsoData), + RefreshInterval: r.RefreshInterval, + } +} + +func ConvertPipelineExternalSecretToExternalSecret(r []pipelineConfigBean.ExternalSecret) []bean.ExternalSecret { + extSec := make([]bean.ExternalSecret, 0, len(r)) + for _, item := range r { + newItem := bean.ExternalSecret{ + Key: item.Key, + Name: item.Name, + Property: item.Property, + IsBinary: item.IsBinary, + } + extSec = append(extSec, newItem) + } + return extSec +} + +func ConvertPipelineEsoDataToEsoData(r []pipelineConfigBean.ESOData) []bean.ESOData { + newEsoData := make([]bean.ESOData, 0, len(r)) + for _, item := range r { + newItem := bean.ESOData{ + SecretKey: item.SecretKey, + Key: item.Key, + Property: item.Property, + } + newEsoData = append(newEsoData, newItem) + } + return newEsoData +} diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index 3630a3aa7e..2f54aa07e7 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -41,6 +41,8 @@ type ESOData struct { Property string `json:"property,omitempty"` } +// there is an adapter written in pkg/bean folder to convert below ConfigData struct to pkg/bean's ConfigData + type ConfigData struct { Name string `json:"name"` Type string `json:"type"` diff --git a/wire_gen.go b/wire_gen.go index c236db97e6..3d05752640 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -949,7 +949,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) if err != nil { return nil, err } From f37964040b6bfd1f8e181c87e9b01b03b8c653fe Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 13 Sep 2024 02:42:51 +0530 Subject: [PATCH 03/32] resolve cm cs data in published config code incorporated --- .../DeploymentConfigurationService.go | 144 ++++++++++++++++-- pkg/configDiff/bean/bean.go | 15 ++ pkg/pipeline/DeploymentConfigService.go | 1 + wire_gen.go | 2 +- 4 files changed, 152 insertions(+), 10 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 46eb911249..45b096b57d 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -5,7 +5,9 @@ import ( "encoding/json" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/util" + bean3 "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/configDiff/adaptor" @@ -17,10 +19,12 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" repository6 "github.com/devtron-labs/devtron/pkg/variables/repository" util2 "github.com/devtron-labs/devtron/util" + "github.com/go-pg/pg" "go.uber.org/zap" "net/http" "strconv" @@ -42,6 +46,8 @@ type DeploymentConfigurationServiceImpl struct { pipelineStrategyHistoryRepository repository3.PipelineStrategyHistoryRepository configMapHistoryRepository repository3.ConfigMapHistoryRepository scopedVariableManager variables.ScopedVariableCMCSManager + configMapRepository chartConfig.ConfigMapRepository + deploymentConfigService pipeline.PipelineDeploymentConfigService } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, @@ -54,6 +60,8 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, pipelineStrategyHistoryRepository repository3.PipelineStrategyHistoryRepository, configMapHistoryRepository repository3.ConfigMapHistoryRepository, scopedVariableManager variables.ScopedVariableCMCSManager, + configMapRepository chartConfig.ConfigMapRepository, + deploymentConfigService pipeline.PipelineDeploymentConfigService, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ logger: logger, @@ -66,6 +74,8 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, pipelineStrategyHistoryRepository: pipelineStrategyHistoryRepository, configMapHistoryRepository: configMapHistoryRepository, scopedVariableManager: scopedVariableManager, + configMapRepository: configMapRepository, + deploymentConfigService: deploymentConfigService, } return deploymentConfigurationService, nil @@ -107,12 +117,15 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con var err error var envId int var appId int + var clusterId int if configDataQueryParams.IsEnvNameProvided() { - envId, err = impl.environmentRepository.FindIdByName(configDataQueryParams.EnvName) + env, err := impl.environmentRepository.FindByName(configDataQueryParams.EnvName) if err != nil { impl.logger.Errorw("GetAllConfigData, error in getting environment model by envName", "envName", configDataQueryParams.EnvName, "err", err) return nil, err } + envId = env.Id + clusterId = env.ClusterId } appId, err = impl.appRepository.FindAppIdByName(configDataQueryParams.AppName) if err != nil { @@ -127,7 +140,7 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) } // this would be the default case - return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId) + return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess) } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { @@ -321,12 +334,12 @@ func (impl *DeploymentConfigurationServiceImpl) encodeSecretDataFromNonAdminUser } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForAppConfiguration(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, - appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + appId, envId, clusterId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { configDataDto := &bean2.DeploymentAndCmCsConfigDto{} var err error switch configDataQueryParams.ConfigType { default: // keeping default as PublishedOnly - configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId) + configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess) if err != nil { impl.logger.Errorw("GetAllConfigData, error in config data for PublishedOnly", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err @@ -371,7 +384,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsEditDataForPublishedOnly( return configDataDto, nil } -func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(envId, appId int) (*bean2.DeploymentAndCmCsConfigDto, error) { +func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { configDataDto := &bean2.DeploymentAndCmCsConfigDto{} secretData, err := impl.getSecretConfigResponse("", 0, envId, appId) @@ -399,14 +412,127 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(e return nil, err } - cmConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(cmRespJson).WithResourceType(bean.CM) - secretConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(secretRespJson).WithResourceType(bean.CS) + resolvedCmCsMetadataDto, err := impl.ResolveCmCs(ctx, envId, appId, clusterId, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("error in resolving cm and cs for published only config only response", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + cmConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(cmRespJson).WithResourceType(bean.CM). + WithResolvedValue(resolvedCmCsMetadataDto.ResolvedConfigMapData).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCM) + + secretConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(secretRespJson).WithResourceType(bean.CS). + WithResolvedValue(resolvedCmCsMetadataDto.ResolvedSecretData).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCS) configDataDto.WithConfigMapData(cmConfigData).WithSecretData(secretConfigData) return configDataDto, nil } +func (impl *DeploymentConfigurationServiceImpl) getMergedCmCs(envId, appId int) (*bean2.CmCsMetadataDto, error) { + configAppLevel, err := impl.configMapRepository.GetByAppIdAppLevel(appId) + if err != nil && pg.ErrNoRows != err { + impl.logger.Errorw("error in getting CM/CS app level data", "appId", appId, "err", err) + return nil, err + } + var configMapAppLevel string + var secretAppLevel string + if configAppLevel != nil && configAppLevel.Id > 0 { + configMapAppLevel = configAppLevel.ConfigMapData + secretAppLevel = configAppLevel.SecretData + } + configEnvLevel, err := impl.configMapRepository.GetByAppIdAndEnvIdEnvLevel(appId, envId) + if err != nil && pg.ErrNoRows != err { + impl.logger.Errorw("error in getting CM/CS env level data", "appId", appId, "envId", envId, "err", err) + return nil, err + } + var configMapEnvLevel string + var secretEnvLevel string + if configEnvLevel != nil && configEnvLevel.Id > 0 { + configMapEnvLevel = configEnvLevel.ConfigMapData + secretEnvLevel = configEnvLevel.SecretData + } + mergedConfigMap, err := impl.deploymentConfigService.GetMergedCMCSConfigMap(configMapAppLevel, configMapEnvLevel, repository3.CONFIGMAP_TYPE) + if err != nil { + impl.logger.Errorw("error in merging app level and env level CM configs", "err", err) + return nil, err + } + + mergedSecret, err := impl.deploymentConfigService.GetMergedCMCSConfigMap(secretAppLevel, secretEnvLevel, repository3.SECRET_TYPE) + if err != nil { + impl.logger.Errorw("error in merging app level and env level CM configs", "err", err) + return nil, err + } + return &bean2.CmCsMetadataDto{ + CmMap: mergedConfigMap, + SecretMap: mergedSecret, + ConfigAppLevelId: configAppLevel.Id, + ConfigEnvLevelId: configEnvLevel.Id, + }, nil +} + +func (impl *DeploymentConfigurationServiceImpl) ResolveCmCs(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool) (*bean2.ResolvedCmCsMetadataDto, error) { + scope := resourceQualifiers.Scope{ + AppId: appId, + EnvId: envId, + ClusterId: clusterId, + } + cmcsMetadataDto, err := impl.getMergedCmCs(envId, appId) + if err != nil { + impl.logger.Errorw("error in getting merged cm cs", "appId", appId, "envId", envId, "err", err) + return nil, err + } + resolvedConfigList, resolvedSecretList, variableMapCM, variableMapCS, err := impl.scopedVariableManager.ResolveCMCS(ctx, scope, cmcsMetadataDto.ConfigAppLevelId, cmcsMetadataDto.ConfigEnvLevelId, cmcsMetadataDto.CmMap, cmcsMetadataDto.SecretMap) + if err != nil { + impl.logger.Errorw("error in resolving CM/CS", "scope", scope, "appId", appId, "envId", envId, "err", err) + return nil, err + } + + resolvedConfigString, resolvedSecretString, err := impl.getStringifiedCmCs(resolvedConfigList, resolvedSecretList, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("error in getStringifiedCmCs", "resolvedConfigList", resolvedConfigList, "err", err) + return nil, err + } + resolvedData := &bean2.ResolvedCmCsMetadataDto{ + VariableMapCM: variableMapCM, + VariableMapCS: variableMapCS, + ResolvedSecretData: resolvedSecretString, + ResolvedConfigMapData: resolvedConfigString, + } + + return resolvedData, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getStringifiedCmCs(resolvedCmMap map[string]*bean3.ConfigData, resolvedSecretMap map[string]*bean3.ConfigData, + userHasAdminAccess bool) (string, string, error) { + + resolvedConfigDataList := make([]*bean.ConfigData, 0, len(resolvedCmMap)) + resolvedSecretDataList := make([]*bean.ConfigData, 0, len(resolvedSecretMap)) + + for _, resolvedConfigData := range resolvedCmMap { + resolvedConfigDataList = append(resolvedConfigDataList, adapter.ConvertConfigDataToPipelineConfigData(resolvedConfigData)) + } + + for _, resolvedSecretData := range resolvedSecretMap { + resolvedSecretDataList = append(resolvedSecretDataList, adapter.ConvertConfigDataToPipelineConfigData(resolvedSecretData)) + } + if len(resolvedSecretMap) > 0 { + impl.encodeSecretDataFromNonAdminUsers(resolvedSecretDataList, userHasAdminAccess) + } + resolvedConfigDataReq := &bean.ConfigDataRequest{ConfigData: resolvedConfigDataList} + resolvedConfigDataString, err := utils.ConvertToString(resolvedConfigDataReq) + if err != nil { + impl.logger.Errorw(" error in converting resolved config data to string", "resolvedConfigDataReq", resolvedConfigDataReq, "err", err) + return "", "", err + } + resolvedSecretDataReq := &bean.ConfigDataRequest{ConfigData: resolvedSecretDataList} + resolvedSecretDataString, err := utils.ConvertToString(resolvedSecretDataReq) + if err != nil { + impl.logger.Errorw(" error in converting resolved config data to string", "err", err) + return "", "", err + } + return resolvedConfigDataString, resolvedSecretDataString, nil +} func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx context.Context, appId, envId int) (json.RawMessage, error) { if envId > 0 { return impl.getDeploymentTemplateForEnvLevel(ctx, appId, envId) @@ -415,13 +541,13 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx } func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, - appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + appId, envId, clusterId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { if configDataQueryParams.IsRequestMadeForOneResource() { return impl.getCmCsEditDataForPublishedOnly(configDataQueryParams, envId, appId) } //ConfigMapsData and SecretsData are populated here - configData, err := impl.getCmCsPublishedConfigResponse(envId, appId) + configData, err := impl.getCmCsPublishedConfigResponse(ctx, envId, appId, clusterId, userHasAdminAccess) if err != nil { impl.logger.Errorw("getPublishedConfigData, error in getting cm cs for PublishedOnly state", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) return nil, err diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 8642903bda..4e75357e04 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -4,6 +4,7 @@ import "C" import ( "encoding/json" "fmt" + bean3 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) @@ -181,3 +182,17 @@ func (r *ConfigDataQueryParams) IsRequestMadeForOneResource() bool { const ( InvalidConfigTypeErr = "invalid config type provided, please send a valid config type" ) + +type CmCsMetadataDto struct { + CmMap map[string]*bean3.ConfigData + SecretMap map[string]*bean3.ConfigData + ConfigAppLevelId int + ConfigEnvLevelId int +} + +type ResolvedCmCsMetadataDto struct { + ResolvedConfigMapData string + ResolvedSecretData string + VariableMapCM map[string]map[string]string + VariableMapCS map[string]map[string]string +} diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index 798f7da2c4..ff699b722a 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -38,6 +38,7 @@ import ( type PipelineDeploymentConfigService interface { GetLatestDeploymentConfigurationByPipelineId(ctx context.Context, pipelineId int, userHasAdminAccess bool) (*history.AllDeploymentConfigurationDetail, error) + GetMergedCMCSConfigMap(appLevelConfig, envLevelConfig string, configType repository2.ConfigType) (map[string]*bean.ConfigData, error) } type PipelineDeploymentConfigServiceImpl struct { diff --git a/wire_gen.go b/wire_gen.go index 3d05752640..49a316fd4e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -949,7 +949,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl) if err != nil { return nil, err } From aa565e0a0c22a63eefdd033edbe09f605fdbedc4 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 16 Sep 2024 14:54:41 +0530 Subject: [PATCH 04/32] wip: bug fixes --- .../DeploymentConfigurationRestHandler.go | 10 ++- .../DeploymentConfigurationService.go | 62 ++++++++++--------- .../DeploymentTemplateService.go | 30 +++++++++ 3 files changed, 71 insertions(+), 31 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index f5a2551f81..3c6876d9d1 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -1,17 +1,20 @@ package restHandler import ( + "context" "fmt" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/configDiff" "github.com/devtron-labs/devtron/pkg/configDiff/bean" + util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" "github.com/gorilla/schema" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" "net/http" + "time" ) type DeploymentConfigurationRestHandler interface { @@ -97,9 +100,12 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp return } //RBAC END + isSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*") userHasAdminAccess := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, object) - - res, err := handler.deploymentConfigurationService.GetAllConfigData(r.Context(), configDataQueryParams, userHasAdminAccess) + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + ctx = util2.SetSuperAdminInContext(ctx, isSuperAdmin) + defer cancel() + res, err := handler.deploymentConfigurationService.GetAllConfigData(ctx, configDataQueryParams, userHasAdminAccess) if err != nil { handler.logger.Errorw("service err, GetAllConfigData ", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 45b096b57d..e80096ed96 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -6,7 +6,6 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/cluster/repository" @@ -26,8 +25,6 @@ import ( util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" - "net/http" - "strconv" ) type DeploymentConfigurationService interface { @@ -111,9 +108,6 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en } func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { - if !configDataQueryParams.IsValidConfigType() { - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: strconv.Itoa(http.StatusBadRequest), InternalMessage: bean2.InvalidConfigTypeErr, UserMessage: bean2.InvalidConfigTypeErr} - } var err error var envId int var appId int @@ -533,11 +527,32 @@ func (impl *DeploymentConfigurationServiceImpl) getStringifiedCmCs(resolvedCmMap } return resolvedConfigDataString, resolvedSecretDataString, nil } -func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx context.Context, appId, envId int) (json.RawMessage, error) { +func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx context.Context, appId, envId int) (*bean2.DeploymentAndCmCsConfig, error) { if envId > 0 { - return impl.getDeploymentTemplateForEnvLevel(ctx, appId, envId) + deplTemplateResp, err := impl.getDeploymentTemplateForEnvLevel(ctx, appId, envId) + if err != nil { + impl.logger.Errorw("error in getting deployment template env level", "err", err) + return nil, err + } + deploymentJson := json.RawMessage{} + err = deploymentJson.UnmarshalJSON([]byte(deplTemplateResp.Data)) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string deploymentTemplateResponse data into json Raw message", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + variableSnapShotMap := make(map[string]map[string]string, len(deplTemplateResp.VariableSnapshot)) + variableSnapShotMap[bean.DeploymentTemplate.ToString()] = deplTemplateResp.VariableSnapshot + + return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deploymentJson).WithResourceType(bean.DeploymentTemplate). + WithResolvedValue(deplTemplateResp.ResolvedData).WithVariableSnapshot(variableSnapShotMap), nil + } + deplJson, err := impl.getBaseDeploymentTemplate(appId) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, getting base depl. template", "appid", appId, "err", err) + return nil, err } - return impl.getBaseDeploymentTemplate(appId) + return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deplJson).WithResourceType(bean.DeploymentTemplate), nil } func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, @@ -552,14 +567,13 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx conte impl.logger.Errorw("getPublishedConfigData, error in getting cm cs for PublishedOnly state", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) return nil, err } - deploymentTemplateJsonData, err := impl.getPublishedDeploymentConfig(ctx, appId, envId) + deploymentTemplateData, err := impl.getPublishedDeploymentConfig(ctx, appId, envId) if err != nil { impl.logger.Errorw("getPublishedConfigData, error in getting publishedOnly deployment config ", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - deploymentConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(deploymentTemplateJsonData).WithResourceType(bean.DeploymentTemplate) - configData.WithDeploymentTemplateData(deploymentConfig) + configData.WithDeploymentTemplateData(deploymentTemplateData) return configData, nil } @@ -569,35 +583,25 @@ func (impl *DeploymentConfigurationServiceImpl) getBaseDeploymentTemplate(appId impl.logger.Errorw("error in getting base deployment template for appId", "appId", appId, "err", err) return nil, err } + return deploymentTemplateData.DefaultAppOverride, nil } -func (impl *DeploymentConfigurationServiceImpl) getDeploymentTemplateForEnvLevel(ctx context.Context, appId, envId int) (json.RawMessage, error) { +func (impl *DeploymentConfigurationServiceImpl) getDeploymentTemplateForEnvLevel(ctx context.Context, appId, envId int) (generateManifest.DeploymentTemplateResponse, error) { deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ AppId: appId, EnvId: envId, RequestDataMode: generateManifest.Values, Type: repository2.PublishedOnEnvironments, } - deploymentTemplateResponse, err := impl.deploymentTemplateService.GetDeploymentTemplate(ctx, deploymentTemplateRequest) + var deploymentTemplateResponse generateManifest.DeploymentTemplateResponse + var err error + deploymentTemplateResponse, err = impl.deploymentTemplateService.GetDeploymentTemplate(ctx, deploymentTemplateRequest) if err != nil { impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in getting deployment template for ", "deploymentTemplateRequest", deploymentTemplateRequest, "err", err) - return nil, err - } - deploymentJson := json.RawMessage{} - err = deploymentJson.UnmarshalJSON([]byte(deploymentTemplateResponse.Data)) - if err != nil { - impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string deploymentTemplateResponse data into json Raw message", "data", deploymentTemplateResponse.Data, "err", err) - return nil, err - } - return deploymentJson, nil -} - -func (impl *DeploymentConfigurationServiceImpl) getDeploymentConfig(ctx context.Context, appId, envId int) (json.RawMessage, error) { - if envId > 0 { - return impl.getDeploymentTemplateForEnvLevel(ctx, appId, envId) + return deploymentTemplateResponse, err } - return impl.getBaseDeploymentTemplate(appId) + return deploymentTemplateResponse, nil } func (impl *DeploymentConfigurationServiceImpl) getSecretConfigResponse(resourceName string, resourceId, envId, appId int) (*bean.ConfigDataRequest, error) { diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index d4a7a2270d..7148c1b4f2 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -62,6 +62,7 @@ type DeploymentTemplateService interface { GetDeploymentTemplate(ctx context.Context, request DeploymentTemplateRequest) (DeploymentTemplateResponse, error) GenerateManifest(ctx context.Context, request *DeploymentTemplateRequest, valuesYaml string) (*openapi2.TemplateChartResponse, error) GetRestartWorkloadData(ctx context.Context, appIds []int, envId int) (*RestartPodResponse, error) + GetDeploymentTemplateWithResolvedData(ctx context.Context, request DeploymentTemplateRequest) (DeploymentTemplateResponse, error) } type DeploymentTemplateServiceImpl struct { Logger *zap.SugaredLogger @@ -243,6 +244,35 @@ func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Cont return result, nil } +func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplateWithResolvedData(ctx context.Context, request DeploymentTemplateRequest) (DeploymentTemplateResponse, error) { + var result DeploymentTemplateResponse + var values, resolvedValue string + var err error + var variableSnapshot map[string]string + + if request.Values != "" { + values = request.Values + resolvedValue, variableSnapshot, err = impl.resolveTemplateVariables(ctx, request.Values, request) + if err != nil { + return result, err + } + } + if request.RequestDataMode == Values { + result.Data = values + result.ResolvedData = resolvedValue + result.VariableSnapshot = variableSnapshot + return result, nil + } + + request = impl.setRequestMetadata(&request) + manifest, err := impl.GenerateManifest(ctx, &request, resolvedValue) + if err != nil { + return result, err + } + result.Data = *manifest.Manifest + return result, nil +} + func (impl DeploymentTemplateServiceImpl) setRequestMetadata(request *DeploymentTemplateRequest) DeploymentTemplateRequest { // set dummy data for templating. // for some case we may not know the envname and pipelinename, so we want to show this dummy data as a placeholder From 5ffb0f8b2801a4f964d8c4ebf26f8adf4b30220f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 17 Sep 2024 16:26:04 +0530 Subject: [PATCH 05/32] wip: new config area to resolve data for given values --- .../DeploymentConfigurationService.go | 19 +++++++++++++++++++ pkg/configDiff/bean/bean.go | 2 ++ .../DeploymentTemplateService.go | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index e80096ed96..5e16ed742c 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -132,11 +132,30 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con return impl.getConfigDataForCdRollback(ctx, configDataQueryParams, userHasAdminAccess) case bean2.DeploymentHistory.ToString(): return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) + case bean2.ResolveData.ToString(): + // this only supports resolution of deployment template data as of now + return impl.getResolvedConfigDataForValues(ctx, configDataQueryParams.Values, appId, envId) } // this would be the default case return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess) } +func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(ctx context.Context, values string, appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { + configDataDto := &bean2.DeploymentAndCmCsConfigDto{} + var err error + deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ + AppId: appId, + EnvId: envId, + RequestDataMode: generateManifest.Values, + } + resolvedTemplate, _, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, values, deploymentTemplateRequest) + if err != nil { + impl.logger.Errorw("error in getting resolved data for cm draft data ", "appid", appId, "err", err) + return nil, err + } + return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(resolvedTemplate)), nil +} + func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { // wfrId is expected in this case to return the expected data return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 4e75357e04..dde456f276 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -32,6 +32,7 @@ const ( AppConfiguration ConfigArea = "AppConfiguration" DeploymentHistory ConfigArea = "DeploymentHistory" CdRollback ConfigArea = "CdRollback" + ResolveData ConfigArea = "ResolveData" ) func (r ConfigArea) ToString() string { @@ -155,6 +156,7 @@ type ConfigDataQueryParams struct { UserId int32 `schema:"-"` WfrId int `schema:"wfrId"` ConfigArea string `schema:"configArea"` + Values string `schema:"values"` } // FilterCriteria []string `schema:"filterCriteria"` diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index 7148c1b4f2..2ee90ab579 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -63,6 +63,7 @@ type DeploymentTemplateService interface { GenerateManifest(ctx context.Context, request *DeploymentTemplateRequest, valuesYaml string) (*openapi2.TemplateChartResponse, error) GetRestartWorkloadData(ctx context.Context, appIds []int, envId int) (*RestartPodResponse, error) GetDeploymentTemplateWithResolvedData(ctx context.Context, request DeploymentTemplateRequest) (DeploymentTemplateResponse, error) + ResolveTemplateVariables(ctx context.Context, values string, request DeploymentTemplateRequest) (string, map[string]string, error) } type DeploymentTemplateServiceImpl struct { Logger *zap.SugaredLogger @@ -587,3 +588,21 @@ func (impl DeploymentTemplateServiceImpl) GetRestartWorkloadData(ctx context.Con } return podResp, nil } + +func (impl DeploymentTemplateServiceImpl) ResolveTemplateVariables(ctx context.Context, values string, request DeploymentTemplateRequest) (string, map[string]string, error) { + + isSuperAdmin, err := util2.GetIsSuperAdminFromContext(ctx) + if err != nil { + return values, nil, err + } + scope, err := impl.extractScopeData(request) + if err != nil { + return values, nil, err + } + maskUnknownVariableForHelmGenerate := request.RequestDataMode == Manifest + resolvedTemplate, variableSnapshot, err := impl.scopedVariableManager.ExtractVariablesAndResolveTemplate(scope, values, parsers.JsonVariableTemplate, isSuperAdmin, maskUnknownVariableForHelmGenerate) + if err != nil { + return values, variableSnapshot, err + } + return resolvedTemplate, variableSnapshot, nil +} From 6d95766104e9653249cc3c589d26a025c91e480c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 17 Sep 2024 17:12:36 +0530 Subject: [PATCH 06/32] wip: fix --- pkg/configDiff/DeploymentConfigurationService.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 5e16ed742c..3bb2e89d9f 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -145,9 +145,11 @@ func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(c var err error deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ AppId: appId, - EnvId: envId, RequestDataMode: generateManifest.Values, } + if envId > 0 { + deploymentTemplateRequest.EnvId = envId + } resolvedTemplate, _, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, values, deploymentTemplateRequest) if err != nil { impl.logger.Errorw("error in getting resolved data for cm draft data ", "appid", appId, "err", err) From 4aad54a0da80ff63e4e2828c9948683c1140d659 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 17 Sep 2024 18:19:24 +0530 Subject: [PATCH 07/32] wip: marshal resolved template before sending back res p --- pkg/configDiff/DeploymentConfigurationService.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 3bb2e89d9f..27e5c224c6 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -152,10 +152,15 @@ func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(c } resolvedTemplate, _, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, values, deploymentTemplateRequest) if err != nil { - impl.logger.Errorw("error in getting resolved data for cm draft data ", "appid", appId, "err", err) + impl.logger.Errorw("error in getting resolved data for cm draft data ", "appId", appId, "err", err) return nil, err } - return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(resolvedTemplate)), nil + resolvedJson, err := json.Marshal(resolvedTemplate) + if err != nil { + impl.logger.Errorw("marshalling resolved deployment template ", "appId", appId, "resolvedTemplate", resolvedTemplate, "err", err) + return nil, err + } + return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(string(resolvedJson))), nil } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { From 711b682bc07f218c43a19ba09efd43b152766b71 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 17 Sep 2024 18:54:55 +0530 Subject: [PATCH 08/32] convert to json raw message for resolved data --- .../DeploymentConfigurationService.go | 43 ++++++++++++++++--- pkg/configDiff/bean/bean.go | 4 +- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 27e5c224c6..8c9eef1df6 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -160,7 +160,12 @@ func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(c impl.logger.Errorw("marshalling resolved deployment template ", "appId", appId, "resolvedTemplate", resolvedTemplate, "err", err) return nil, err } - return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(string(resolvedJson))), nil + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedJson) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "resolvedJson", resolvedJson, "err", err) + return nil, err + } + return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(resolvedConfigDataStringJson)), nil } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { @@ -192,11 +197,16 @@ func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx c if err != nil { impl.logger.Errorw("error while resolving template from history", "deploymentHistoryId", deploymentHistory.Id, "pipelineId", configDataQueryParams.PipelineId, "err", err) } + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedTemplate) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedTemplate", "resolvedTemplate", resolvedTemplate, "err", err) + return nil, err + } deploymentConfig := bean2.NewDeploymentAndCmCsConfig(). WithConfigData(deploymentJson). WithResourceType(bean.DeploymentTemplate). WithVariableSnapshot(map[string]map[string]string{bean.DeploymentTemplate.ToString(): variableSnapshotMap}). - WithResolvedValue(resolvedTemplate) + WithResolvedValue(resolvedConfigDataStringJson) return deploymentConfig, nil } @@ -318,12 +328,16 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err } - + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedConfigDataString) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedConfigDataString", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) + return nil, err + } cmConfigData := bean2.NewDeploymentAndCmCsConfig(). WithConfigData(configDataJson). WithResourceType(resourceType). WithVariableSnapshot(variableSnapshotMap). - WithResolvedValue(resolvedConfigDataString) + WithResolvedValue(resolvedConfigDataStringJson) return cmConfigData, nil } @@ -437,12 +451,22 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(c impl.logger.Errorw("error in resolving cm and cs for published only config only response", "appId", appId, "envId", envId, "err", err) return nil, err } + resolvedConfigMapDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedCmCsMetadataDto.ResolvedConfigMapData) + if err != nil { + impl.logger.Errorw("error in ConvertToJsonRawMessage for resolvedConfigMapDataStringJson", "resolvedCmData", resolvedCmCsMetadataDto.ResolvedConfigMapData, "err", err) + return nil, err + } + resolvedSecretDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedCmCsMetadataDto.ResolvedSecretData) + if err != nil { + impl.logger.Errorw(" error in ConvertToJsonRawMessage for resolvedConfigDataString", "err", err) + return nil, err + } cmConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(cmRespJson).WithResourceType(bean.CM). - WithResolvedValue(resolvedCmCsMetadataDto.ResolvedConfigMapData).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCM) + WithResolvedValue(resolvedConfigMapDataStringJson).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCM) secretConfigData := bean2.NewDeploymentAndCmCsConfig().WithConfigData(secretRespJson).WithResourceType(bean.CS). - WithResolvedValue(resolvedCmCsMetadataDto.ResolvedSecretData).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCS) + WithResolvedValue(resolvedSecretDataStringJson).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCS) configDataDto.WithConfigMapData(cmConfigData).WithSecretData(secretConfigData) return configDataDto, nil @@ -570,8 +594,13 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx variableSnapShotMap := make(map[string]map[string]string, len(deplTemplateResp.VariableSnapshot)) variableSnapShotMap[bean.DeploymentTemplate.ToString()] = deplTemplateResp.VariableSnapshot + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(deplTemplateResp.ResolvedData) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedConfigDataString", "resolvedData", deplTemplateResp.ResolvedData, "err", err) + return nil, err + } return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deploymentJson).WithResourceType(bean.DeploymentTemplate). - WithResolvedValue(deplTemplateResp.ResolvedData).WithVariableSnapshot(variableSnapShotMap), nil + WithResolvedValue(resolvedConfigDataStringJson).WithVariableSnapshot(variableSnapShotMap), nil } deplJson, err := impl.getBaseDeploymentTemplate(appId) if err != nil { diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index dde456f276..1f6805d1b2 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -88,7 +88,7 @@ type DeploymentAndCmCsConfig struct { ResourceType bean.ResourceType `json:"resourceType"` Data json.RawMessage `json:"data"` VariableSnapshot map[string]map[string]string `json:"variableSnapshot"` // for deployment->{Deployment Template: resolvedValuesMap}, for cm->{cmComponentName: resolvedValuesMap} - ResolvedValue string `json:"resolvedValue"` + ResolvedValue json.RawMessage `json:"resolvedValue"` } func NewDeploymentAndCmCsConfig() *DeploymentAndCmCsConfig { @@ -110,7 +110,7 @@ func (r *DeploymentAndCmCsConfig) WithVariableSnapshot(snapshot map[string]map[s return r } -func (r *DeploymentAndCmCsConfig) WithResolvedValue(resolvedValue string) *DeploymentAndCmCsConfig { +func (r *DeploymentAndCmCsConfig) WithResolvedValue(resolvedValue json.RawMessage) *DeploymentAndCmCsConfig { r.ResolvedValue = resolvedValue return r } From d0714dd58e03955c9860434f7d5bcfa30655c37e Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 17 Sep 2024 23:23:39 +0530 Subject: [PATCH 09/32] wip: fix --- pkg/configDiff/DeploymentConfigurationService.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 8c9eef1df6..d632cbf017 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -155,14 +155,9 @@ func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(c impl.logger.Errorw("error in getting resolved data for cm draft data ", "appId", appId, "err", err) return nil, err } - resolvedJson, err := json.Marshal(resolvedTemplate) - if err != nil { - impl.logger.Errorw("marshalling resolved deployment template ", "appId", appId, "resolvedTemplate", resolvedTemplate, "err", err) - return nil, err - } - resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedJson) + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedTemplate) if err != nil { - impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "resolvedJson", resolvedJson, "err", err) + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "resolvedJson", resolvedTemplate, "err", err) return nil, err } return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(resolvedConfigDataStringJson)), nil From a7033a18454387daeda4fc0b0095e5ed5731da86 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 18 Sep 2024 14:01:53 +0530 Subject: [PATCH 10/32] wip: convert to json raw --- .../DeploymentConfigurationService.go | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index d632cbf017..2eac95e67b 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -155,12 +155,8 @@ func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(c impl.logger.Errorw("error in getting resolved data for cm draft data ", "appId", appId, "err", err) return nil, err } - resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedTemplate) - if err != nil { - impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "resolvedJson", resolvedTemplate, "err", err) - return nil, err - } - return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(resolvedConfigDataStringJson)), nil + + return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(json.RawMessage(resolvedTemplate))), nil } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { @@ -192,16 +188,12 @@ func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx c if err != nil { impl.logger.Errorw("error while resolving template from history", "deploymentHistoryId", deploymentHistory.Id, "pipelineId", configDataQueryParams.PipelineId, "err", err) } - resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedTemplate) - if err != nil { - impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedTemplate", "resolvedTemplate", resolvedTemplate, "err", err) - return nil, err - } + deploymentConfig := bean2.NewDeploymentAndCmCsConfig(). WithConfigData(deploymentJson). WithResourceType(bean.DeploymentTemplate). WithVariableSnapshot(map[string]map[string]string{bean.DeploymentTemplate.ToString(): variableSnapshotMap}). - WithResolvedValue(resolvedConfigDataStringJson) + WithResolvedValue(json.RawMessage(resolvedTemplate)) return deploymentConfig, nil } @@ -589,20 +581,27 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx variableSnapShotMap := make(map[string]map[string]string, len(deplTemplateResp.VariableSnapshot)) variableSnapShotMap[bean.DeploymentTemplate.ToString()] = deplTemplateResp.VariableSnapshot - resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(deplTemplateResp.ResolvedData) - if err != nil { - impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedConfigDataString", "resolvedData", deplTemplateResp.ResolvedData, "err", err) - return nil, err - } return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deploymentJson).WithResourceType(bean.DeploymentTemplate). - WithResolvedValue(resolvedConfigDataStringJson).WithVariableSnapshot(variableSnapShotMap), nil + WithResolvedValue(json.RawMessage(deplTemplateResp.ResolvedData)).WithVariableSnapshot(variableSnapShotMap), nil } deplJson, err := impl.getBaseDeploymentTemplate(appId) if err != nil { - impl.logger.Errorw("getDeploymentTemplateForEnvLevel, getting base depl. template", "appid", appId, "err", err) + impl.logger.Errorw("getting base depl. template", "appid", appId, "err", err) + return nil, err + } + deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ + AppId: appId, + RequestDataMode: generateManifest.Values, + } + resolvedTemplate, variableSnapshot, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, string(deplJson), deploymentTemplateRequest) + if err != nil { + impl.logger.Errorw("error in getting resolved data for base deployment template", "appid", appId, "err", err) return nil, err } - return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deplJson).WithResourceType(bean.DeploymentTemplate), nil + + variableSnapShotMap := map[string]map[string]string{bean.DeploymentTemplate.ToString(): variableSnapshot} + return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deplJson).WithResourceType(bean.DeploymentTemplate). + WithResolvedValue(json.RawMessage(resolvedTemplate)).WithVariableSnapshot(variableSnapShotMap), nil } func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, From 28dcbd4d4c6d96eca78a51f196e157e67a20e71d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 18 Sep 2024 18:19:57 +0530 Subject: [PATCH 11/32] wip: take values from path param values --- api/restHandler/DeploymentConfigurationRestHandler.go | 5 ++++- api/router/DeploymentConfigRouter.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 3c6876d9d1..eeb0c2c2ae 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -10,6 +10,7 @@ import ( "github.com/devtron-labs/devtron/pkg/configDiff/bean" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" + "github.com/gorilla/mux" "github.com/gorilla/schema" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" @@ -90,7 +91,9 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - + vars := mux.Vars(r) + values := vars["values"] + configDataQueryParams.Values = values //RBAC START token := r.Header.Get(common.TokenHeaderKey) object := handler.enforcerUtil.GetAppRBACName(configDataQueryParams.AppName) diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index a8a568d604..5dbab24b30 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -26,6 +26,6 @@ func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRoute Methods("GET") configRouter.Path("/data"). HandlerFunc(router.deploymentGroupRestHandler.GetConfigData). - Methods("GET") + Methods("POST") } From 425780f09a9b692bc317630dd7b90afa74f9b939 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 18 Sep 2024 19:03:31 +0530 Subject: [PATCH 12/32] decode values payload --- .../DeploymentConfigurationRestHandler.go | 15 +++++++++++---- pkg/configDiff/bean/bean.go | 4 ++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index eeb0c2c2ae..fd0b06ad56 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -2,6 +2,7 @@ package restHandler import ( "context" + "encoding/json" "fmt" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -10,7 +11,6 @@ import ( "github.com/devtron-labs/devtron/pkg/configDiff/bean" util2 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" - "github.com/gorilla/mux" "github.com/gorilla/schema" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" @@ -91,9 +91,16 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - vars := mux.Vars(r) - values := vars["values"] - configDataQueryParams.Values = values + valuesPayload := &bean.ValuesDto{} + configDataQueryParams.UserId = userId + decoder := json.NewDecoder(r.Body) + err = decoder.Decode(valuesPayload) + if err != nil { + handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + configDataQueryParams.Values = valuesPayload.Values //RBAC START token := r.Header.Get(common.TokenHeaderKey) object := handler.enforcerUtil.GetAppRBACName(configDataQueryParams.AppName) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 1f6805d1b2..e897ce010b 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -198,3 +198,7 @@ type ResolvedCmCsMetadataDto struct { VariableMapCM map[string]map[string]string VariableMapCS map[string]map[string]string } + +type ValuesDto struct { + Values string `json:"values"` +} From fa5217176089a037e863fab4c44284f718781b12 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 18 Sep 2024 22:48:26 +0530 Subject: [PATCH 13/32] if content of post config data is >0 then only parse the payload else not --- .../DeploymentConfigurationRestHandler.go | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index fd0b06ad56..9e853e7dd2 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -91,16 +91,19 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - valuesPayload := &bean.ValuesDto{} - configDataQueryParams.UserId = userId - decoder := json.NewDecoder(r.Body) - err = decoder.Decode(valuesPayload) - if err != nil { - handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return + if r.ContentLength > 0 { + valuesPayload := &bean.ValuesDto{} + decoder := json.NewDecoder(r.Body) + err = decoder.Decode(valuesPayload) + if err != nil { + handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + configDataQueryParams.Values = valuesPayload.Values } - configDataQueryParams.Values = valuesPayload.Values + + configDataQueryParams.UserId = userId //RBAC START token := r.Header.Get(common.TokenHeaderKey) object := handler.enforcerUtil.GetAppRBACName(configDataQueryParams.AppName) From 530301eb9c663f1596d1687ef890a17d7fd164d9 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 23 Sep 2024 00:32:01 +0530 Subject: [PATCH 14/32] add template version and isAppMetrics enabled in case of deployment template and some more metadata added for pipeline strategy --- .../DeploymentConfigurationService.go | 38 +++++++++++---- pkg/configDiff/bean/bean.go | 25 ++++++++++ .../DeploymentTemplateService.go | 48 +++++++++++++++---- pkg/generateManifest/bean.go | 8 ++-- wire_gen.go | 4 +- 5 files changed, 98 insertions(+), 25 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 2eac95e67b..17218aec6b 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -13,6 +13,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" "github.com/devtron-labs/devtron/pkg/configDiff/helper" "github.com/devtron-labs/devtron/pkg/configDiff/utils" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" @@ -45,6 +46,7 @@ type DeploymentConfigurationServiceImpl struct { scopedVariableManager variables.ScopedVariableCMCSManager configMapRepository chartConfig.ConfigMapRepository deploymentConfigService pipeline.PipelineDeploymentConfigService + chartRefService chartRef.ChartRefService } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, @@ -59,6 +61,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, scopedVariableManager variables.ScopedVariableCMCSManager, configMapRepository chartConfig.ConfigMapRepository, deploymentConfigService pipeline.PipelineDeploymentConfigService, + chartRefService chartRef.ChartRefService, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ logger: logger, @@ -73,6 +76,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, scopedVariableManager: scopedVariableManager, configMapRepository: configMapRepository, deploymentConfigService: deploymentConfigService, + chartRefService: chartRefService, } return deploymentConfigurationService, nil @@ -193,7 +197,8 @@ func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx c WithConfigData(deploymentJson). WithResourceType(bean.DeploymentTemplate). WithVariableSnapshot(map[string]map[string]string{bean.DeploymentTemplate.ToString(): variableSnapshotMap}). - WithResolvedValue(json.RawMessage(resolvedTemplate)) + WithResolvedValue(json.RawMessage(resolvedTemplate)). + WithDeploymentConfigMetadata(deploymentHistory.TemplateVersion, deploymentHistory.IsAppMetricsEnabled) return deploymentConfig, nil } @@ -209,7 +214,10 @@ func (impl *DeploymentConfigurationServiceImpl) getPipelineStrategyConfigHistory impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "pipelineStrategyHistoryConfig", pipelineStrategyHistory.Config, "err", err) return nil, err } - pipelineConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(pipelineStrategyJson).WithResourceType(bean.PipelineStrategy) + pipelineConfig := bean2.NewDeploymentAndCmCsConfig(). + WithConfigData(pipelineStrategyJson). + WithResourceType(bean.PipelineStrategy). + WithPipelineStrategyMetadata(pipelineStrategyHistory.PipelineTriggerType, string(pipelineStrategyHistory.Strategy)) return pipelineConfig, nil } @@ -582,9 +590,10 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx variableSnapShotMap[bean.DeploymentTemplate.ToString()] = deplTemplateResp.VariableSnapshot return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deploymentJson).WithResourceType(bean.DeploymentTemplate). - WithResolvedValue(json.RawMessage(deplTemplateResp.ResolvedData)).WithVariableSnapshot(variableSnapShotMap), nil + WithResolvedValue(json.RawMessage(deplTemplateResp.ResolvedData)).WithVariableSnapshot(variableSnapShotMap). + WithDeploymentConfigMetadata(deplTemplateResp.TemplateVersion, deplTemplateResp.IsAppMetricsEnabled), nil } - deplJson, err := impl.getBaseDeploymentTemplate(appId) + deplMetadata, err := impl.getBaseDeploymentTemplate(appId) if err != nil { impl.logger.Errorw("getting base depl. template", "appid", appId, "err", err) return nil, err @@ -593,15 +602,16 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx AppId: appId, RequestDataMode: generateManifest.Values, } - resolvedTemplate, variableSnapshot, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, string(deplJson), deploymentTemplateRequest) + resolvedTemplate, variableSnapshot, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, string(deplMetadata.DeploymentTemplateJson), deploymentTemplateRequest) if err != nil { impl.logger.Errorw("error in getting resolved data for base deployment template", "appid", appId, "err", err) return nil, err } variableSnapShotMap := map[string]map[string]string{bean.DeploymentTemplate.ToString(): variableSnapshot} - return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deplJson).WithResourceType(bean.DeploymentTemplate). - WithResolvedValue(json.RawMessage(resolvedTemplate)).WithVariableSnapshot(variableSnapShotMap), nil + return bean2.NewDeploymentAndCmCsConfig().WithConfigData(deplMetadata.DeploymentTemplateJson).WithResourceType(bean.DeploymentTemplate). + WithResolvedValue(json.RawMessage(resolvedTemplate)).WithVariableSnapshot(variableSnapShotMap). + WithDeploymentConfigMetadata(deplMetadata.TemplateVersion, deplMetadata.IsAppMetricsEnabled), nil } func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, @@ -626,14 +636,22 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx conte return configData, nil } -func (impl *DeploymentConfigurationServiceImpl) getBaseDeploymentTemplate(appId int) (json.RawMessage, error) { +func (impl *DeploymentConfigurationServiceImpl) getBaseDeploymentTemplate(appId int) (*bean2.DeploymentTemplateMetadata, error) { deploymentTemplateData, err := impl.chartService.FindLatestChartForAppByAppId(appId) if err != nil { impl.logger.Errorw("error in getting base deployment template for appId", "appId", appId, "err", err) return nil, err } - - return deploymentTemplateData.DefaultAppOverride, nil + _, _, version, _, err := impl.chartRefService.GetRefChart(deploymentTemplateData.ChartRefId) + if err != nil { + impl.logger.Errorw("error in getting chart ref by chartRefId ", "chartRefId", deploymentTemplateData.ChartRefId, "err", err) + return nil, err + } + return &bean2.DeploymentTemplateMetadata{ + DeploymentTemplateJson: deploymentTemplateData.DefaultAppOverride, + IsAppMetricsEnabled: deploymentTemplateData.IsAppMetricsEnabled, + TemplateVersion: version, + }, nil } func (impl *DeploymentConfigurationServiceImpl) getDeploymentTemplateForEnvLevel(ctx context.Context, appId, envId int) (generateManifest.DeploymentTemplateResponse, error) { diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index e897ce010b..84e07ce257 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -4,6 +4,7 @@ import "C" import ( "encoding/json" "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean3 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) @@ -89,6 +90,12 @@ type DeploymentAndCmCsConfig struct { Data json.RawMessage `json:"data"` VariableSnapshot map[string]map[string]string `json:"variableSnapshot"` // for deployment->{Deployment Template: resolvedValuesMap}, for cm->{cmComponentName: resolvedValuesMap} ResolvedValue json.RawMessage `json:"resolvedValue"` + // for deployment template + TemplateVersion string `json:"templateVersion,omitempty"` + IsAppMetricsEnabled bool `json:"isAppMetricsEnabled,omitempty"` + //for pipeline strategy + PipelineTriggerType pipelineConfig.TriggerType `json:"pipelineTriggerType,omitempty"` + Strategy string `json:"strategy,omitempty"` } func NewDeploymentAndCmCsConfig() *DeploymentAndCmCsConfig { @@ -115,6 +122,18 @@ func (r *DeploymentAndCmCsConfig) WithResolvedValue(resolvedValue json.RawMessag return r } +func (r *DeploymentAndCmCsConfig) WithDeploymentConfigMetadata(templateVersion string, isAppMetricsEnabled bool) *DeploymentAndCmCsConfig { + r.TemplateVersion = templateVersion + r.IsAppMetricsEnabled = isAppMetricsEnabled + return r +} + +func (r *DeploymentAndCmCsConfig) WithPipelineStrategyMetadata(pipelineTriggerType pipelineConfig.TriggerType, strategy string) *DeploymentAndCmCsConfig { + r.PipelineTriggerType = pipelineTriggerType + r.Strategy = strategy + return r +} + type DeploymentAndCmCsConfigDto struct { DeploymentTemplate *DeploymentAndCmCsConfig `json:"deploymentTemplate"` ConfigMapsData *DeploymentAndCmCsConfig `json:"configMapData"` @@ -202,3 +221,9 @@ type ResolvedCmCsMetadataDto struct { type ValuesDto struct { Values string `json:"values"` } + +type DeploymentTemplateMetadata struct { + DeploymentTemplateJson json.RawMessage + TemplateVersion string + IsAppMetricsEnabled bool +} diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index 2ee90ab579..0bb0d6275b 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -203,6 +203,7 @@ func (impl DeploymentTemplateServiceImpl) FetchDeploymentsWithChartRefs(appId in func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Context, request DeploymentTemplateRequest) (DeploymentTemplateResponse, error) { var result DeploymentTemplateResponse + var response *DeploymentTemplateResponse var values, resolvedValue string var err error var variableSnapshot map[string]string @@ -219,9 +220,9 @@ func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Cont _, values, err = impl.chartRefService.GetAppOverrideForDefaultTemplate(request.ChartRefId) resolvedValue = values case repository.PublishedOnEnvironments: - values, resolvedValue, variableSnapshot, err = impl.fetchResolvedTemplateForPublishedEnvs(ctx, request) + response, err = impl.fetchResolvedTemplateForPublishedEnvs(ctx, request) case repository.DeployedOnSelfEnvironment, repository.DeployedOnOtherEnvironment: - values, resolvedValue, variableSnapshot, err = impl.fetchTemplateForDeployedEnv(ctx, request) + response, err = impl.fetchTemplateForDeployedEnv(ctx, request) } if err != nil { impl.Logger.Errorw("error in getting values", "err", err) @@ -233,9 +234,18 @@ func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Cont result.Data = values result.ResolvedData = resolvedValue result.VariableSnapshot = variableSnapshot + if response != nil { + result.Data = response.Data + result.ResolvedData = response.ResolvedData + result.VariableSnapshot = response.VariableSnapshot + result.TemplateVersion = response.TemplateVersion + result.IsAppMetricsEnabled = response.IsAppMetricsEnabled + } return result, nil } - + if variableSnapshot != nil { + result.VariableSnapshot = variableSnapshot + } request = impl.setRequestMetadata(&request) manifest, err := impl.GenerateManifest(ctx, &request, resolvedValue) if err != nil { @@ -312,7 +322,7 @@ func (impl DeploymentTemplateServiceImpl) setRequestMetadata(request *Deployment return *request } -func (impl DeploymentTemplateServiceImpl) fetchResolvedTemplateForPublishedEnvs(ctx context.Context, request DeploymentTemplateRequest) (string, string, map[string]string, error) { +func (impl DeploymentTemplateServiceImpl) fetchResolvedTemplateForPublishedEnvs(ctx context.Context, request DeploymentTemplateRequest) (*DeploymentTemplateResponse, error) { var values string override, err := impl.propertiesConfigService.GetEnvironmentProperties(request.AppId, request.EnvId, request.ChartRefId) if err == nil && override.GlobalConfig != nil { @@ -323,24 +333,42 @@ func (impl DeploymentTemplateServiceImpl) fetchResolvedTemplateForPublishedEnvs( } } else { impl.Logger.Errorw("error in getting overridden values", "err", err) - return "", "", nil, err + return nil, err + } + _, _, version, _, err := impl.chartRefService.GetRefChart(request.ChartRefId) + if err != nil { + impl.Logger.Errorw("error in getting chart ref by chartRefId ", "chartRefId", request.ChartRefId, "err", err) + return nil, err } resolvedTemplate, variableSnapshot, err := impl.resolveTemplateVariables(ctx, values, request) if err != nil { - return values, values, variableSnapshot, err + impl.Logger.Errorw("error in resolving template variables for env override ", "values", values, "err", err) + return nil, err } - return values, resolvedTemplate, variableSnapshot, nil + return &DeploymentTemplateResponse{ + Data: values, + ResolvedData: resolvedTemplate, + VariableSnapshot: variableSnapshot, + TemplateVersion: version, + IsAppMetricsEnabled: *override.AppMetrics, + }, nil } -func (impl DeploymentTemplateServiceImpl) fetchTemplateForDeployedEnv(ctx context.Context, request DeploymentTemplateRequest) (string, string, map[string]string, error) { +func (impl DeploymentTemplateServiceImpl) fetchTemplateForDeployedEnv(ctx context.Context, request DeploymentTemplateRequest) (*DeploymentTemplateResponse, error) { historyObject, err := impl.deploymentTemplateHistoryService.GetHistoryForDeployedTemplateById(ctx, request.DeploymentTemplateHistoryId, request.PipelineId) if err != nil { impl.Logger.Errorw("error in getting deployment template history", "err", err, "id", request.DeploymentTemplateHistoryId, "pipelineId", request.PipelineId) - return "", "", nil, err + return nil, err } //todo Subhashish solve variable leak - return historyObject.CodeEditorValue.Value, historyObject.CodeEditorValue.ResolvedValue, historyObject.CodeEditorValue.VariableSnapshot, nil + return &DeploymentTemplateResponse{ + Data: historyObject.CodeEditorValue.Value, + ResolvedData: historyObject.CodeEditorValue.ResolvedValue, + VariableSnapshot: historyObject.CodeEditorValue.VariableSnapshot, + TemplateVersion: historyObject.TemplateVersion, + IsAppMetricsEnabled: *historyObject.IsAppMetricsEnabled, + }, nil } func (impl DeploymentTemplateServiceImpl) resolveTemplateVariables(ctx context.Context, values string, request DeploymentTemplateRequest) (string, map[string]string, error) { diff --git a/pkg/generateManifest/bean.go b/pkg/generateManifest/bean.go index 124a0cf083..26a465dd3f 100644 --- a/pkg/generateManifest/bean.go +++ b/pkg/generateManifest/bean.go @@ -69,9 +69,11 @@ var ReleaseIdentifier = &gRPC.ReleaseIdentifier{ } type DeploymentTemplateResponse struct { - Data string `json:"data"` - ResolvedData string `json:"resolvedData"` - VariableSnapshot map[string]string `json:"variableSnapshot"` + Data string `json:"data"` + ResolvedData string `json:"resolvedData"` + VariableSnapshot map[string]string `json:"variableSnapshot"` + TemplateVersion string `json:"-"` + IsAppMetricsEnabled bool `json:"-"` } type RestartPodResponse struct { diff --git a/wire_gen.go b/wire_gen.go index 193b737d35..8c581de9e1 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -949,7 +949,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl, chartRefServiceImpl) if err != nil { return nil, err } From 89656bfe8a945643322ffa71abfdaa47e2d72297 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 25 Sep 2024 18:19:26 +0530 Subject: [PATCH 15/32] pipeline strategy in published data req --- .../DeploymentConfigurationService.go | 39 ++++++++++++++++++- pkg/pipeline/DeploymentConfigService.go | 1 + wire_gen.go | 2 +- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 17218aec6b..34bd16b3ec 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -6,6 +6,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean3 "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/cluster/repository" @@ -47,6 +48,7 @@ type DeploymentConfigurationServiceImpl struct { configMapRepository chartConfig.ConfigMapRepository deploymentConfigService pipeline.PipelineDeploymentConfigService chartRefService chartRef.ChartRefService + pipelineRepository pipelineConfig.PipelineRepository } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, @@ -62,6 +64,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, configMapRepository chartConfig.ConfigMapRepository, deploymentConfigService pipeline.PipelineDeploymentConfigService, chartRefService chartRef.ChartRefService, + pipelineRepository pipelineConfig.PipelineRepository, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ logger: logger, @@ -77,6 +80,7 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, configMapRepository: configMapRepository, deploymentConfigService: deploymentConfigService, chartRefService: chartRefService, + pipelineRepository: pipelineRepository, } return deploymentConfigurationService, nil @@ -631,11 +635,44 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx conte impl.logger.Errorw("getPublishedConfigData, error in getting publishedOnly deployment config ", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - configData.WithDeploymentTemplateData(deploymentTemplateData) + + pipelineConfigData, err := impl.getPublishedPipelineStrategyConfig(ctx, appId, envId) + if err != nil { + impl.logger.Errorw("getPublishedConfigData, error in getting publishedOnly pipeline strategy ", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } + configData.WithPipelineConfigData(pipelineConfigData) return configData, nil } +func (impl *DeploymentConfigurationServiceImpl) getPublishedPipelineStrategyConfig(ctx context.Context, appId int, envId int) (*bean2.DeploymentAndCmCsConfig, error) { + pipelineStrategyJson := json.RawMessage{} + pipelineConfig := bean2.NewDeploymentAndCmCsConfig() + if envId == 0 { + return pipelineConfig, nil + } + pipeline, err := impl.pipelineRepository.FindActiveByAppIdAndEnvId(appId, envId) + if err != nil { + impl.logger.Errorw("error in FindActiveByAppIdAndEnvId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + pipelineStrategy, err := impl.deploymentConfigService.GetLatestPipelineStrategyConfig(pipeline) + if err != nil { + impl.logger.Errorw("error in GetLatestPipelineStrategyConfig", "pipelineId", pipeline.Id, "err", err) + return nil, err + } + err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategy.CodeEditorValue.Value)) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "err", err) + return nil, err + } + pipelineConfig.WithConfigData(pipelineStrategyJson). + WithResourceType(bean.PipelineStrategy). + WithPipelineStrategyMetadata(pipelineStrategy.PipelineTriggerType, string(pipelineStrategy.Strategy)) + return pipelineConfig, nil +} + func (impl *DeploymentConfigurationServiceImpl) getBaseDeploymentTemplate(appId int) (*bean2.DeploymentTemplateMetadata, error) { deploymentTemplateData, err := impl.chartService.FindLatestChartForAppByAppId(appId) if err != nil { diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index ff699b722a..bb35e4fc41 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -39,6 +39,7 @@ import ( type PipelineDeploymentConfigService interface { GetLatestDeploymentConfigurationByPipelineId(ctx context.Context, pipelineId int, userHasAdminAccess bool) (*history.AllDeploymentConfigurationDetail, error) GetMergedCMCSConfigMap(appLevelConfig, envLevelConfig string, configType repository2.ConfigType) (map[string]*bean.ConfigData, error) + GetLatestPipelineStrategyConfig(pipeline *pipelineConfig.Pipeline) (*history.HistoryDetailDto, error) } type PipelineDeploymentConfigServiceImpl struct { diff --git a/wire_gen.go b/wire_gen.go index 578b2e3cfe..bb140cf073 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -949,7 +949,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl, chartRefServiceImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl, chartRefServiceImpl, pipelineRepositoryImpl) if err != nil { return nil, err } From 53e102a9d7990afca523b4f20dedc730bcf7a8f9 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 26 Sep 2024 13:49:54 +0530 Subject: [PATCH 16/32] making config/data api backward compatible --- .../DeploymentConfigurationRestHandler.go | 21 +++++++++++-------- api/router/DeploymentConfigRouter.go | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 9e853e7dd2..13926f7fbd 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -91,16 +91,19 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if r.ContentLength > 0 { - valuesPayload := &bean.ValuesDto{} - decoder := json.NewDecoder(r.Body) - err = decoder.Decode(valuesPayload) - if err != nil { - handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return + switch r.Method { + case http.MethodPost: + if r.ContentLength > 0 { + valuesPayload := &bean.ValuesDto{} + decoder := json.NewDecoder(r.Body) + err = decoder.Decode(valuesPayload) + if err != nil { + handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + configDataQueryParams.Values = valuesPayload.Values } - configDataQueryParams.Values = valuesPayload.Values } configDataQueryParams.UserId = userId diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index 5dbab24b30..b442c13424 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -26,6 +26,6 @@ func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRoute Methods("GET") configRouter.Path("/data"). HandlerFunc(router.deploymentGroupRestHandler.GetConfigData). - Methods("POST") + Methods("GET", "POST") } From 060d5d7157d1039b51713c5c474074730e70b806 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 26 Sep 2024 16:33:27 +0530 Subject: [PATCH 17/32] handle for pg no rows --- pkg/configDiff/DeploymentConfigurationService.go | 6 +++++- pkg/configDiff/bean/bean.go | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 34bd16b3ec..58b7155a5d 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -7,6 +7,7 @@ import ( appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" bean3 "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/cluster/repository" @@ -27,6 +28,7 @@ import ( util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" + "net/http" ) type DeploymentConfigurationService interface { @@ -175,9 +177,11 @@ func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx c func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfig, error) { deploymentJson := json.RawMessage{} deploymentHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(configDataQueryParams.PipelineId, configDataQueryParams.WfrId) - if err != nil { + if err != nil && !util.IsErrNoRows(err) { impl.logger.Errorw("error in getting deployment template history for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err + } else if util.IsErrNoRows(err) { + return nil, util.GetApiError(http.StatusNotFound, bean2.NoDeploymentDoneForSelectedImage, bean2.NoDeploymentDoneForSelectedImage) } err = deploymentJson.UnmarshalJSON([]byte(deploymentHistory.Template)) if err != nil { diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 84e07ce257..1ccf303950 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -227,3 +227,7 @@ type DeploymentTemplateMetadata struct { TemplateVersion string IsAppMetricsEnabled bool } + +const ( + NoDeploymentDoneForSelectedImage = "there were no deployments done for the selected image" +) From 7a7627e5420b2d04177835c0bd4c06789af3d439 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 7 Oct 2024 12:34:42 +0530 Subject: [PATCH 18/32] merge develop here --- cmd/external-app/wire_gen.go | 2 +- go.mod | 4 +- go.sum | 2 + pkg/pipeline/adapter/adapter.go | 6 +- vendor/github.com/docker/distribution/LICENSE | 202 ++++++++ .../docker/distribution/digestset/set.go | 247 ++++++++++ .../docker/distribution/reference/helpers.go | 42 ++ .../distribution/reference/normalize.go | 199 ++++++++ .../distribution/reference/reference.go | 433 ++++++++++++++++++ .../docker/distribution/reference/regexp.go | 143 ++++++ vendor/modules.txt | 4 + 11 files changed, 1280 insertions(+), 4 deletions(-) create mode 100644 vendor/github.com/docker/distribution/LICENSE create mode 100644 vendor/github.com/docker/distribution/digestset/set.go create mode 100644 vendor/github.com/docker/distribution/reference/helpers.go create mode 100644 vendor/github.com/docker/distribution/reference/normalize.go create mode 100644 vendor/github.com/docker/distribution/reference/reference.go create mode 100644 vendor/github.com/docker/distribution/reference/regexp.go diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 9c1d691030..db064d11f4 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -317,7 +317,7 @@ func InitializeApp() (*App, error) { environmentRestHandlerImpl := cluster2.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceImpl, k8sServiceImpl, k8sCommonServiceImpl) environmentRouterImpl := cluster2.NewEnvironmentRouterImpl(environmentRestHandlerImpl) ciPipelineRepositoryImpl := pipelineConfig.NewCiPipelineRepositoryImpl(db, sugaredLogger, transactionUtilImpl) - enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl) + enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl, enforcerImpl, dbMigrationServiceImpl) k8sApplicationRestHandlerImpl := application2.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables, fluxApplicationServiceImpl, argoApplicationReadServiceImpl) k8sApplicationRouterImpl := application2.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceImpl, attributesServiceImpl) diff --git a/go.mod b/go.mod index 28b83cf7db..07f32cd764 100644 --- a/go.mod +++ b/go.mod @@ -212,7 +212,7 @@ require ( github.com/nats-io/nats.go v1.28.0 // indirect github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -282,6 +282,8 @@ require ( xorm.io/xorm v1.0.3 // indirect ) +require github.com/docker/distribution v2.8.2+incompatible + require gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect replace ( diff --git a/go.sum b/go.sum index 9f48ec4b43..ff05b0d06f 100644 --- a/go.sum +++ b/go.sum @@ -807,6 +807,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index bd1ef81821..feb6991b8c 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -244,6 +244,7 @@ func ConvertConfigDataToPipelineConfigData(r *bean.ConfigData) *pipelineConfigBe DefaultExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.DefaultExternalSecret), RoleARN: r.RoleARN, SubPath: r.SubPath, + ESOSubPath: r.ESOSubPath, FilePermission: r.FilePermission, Overridden: r.Overridden, } @@ -253,7 +254,7 @@ func ConvertESOSecretDataToPipelineESOSecretData(r bean.ESOSecretData) pipelineC return pipelineConfigBean.ESOSecretData{ SecretStore: r.SecretStore, SecretStoreRef: r.SecretStoreRef, - EsoData: ConvertEsoDataToPipelineEsoData(r.EsoData), + ESOData: ConvertEsoDataToPipelineEsoData(r.ESOData), RefreshInterval: r.RefreshInterval, } } @@ -304,6 +305,7 @@ func ConvertPipelineConfigDataToConfigData(r *pipelineConfigBean.ConfigData) *be DefaultExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.DefaultExternalSecret), RoleARN: r.RoleARN, SubPath: r.SubPath, + ESOSubPath: r.ESOSubPath, FilePermission: r.FilePermission, Overridden: r.Overridden, } @@ -313,7 +315,7 @@ func ConvertPipelineESOSecretDataToESOSecretData(r pipelineConfigBean.ESOSecretD return bean.ESOSecretData{ SecretStore: r.SecretStore, SecretStoreRef: r.SecretStoreRef, - EsoData: ConvertPipelineEsoDataToEsoData(r.EsoData), + ESOData: ConvertPipelineEsoDataToEsoData(r.ESOData), RefreshInterval: r.RefreshInterval, } } diff --git a/vendor/github.com/docker/distribution/LICENSE b/vendor/github.com/docker/distribution/LICENSE new file mode 100644 index 0000000000..e06d208186 --- /dev/null +++ b/vendor/github.com/docker/distribution/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/docker/distribution/digestset/set.go b/vendor/github.com/docker/distribution/digestset/set.go new file mode 100644 index 0000000000..71327dca72 --- /dev/null +++ b/vendor/github.com/docker/distribution/digestset/set.go @@ -0,0 +1,247 @@ +package digestset + +import ( + "errors" + "sort" + "strings" + "sync" + + digest "github.com/opencontainers/go-digest" +) + +var ( + // ErrDigestNotFound is used when a matching digest + // could not be found in a set. + ErrDigestNotFound = errors.New("digest not found") + + // ErrDigestAmbiguous is used when multiple digests + // are found in a set. None of the matching digests + // should be considered valid matches. + ErrDigestAmbiguous = errors.New("ambiguous digest string") +) + +// Set is used to hold a unique set of digests which +// may be easily referenced by easily referenced by a string +// representation of the digest as well as short representation. +// The uniqueness of the short representation is based on other +// digests in the set. If digests are omitted from this set, +// collisions in a larger set may not be detected, therefore it +// is important to always do short representation lookups on +// the complete set of digests. To mitigate collisions, an +// appropriately long short code should be used. +type Set struct { + mutex sync.RWMutex + entries digestEntries +} + +// NewSet creates an empty set of digests +// which may have digests added. +func NewSet() *Set { + return &Set{ + entries: digestEntries{}, + } +} + +// checkShortMatch checks whether two digests match as either whole +// values or short values. This function does not test equality, +// rather whether the second value could match against the first +// value. +func checkShortMatch(alg digest.Algorithm, hex, shortAlg, shortHex string) bool { + if len(hex) == len(shortHex) { + if hex != shortHex { + return false + } + if len(shortAlg) > 0 && string(alg) != shortAlg { + return false + } + } else if !strings.HasPrefix(hex, shortHex) { + return false + } else if len(shortAlg) > 0 && string(alg) != shortAlg { + return false + } + return true +} + +// Lookup looks for a digest matching the given string representation. +// If no digests could be found ErrDigestNotFound will be returned +// with an empty digest value. If multiple matches are found +// ErrDigestAmbiguous will be returned with an empty digest value. +func (dst *Set) Lookup(d string) (digest.Digest, error) { + dst.mutex.RLock() + defer dst.mutex.RUnlock() + if len(dst.entries) == 0 { + return "", ErrDigestNotFound + } + var ( + searchFunc func(int) bool + alg digest.Algorithm + hex string + ) + dgst, err := digest.Parse(d) + if err == digest.ErrDigestInvalidFormat { + hex = d + searchFunc = func(i int) bool { + return dst.entries[i].val >= d + } + } else { + hex = dgst.Hex() + alg = dgst.Algorithm() + searchFunc = func(i int) bool { + if dst.entries[i].val == hex { + return dst.entries[i].alg >= alg + } + return dst.entries[i].val >= hex + } + } + idx := sort.Search(len(dst.entries), searchFunc) + if idx == len(dst.entries) || !checkShortMatch(dst.entries[idx].alg, dst.entries[idx].val, string(alg), hex) { + return "", ErrDigestNotFound + } + if dst.entries[idx].alg == alg && dst.entries[idx].val == hex { + return dst.entries[idx].digest, nil + } + if idx+1 < len(dst.entries) && checkShortMatch(dst.entries[idx+1].alg, dst.entries[idx+1].val, string(alg), hex) { + return "", ErrDigestAmbiguous + } + + return dst.entries[idx].digest, nil +} + +// Add adds the given digest to the set. An error will be returned +// if the given digest is invalid. If the digest already exists in the +// set, this operation will be a no-op. +func (dst *Set) Add(d digest.Digest) error { + if err := d.Validate(); err != nil { + return err + } + dst.mutex.Lock() + defer dst.mutex.Unlock() + entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d} + searchFunc := func(i int) bool { + if dst.entries[i].val == entry.val { + return dst.entries[i].alg >= entry.alg + } + return dst.entries[i].val >= entry.val + } + idx := sort.Search(len(dst.entries), searchFunc) + if idx == len(dst.entries) { + dst.entries = append(dst.entries, entry) + return nil + } else if dst.entries[idx].digest == d { + return nil + } + + entries := append(dst.entries, nil) + copy(entries[idx+1:], entries[idx:len(entries)-1]) + entries[idx] = entry + dst.entries = entries + return nil +} + +// Remove removes the given digest from the set. An err will be +// returned if the given digest is invalid. If the digest does +// not exist in the set, this operation will be a no-op. +func (dst *Set) Remove(d digest.Digest) error { + if err := d.Validate(); err != nil { + return err + } + dst.mutex.Lock() + defer dst.mutex.Unlock() + entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d} + searchFunc := func(i int) bool { + if dst.entries[i].val == entry.val { + return dst.entries[i].alg >= entry.alg + } + return dst.entries[i].val >= entry.val + } + idx := sort.Search(len(dst.entries), searchFunc) + // Not found if idx is after or value at idx is not digest + if idx == len(dst.entries) || dst.entries[idx].digest != d { + return nil + } + + entries := dst.entries + copy(entries[idx:], entries[idx+1:]) + entries = entries[:len(entries)-1] + dst.entries = entries + + return nil +} + +// All returns all the digests in the set +func (dst *Set) All() []digest.Digest { + dst.mutex.RLock() + defer dst.mutex.RUnlock() + retValues := make([]digest.Digest, len(dst.entries)) + for i := range dst.entries { + retValues[i] = dst.entries[i].digest + } + + return retValues +} + +// ShortCodeTable returns a map of Digest to unique short codes. The +// length represents the minimum value, the maximum length may be the +// entire value of digest if uniqueness cannot be achieved without the +// full value. This function will attempt to make short codes as short +// as possible to be unique. +func ShortCodeTable(dst *Set, length int) map[digest.Digest]string { + dst.mutex.RLock() + defer dst.mutex.RUnlock() + m := make(map[digest.Digest]string, len(dst.entries)) + l := length + resetIdx := 0 + for i := 0; i < len(dst.entries); i++ { + var short string + extended := true + for extended { + extended = false + if len(dst.entries[i].val) <= l { + short = dst.entries[i].digest.String() + } else { + short = dst.entries[i].val[:l] + for j := i + 1; j < len(dst.entries); j++ { + if checkShortMatch(dst.entries[j].alg, dst.entries[j].val, "", short) { + if j > resetIdx { + resetIdx = j + } + extended = true + } else { + break + } + } + if extended { + l++ + } + } + } + m[dst.entries[i].digest] = short + if i >= resetIdx { + l = length + } + } + return m +} + +type digestEntry struct { + alg digest.Algorithm + val string + digest digest.Digest +} + +type digestEntries []*digestEntry + +func (d digestEntries) Len() int { + return len(d) +} + +func (d digestEntries) Less(i, j int) bool { + if d[i].val != d[j].val { + return d[i].val < d[j].val + } + return d[i].alg < d[j].alg +} + +func (d digestEntries) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} diff --git a/vendor/github.com/docker/distribution/reference/helpers.go b/vendor/github.com/docker/distribution/reference/helpers.go new file mode 100644 index 0000000000..978df7eabb --- /dev/null +++ b/vendor/github.com/docker/distribution/reference/helpers.go @@ -0,0 +1,42 @@ +package reference + +import "path" + +// IsNameOnly returns true if reference only contains a repo name. +func IsNameOnly(ref Named) bool { + if _, ok := ref.(NamedTagged); ok { + return false + } + if _, ok := ref.(Canonical); ok { + return false + } + return true +} + +// FamiliarName returns the familiar name string +// for the given named, familiarizing if needed. +func FamiliarName(ref Named) string { + if nn, ok := ref.(normalizedNamed); ok { + return nn.Familiar().Name() + } + return ref.Name() +} + +// FamiliarString returns the familiar string representation +// for the given reference, familiarizing if needed. +func FamiliarString(ref Reference) string { + if nn, ok := ref.(normalizedNamed); ok { + return nn.Familiar().String() + } + return ref.String() +} + +// FamiliarMatch reports whether ref matches the specified pattern. +// See https://godoc.org/path#Match for supported patterns. +func FamiliarMatch(pattern string, ref Reference) (bool, error) { + matched, err := path.Match(pattern, FamiliarString(ref)) + if namedRef, isNamed := ref.(Named); isNamed && !matched { + matched, _ = path.Match(pattern, FamiliarName(namedRef)) + } + return matched, err +} diff --git a/vendor/github.com/docker/distribution/reference/normalize.go b/vendor/github.com/docker/distribution/reference/normalize.go new file mode 100644 index 0000000000..b3dfb7a6d7 --- /dev/null +++ b/vendor/github.com/docker/distribution/reference/normalize.go @@ -0,0 +1,199 @@ +package reference + +import ( + "errors" + "fmt" + "strings" + + "github.com/docker/distribution/digestset" + "github.com/opencontainers/go-digest" +) + +var ( + legacyDefaultDomain = "index.docker.io" + defaultDomain = "docker.io" + officialRepoName = "library" + defaultTag = "latest" +) + +// normalizedNamed represents a name which has been +// normalized and has a familiar form. A familiar name +// is what is used in Docker UI. An example normalized +// name is "docker.io/library/ubuntu" and corresponding +// familiar name of "ubuntu". +type normalizedNamed interface { + Named + Familiar() Named +} + +// ParseNormalizedNamed parses a string into a named reference +// transforming a familiar name from Docker UI to a fully +// qualified reference. If the value may be an identifier +// use ParseAnyReference. +func ParseNormalizedNamed(s string) (Named, error) { + if ok := anchoredIdentifierRegexp.MatchString(s); ok { + return nil, fmt.Errorf("invalid repository name (%s), cannot specify 64-byte hexadecimal strings", s) + } + domain, remainder := splitDockerDomain(s) + var remoteName string + if tagSep := strings.IndexRune(remainder, ':'); tagSep > -1 { + remoteName = remainder[:tagSep] + } else { + remoteName = remainder + } + if strings.ToLower(remoteName) != remoteName { + return nil, errors.New("invalid reference format: repository name must be lowercase") + } + + ref, err := Parse(domain + "/" + remainder) + if err != nil { + return nil, err + } + named, isNamed := ref.(Named) + if !isNamed { + return nil, fmt.Errorf("reference %s has no name", ref.String()) + } + return named, nil +} + +// ParseDockerRef normalizes the image reference following the docker convention. This is added +// mainly for backward compatibility. +// The reference returned can only be either tagged or digested. For reference contains both tag +// and digest, the function returns digested reference, e.g. docker.io/library/busybox:latest@ +// sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa will be returned as +// docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa. +func ParseDockerRef(ref string) (Named, error) { + named, err := ParseNormalizedNamed(ref) + if err != nil { + return nil, err + } + if _, ok := named.(NamedTagged); ok { + if canonical, ok := named.(Canonical); ok { + // The reference is both tagged and digested, only + // return digested. + newNamed, err := WithName(canonical.Name()) + if err != nil { + return nil, err + } + newCanonical, err := WithDigest(newNamed, canonical.Digest()) + if err != nil { + return nil, err + } + return newCanonical, nil + } + } + return TagNameOnly(named), nil +} + +// splitDockerDomain splits a repository name to domain and remotename string. +// If no valid domain is found, the default domain is used. Repository name +// needs to be already validated before. +func splitDockerDomain(name string) (domain, remainder string) { + i := strings.IndexRune(name, '/') + if i == -1 || (!strings.ContainsAny(name[:i], ".:") && name[:i] != "localhost") { + domain, remainder = defaultDomain, name + } else { + domain, remainder = name[:i], name[i+1:] + } + if domain == legacyDefaultDomain { + domain = defaultDomain + } + if domain == defaultDomain && !strings.ContainsRune(remainder, '/') { + remainder = officialRepoName + "/" + remainder + } + return +} + +// familiarizeName returns a shortened version of the name familiar +// to to the Docker UI. Familiar names have the default domain +// "docker.io" and "library/" repository prefix removed. +// For example, "docker.io/library/redis" will have the familiar +// name "redis" and "docker.io/dmcgowan/myapp" will be "dmcgowan/myapp". +// Returns a familiarized named only reference. +func familiarizeName(named namedRepository) repository { + repo := repository{ + domain: named.Domain(), + path: named.Path(), + } + + if repo.domain == defaultDomain { + repo.domain = "" + // Handle official repositories which have the pattern "library/" + if split := strings.Split(repo.path, "/"); len(split) == 2 && split[0] == officialRepoName { + repo.path = split[1] + } + } + return repo +} + +func (r reference) Familiar() Named { + return reference{ + namedRepository: familiarizeName(r.namedRepository), + tag: r.tag, + digest: r.digest, + } +} + +func (r repository) Familiar() Named { + return familiarizeName(r) +} + +func (t taggedReference) Familiar() Named { + return taggedReference{ + namedRepository: familiarizeName(t.namedRepository), + tag: t.tag, + } +} + +func (c canonicalReference) Familiar() Named { + return canonicalReference{ + namedRepository: familiarizeName(c.namedRepository), + digest: c.digest, + } +} + +// TagNameOnly adds the default tag "latest" to a reference if it only has +// a repo name. +func TagNameOnly(ref Named) Named { + if IsNameOnly(ref) { + namedTagged, err := WithTag(ref, defaultTag) + if err != nil { + // Default tag must be valid, to create a NamedTagged + // type with non-validated input the WithTag function + // should be used instead + panic(err) + } + return namedTagged + } + return ref +} + +// ParseAnyReference parses a reference string as a possible identifier, +// full digest, or familiar name. +func ParseAnyReference(ref string) (Reference, error) { + if ok := anchoredIdentifierRegexp.MatchString(ref); ok { + return digestReference("sha256:" + ref), nil + } + if dgst, err := digest.Parse(ref); err == nil { + return digestReference(dgst), nil + } + + return ParseNormalizedNamed(ref) +} + +// ParseAnyReferenceWithSet parses a reference string as a possible short +// identifier to be matched in a digest set, a full digest, or familiar name. +func ParseAnyReferenceWithSet(ref string, ds *digestset.Set) (Reference, error) { + if ok := anchoredShortIdentifierRegexp.MatchString(ref); ok { + dgst, err := ds.Lookup(ref) + if err == nil { + return digestReference(dgst), nil + } + } else { + if dgst, err := digest.Parse(ref); err == nil { + return digestReference(dgst), nil + } + } + + return ParseNormalizedNamed(ref) +} diff --git a/vendor/github.com/docker/distribution/reference/reference.go b/vendor/github.com/docker/distribution/reference/reference.go new file mode 100644 index 0000000000..b7cd00b0d6 --- /dev/null +++ b/vendor/github.com/docker/distribution/reference/reference.go @@ -0,0 +1,433 @@ +// Package reference provides a general type to represent any way of referencing images within the registry. +// Its main purpose is to abstract tags and digests (content-addressable hash). +// +// Grammar +// +// reference := name [ ":" tag ] [ "@" digest ] +// name := [domain '/'] path-component ['/' path-component]* +// domain := domain-component ['.' domain-component]* [':' port-number] +// domain-component := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/ +// port-number := /[0-9]+/ +// path-component := alpha-numeric [separator alpha-numeric]* +// alpha-numeric := /[a-z0-9]+/ +// separator := /[_.]|__|[-]*/ +// +// tag := /[\w][\w.-]{0,127}/ +// +// digest := digest-algorithm ":" digest-hex +// digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]* +// digest-algorithm-separator := /[+.-_]/ +// digest-algorithm-component := /[A-Za-z][A-Za-z0-9]*/ +// digest-hex := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value +// +// identifier := /[a-f0-9]{64}/ +// short-identifier := /[a-f0-9]{6,64}/ +package reference + +import ( + "errors" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" +) + +const ( + // NameTotalLengthMax is the maximum total number of characters in a repository name. + NameTotalLengthMax = 255 +) + +var ( + // ErrReferenceInvalidFormat represents an error while trying to parse a string as a reference. + ErrReferenceInvalidFormat = errors.New("invalid reference format") + + // ErrTagInvalidFormat represents an error while trying to parse a string as a tag. + ErrTagInvalidFormat = errors.New("invalid tag format") + + // ErrDigestInvalidFormat represents an error while trying to parse a string as a tag. + ErrDigestInvalidFormat = errors.New("invalid digest format") + + // ErrNameContainsUppercase is returned for invalid repository names that contain uppercase characters. + ErrNameContainsUppercase = errors.New("repository name must be lowercase") + + // ErrNameEmpty is returned for empty, invalid repository names. + ErrNameEmpty = errors.New("repository name must have at least one component") + + // ErrNameTooLong is returned when a repository name is longer than NameTotalLengthMax. + ErrNameTooLong = fmt.Errorf("repository name must not be more than %v characters", NameTotalLengthMax) + + // ErrNameNotCanonical is returned when a name is not canonical. + ErrNameNotCanonical = errors.New("repository name must be canonical") +) + +// Reference is an opaque object reference identifier that may include +// modifiers such as a hostname, name, tag, and digest. +type Reference interface { + // String returns the full reference + String() string +} + +// Field provides a wrapper type for resolving correct reference types when +// working with encoding. +type Field struct { + reference Reference +} + +// AsField wraps a reference in a Field for encoding. +func AsField(reference Reference) Field { + return Field{reference} +} + +// Reference unwraps the reference type from the field to +// return the Reference object. This object should be +// of the appropriate type to further check for different +// reference types. +func (f Field) Reference() Reference { + return f.reference +} + +// MarshalText serializes the field to byte text which +// is the string of the reference. +func (f Field) MarshalText() (p []byte, err error) { + return []byte(f.reference.String()), nil +} + +// UnmarshalText parses text bytes by invoking the +// reference parser to ensure the appropriately +// typed reference object is wrapped by field. +func (f *Field) UnmarshalText(p []byte) error { + r, err := Parse(string(p)) + if err != nil { + return err + } + + f.reference = r + return nil +} + +// Named is an object with a full name +type Named interface { + Reference + Name() string +} + +// Tagged is an object which has a tag +type Tagged interface { + Reference + Tag() string +} + +// NamedTagged is an object including a name and tag. +type NamedTagged interface { + Named + Tag() string +} + +// Digested is an object which has a digest +// in which it can be referenced by +type Digested interface { + Reference + Digest() digest.Digest +} + +// Canonical reference is an object with a fully unique +// name including a name with domain and digest +type Canonical interface { + Named + Digest() digest.Digest +} + +// namedRepository is a reference to a repository with a name. +// A namedRepository has both domain and path components. +type namedRepository interface { + Named + Domain() string + Path() string +} + +// Domain returns the domain part of the Named reference +func Domain(named Named) string { + if r, ok := named.(namedRepository); ok { + return r.Domain() + } + domain, _ := splitDomain(named.Name()) + return domain +} + +// Path returns the name without the domain part of the Named reference +func Path(named Named) (name string) { + if r, ok := named.(namedRepository); ok { + return r.Path() + } + _, path := splitDomain(named.Name()) + return path +} + +func splitDomain(name string) (string, string) { + match := anchoredNameRegexp.FindStringSubmatch(name) + if len(match) != 3 { + return "", name + } + return match[1], match[2] +} + +// SplitHostname splits a named reference into a +// hostname and name string. If no valid hostname is +// found, the hostname is empty and the full value +// is returned as name +// DEPRECATED: Use Domain or Path +func SplitHostname(named Named) (string, string) { + if r, ok := named.(namedRepository); ok { + return r.Domain(), r.Path() + } + return splitDomain(named.Name()) +} + +// Parse parses s and returns a syntactically valid Reference. +// If an error was encountered it is returned, along with a nil Reference. +// NOTE: Parse will not handle short digests. +func Parse(s string) (Reference, error) { + matches := ReferenceRegexp.FindStringSubmatch(s) + if matches == nil { + if s == "" { + return nil, ErrNameEmpty + } + if ReferenceRegexp.FindStringSubmatch(strings.ToLower(s)) != nil { + return nil, ErrNameContainsUppercase + } + return nil, ErrReferenceInvalidFormat + } + + if len(matches[1]) > NameTotalLengthMax { + return nil, ErrNameTooLong + } + + var repo repository + + nameMatch := anchoredNameRegexp.FindStringSubmatch(matches[1]) + if len(nameMatch) == 3 { + repo.domain = nameMatch[1] + repo.path = nameMatch[2] + } else { + repo.domain = "" + repo.path = matches[1] + } + + ref := reference{ + namedRepository: repo, + tag: matches[2], + } + if matches[3] != "" { + var err error + ref.digest, err = digest.Parse(matches[3]) + if err != nil { + return nil, err + } + } + + r := getBestReferenceType(ref) + if r == nil { + return nil, ErrNameEmpty + } + + return r, nil +} + +// ParseNamed parses s and returns a syntactically valid reference implementing +// the Named interface. The reference must have a name and be in the canonical +// form, otherwise an error is returned. +// If an error was encountered it is returned, along with a nil Reference. +// NOTE: ParseNamed will not handle short digests. +func ParseNamed(s string) (Named, error) { + named, err := ParseNormalizedNamed(s) + if err != nil { + return nil, err + } + if named.String() != s { + return nil, ErrNameNotCanonical + } + return named, nil +} + +// WithName returns a named object representing the given string. If the input +// is invalid ErrReferenceInvalidFormat will be returned. +func WithName(name string) (Named, error) { + if len(name) > NameTotalLengthMax { + return nil, ErrNameTooLong + } + + match := anchoredNameRegexp.FindStringSubmatch(name) + if match == nil || len(match) != 3 { + return nil, ErrReferenceInvalidFormat + } + return repository{ + domain: match[1], + path: match[2], + }, nil +} + +// WithTag combines the name from "name" and the tag from "tag" to form a +// reference incorporating both the name and the tag. +func WithTag(name Named, tag string) (NamedTagged, error) { + if !anchoredTagRegexp.MatchString(tag) { + return nil, ErrTagInvalidFormat + } + var repo repository + if r, ok := name.(namedRepository); ok { + repo.domain = r.Domain() + repo.path = r.Path() + } else { + repo.path = name.Name() + } + if canonical, ok := name.(Canonical); ok { + return reference{ + namedRepository: repo, + tag: tag, + digest: canonical.Digest(), + }, nil + } + return taggedReference{ + namedRepository: repo, + tag: tag, + }, nil +} + +// WithDigest combines the name from "name" and the digest from "digest" to form +// a reference incorporating both the name and the digest. +func WithDigest(name Named, digest digest.Digest) (Canonical, error) { + if !anchoredDigestRegexp.MatchString(digest.String()) { + return nil, ErrDigestInvalidFormat + } + var repo repository + if r, ok := name.(namedRepository); ok { + repo.domain = r.Domain() + repo.path = r.Path() + } else { + repo.path = name.Name() + } + if tagged, ok := name.(Tagged); ok { + return reference{ + namedRepository: repo, + tag: tagged.Tag(), + digest: digest, + }, nil + } + return canonicalReference{ + namedRepository: repo, + digest: digest, + }, nil +} + +// TrimNamed removes any tag or digest from the named reference. +func TrimNamed(ref Named) Named { + domain, path := SplitHostname(ref) + return repository{ + domain: domain, + path: path, + } +} + +func getBestReferenceType(ref reference) Reference { + if ref.Name() == "" { + // Allow digest only references + if ref.digest != "" { + return digestReference(ref.digest) + } + return nil + } + if ref.tag == "" { + if ref.digest != "" { + return canonicalReference{ + namedRepository: ref.namedRepository, + digest: ref.digest, + } + } + return ref.namedRepository + } + if ref.digest == "" { + return taggedReference{ + namedRepository: ref.namedRepository, + tag: ref.tag, + } + } + + return ref +} + +type reference struct { + namedRepository + tag string + digest digest.Digest +} + +func (r reference) String() string { + return r.Name() + ":" + r.tag + "@" + r.digest.String() +} + +func (r reference) Tag() string { + return r.tag +} + +func (r reference) Digest() digest.Digest { + return r.digest +} + +type repository struct { + domain string + path string +} + +func (r repository) String() string { + return r.Name() +} + +func (r repository) Name() string { + if r.domain == "" { + return r.path + } + return r.domain + "/" + r.path +} + +func (r repository) Domain() string { + return r.domain +} + +func (r repository) Path() string { + return r.path +} + +type digestReference digest.Digest + +func (d digestReference) String() string { + return digest.Digest(d).String() +} + +func (d digestReference) Digest() digest.Digest { + return digest.Digest(d) +} + +type taggedReference struct { + namedRepository + tag string +} + +func (t taggedReference) String() string { + return t.Name() + ":" + t.tag +} + +func (t taggedReference) Tag() string { + return t.tag +} + +type canonicalReference struct { + namedRepository + digest digest.Digest +} + +func (c canonicalReference) String() string { + return c.Name() + "@" + c.digest.String() +} + +func (c canonicalReference) Digest() digest.Digest { + return c.digest +} diff --git a/vendor/github.com/docker/distribution/reference/regexp.go b/vendor/github.com/docker/distribution/reference/regexp.go new file mode 100644 index 0000000000..7860349320 --- /dev/null +++ b/vendor/github.com/docker/distribution/reference/regexp.go @@ -0,0 +1,143 @@ +package reference + +import "regexp" + +var ( + // alphaNumericRegexp defines the alpha numeric atom, typically a + // component of names. This only allows lower case characters and digits. + alphaNumericRegexp = match(`[a-z0-9]+`) + + // separatorRegexp defines the separators allowed to be embedded in name + // components. This allow one period, one or two underscore and multiple + // dashes. + separatorRegexp = match(`(?:[._]|__|[-]*)`) + + // nameComponentRegexp restricts registry path component names to start + // with at least one letter or number, with following parts able to be + // separated by one period, one or two underscore and multiple dashes. + nameComponentRegexp = expression( + alphaNumericRegexp, + optional(repeated(separatorRegexp, alphaNumericRegexp))) + + // domainComponentRegexp restricts the registry domain component of a + // repository name to start with a component as defined by DomainRegexp + // and followed by an optional port. + domainComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`) + + // DomainRegexp defines the structure of potential domain components + // that may be part of image names. This is purposely a subset of what is + // allowed by DNS to ensure backwards compatibility with Docker image + // names. + DomainRegexp = expression( + domainComponentRegexp, + optional(repeated(literal(`.`), domainComponentRegexp)), + optional(literal(`:`), match(`[0-9]+`))) + + // TagRegexp matches valid tag names. From docker/docker:graph/tags.go. + TagRegexp = match(`[\w][\w.-]{0,127}`) + + // anchoredTagRegexp matches valid tag names, anchored at the start and + // end of the matched string. + anchoredTagRegexp = anchored(TagRegexp) + + // DigestRegexp matches valid digests. + DigestRegexp = match(`[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}`) + + // anchoredDigestRegexp matches valid digests, anchored at the start and + // end of the matched string. + anchoredDigestRegexp = anchored(DigestRegexp) + + // NameRegexp is the format for the name component of references. The + // regexp has capturing groups for the domain and name part omitting + // the separating forward slash from either. + NameRegexp = expression( + optional(DomainRegexp, literal(`/`)), + nameComponentRegexp, + optional(repeated(literal(`/`), nameComponentRegexp))) + + // anchoredNameRegexp is used to parse a name value, capturing the + // domain and trailing components. + anchoredNameRegexp = anchored( + optional(capture(DomainRegexp), literal(`/`)), + capture(nameComponentRegexp, + optional(repeated(literal(`/`), nameComponentRegexp)))) + + // ReferenceRegexp is the full supported format of a reference. The regexp + // is anchored and has capturing groups for name, tag, and digest + // components. + ReferenceRegexp = anchored(capture(NameRegexp), + optional(literal(":"), capture(TagRegexp)), + optional(literal("@"), capture(DigestRegexp))) + + // IdentifierRegexp is the format for string identifier used as a + // content addressable identifier using sha256. These identifiers + // are like digests without the algorithm, since sha256 is used. + IdentifierRegexp = match(`([a-f0-9]{64})`) + + // ShortIdentifierRegexp is the format used to represent a prefix + // of an identifier. A prefix may be used to match a sha256 identifier + // within a list of trusted identifiers. + ShortIdentifierRegexp = match(`([a-f0-9]{6,64})`) + + // anchoredIdentifierRegexp is used to check or match an + // identifier value, anchored at start and end of string. + anchoredIdentifierRegexp = anchored(IdentifierRegexp) + + // anchoredShortIdentifierRegexp is used to check if a value + // is a possible identifier prefix, anchored at start and end + // of string. + anchoredShortIdentifierRegexp = anchored(ShortIdentifierRegexp) +) + +// match compiles the string to a regular expression. +var match = regexp.MustCompile + +// literal compiles s into a literal regular expression, escaping any regexp +// reserved characters. +func literal(s string) *regexp.Regexp { + re := match(regexp.QuoteMeta(s)) + + if _, complete := re.LiteralPrefix(); !complete { + panic("must be a literal") + } + + return re +} + +// expression defines a full expression, where each regular expression must +// follow the previous. +func expression(res ...*regexp.Regexp) *regexp.Regexp { + var s string + for _, re := range res { + s += re.String() + } + + return match(s) +} + +// optional wraps the expression in a non-capturing group and makes the +// production optional. +func optional(res ...*regexp.Regexp) *regexp.Regexp { + return match(group(expression(res...)).String() + `?`) +} + +// repeated wraps the regexp in a non-capturing group to get one or more +// matches. +func repeated(res ...*regexp.Regexp) *regexp.Regexp { + return match(group(expression(res...)).String() + `+`) +} + +// group wraps the regexp in a non-capturing group. +func group(res ...*regexp.Regexp) *regexp.Regexp { + return match(`(?:` + expression(res...).String() + `)`) +} + +// capture wraps the expression in a capturing group. +func capture(res ...*regexp.Regexp) *regexp.Regexp { + return match(`(` + expression(res...).String() + `)`) +} + +// anchored anchors the regular expression by adding start and end delimiters. +func anchored(res ...*regexp.Regexp) *regexp.Regexp { + return match(`^` + expression(res...).String() + `$`) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1c716e9c1b..dfcaa933c7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -386,6 +386,10 @@ github.com/distribution/reference # github.com/docker/cli v24.0.6+incompatible ## explicit github.com/docker/cli/cli/config/types +# github.com/docker/distribution v2.8.2+incompatible +## explicit +github.com/docker/distribution/digestset +github.com/docker/distribution/reference # github.com/emicklei/go-restful/v3 v3.11.0 ## explicit; go 1.13 github.com/emicklei/go-restful/v3 From a7202006047f0913b8831283a303384bbb1be6ee Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 7 Oct 2024 14:15:02 +0530 Subject: [PATCH 19/32] fix for chart version --- pkg/generateManifest/DeploymentTemplateService.go | 2 +- pkg/pipeline/PropertiesConfig.go | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index 0bb0d6275b..2b98cd4c63 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -335,7 +335,7 @@ func (impl DeploymentTemplateServiceImpl) fetchResolvedTemplateForPublishedEnvs( impl.Logger.Errorw("error in getting overridden values", "err", err) return nil, err } - _, _, version, _, err := impl.chartRefService.GetRefChart(request.ChartRefId) + _, _, version, _, err := impl.chartRefService.GetRefChart(override.EnvironmentConfig.ChartRefId) if err != nil { impl.Logger.Errorw("error in getting chart ref by chartRefId ", "chartRefId", request.ChartRefId, "err", err) return nil, err diff --git a/pkg/pipeline/PropertiesConfig.go b/pkg/pipeline/PropertiesConfig.go index 26c387897e..ad4e2806d6 100644 --- a/pkg/pipeline/PropertiesConfig.go +++ b/pkg/pipeline/PropertiesConfig.go @@ -129,6 +129,9 @@ func (impl PropertiesConfigServiceImpl) GetEnvironmentProperties(appId, environm IsBasicViewLocked: envOverride.IsBasicViewLocked, CurrentViewEditor: envOverride.CurrentViewEditor, } + if chartRefId == 0 && envOverride.Chart != nil { + environmentProperties.ChartRefId = envOverride.Chart.ChartRefId + } if environmentPropertiesResponse.Namespace == "" { environmentPropertiesResponse.Namespace = envOverride.Namespace @@ -140,8 +143,10 @@ func (impl PropertiesConfigServiceImpl) GetEnvironmentProperties(appId, environm } if errors.IsNotFound(err) { environmentProperties.Id = 0 - environmentProperties.ChartRefId = chartRefId environmentProperties.IsOverride = false + if chartRefId > 0 { + environmentProperties.ChartRefId = chartRefId + } } else { environmentProperties.Id = ecOverride.Id environmentProperties.Latest = ecOverride.Latest @@ -153,6 +158,9 @@ func (impl PropertiesConfigServiceImpl) GetEnvironmentProperties(appId, environm environmentProperties.Active = ecOverride.Active environmentProperties.IsBasicViewLocked = ecOverride.IsBasicViewLocked environmentProperties.CurrentViewEditor = ecOverride.CurrentViewEditor + if chartRefId == 0 && ecOverride.Chart != nil { + environmentProperties.ChartRefId = ecOverride.Chart.ChartRefId + } } environmentPropertiesResponse.ChartRefId = chartRefId environmentPropertiesResponse.EnvironmentConfig = *environmentProperties From 596b86beb5022a93bde14a6d26e3724ecb5a4e77 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 7 Oct 2024 19:40:14 +0530 Subject: [PATCH 20/32] remove getResolvedConfigDataForValues --- .../DeploymentConfigurationRestHandler.go | 15 ------------- api/router/DeploymentConfigRouter.go | 2 +- .../DeploymentConfigurationService.go | 22 ------------------- pkg/configDiff/bean/bean.go | 1 - 4 files changed, 1 insertion(+), 39 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 13926f7fbd..2e29d68cb6 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -2,7 +2,6 @@ package restHandler import ( "context" - "encoding/json" "fmt" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -91,20 +90,6 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - switch r.Method { - case http.MethodPost: - if r.ContentLength > 0 { - valuesPayload := &bean.ValuesDto{} - decoder := json.NewDecoder(r.Body) - err = decoder.Decode(valuesPayload) - if err != nil { - handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - configDataQueryParams.Values = valuesPayload.Values - } - } configDataQueryParams.UserId = userId //RBAC START diff --git a/api/router/DeploymentConfigRouter.go b/api/router/DeploymentConfigRouter.go index b442c13424..a8a568d604 100644 --- a/api/router/DeploymentConfigRouter.go +++ b/api/router/DeploymentConfigRouter.go @@ -26,6 +26,6 @@ func (router DeploymentConfigurationRouterImpl) initDeploymentConfigurationRoute Methods("GET") configRouter.Path("/data"). HandlerFunc(router.deploymentGroupRestHandler.GetConfigData). - Methods("GET", "POST") + Methods("GET") } diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 58b7155a5d..1ba6d6e7c5 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -142,33 +142,11 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con return impl.getConfigDataForCdRollback(ctx, configDataQueryParams, userHasAdminAccess) case bean2.DeploymentHistory.ToString(): return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) - case bean2.ResolveData.ToString(): - // this only supports resolution of deployment template data as of now - return impl.getResolvedConfigDataForValues(ctx, configDataQueryParams.Values, appId, envId) } // this would be the default case return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess) } -func (impl *DeploymentConfigurationServiceImpl) getResolvedConfigDataForValues(ctx context.Context, values string, appId, envId int) (*bean2.DeploymentAndCmCsConfigDto, error) { - configDataDto := &bean2.DeploymentAndCmCsConfigDto{} - var err error - deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ - AppId: appId, - RequestDataMode: generateManifest.Values, - } - if envId > 0 { - deploymentTemplateRequest.EnvId = envId - } - resolvedTemplate, _, err := impl.deploymentTemplateService.ResolveTemplateVariables(ctx, values, deploymentTemplateRequest) - if err != nil { - impl.logger.Errorw("error in getting resolved data for cm draft data ", "appId", appId, "err", err) - return nil, err - } - - return configDataDto.WithDeploymentTemplateData(bean2.NewDeploymentAndCmCsConfig().WithResolvedValue(json.RawMessage(resolvedTemplate))), nil -} - func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { // wfrId is expected in this case to return the expected data return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 1ccf303950..94bc3ff7a4 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -175,7 +175,6 @@ type ConfigDataQueryParams struct { UserId int32 `schema:"-"` WfrId int `schema:"wfrId"` ConfigArea string `schema:"configArea"` - Values string `schema:"values"` } // FilterCriteria []string `schema:"filterCriteria"` From fba905e92537c0c7b7ac2ec05a7758d61786d761 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 8 Oct 2024 13:05:01 +0530 Subject: [PATCH 21/32] handle nil pipeline strategy in case of custom chart --- .../DeploymentConfigurationRestHandler.go | 4 ++-- pkg/configDiff/DeploymentConfigurationService.go | 13 +++++++++---- pkg/generateManifest/DeploymentTemplateService.go | 7 ++++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 2e29d68cb6..e19dcb91ed 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -103,9 +103,9 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp //RBAC END isSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*") userHasAdminAccess := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, object) - ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + ctx, _ := context.WithTimeout(r.Context(), 60*time.Second) ctx = util2.SetSuperAdminInContext(ctx, isSuperAdmin) - defer cancel() + res, err := handler.deploymentConfigurationService.GetAllConfigData(ctx, configDataQueryParams, userHasAdminAccess) if err != nil { handler.logger.Errorw("service err, GetAllConfigData ", "err", err) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 1ba6d6e7c5..70c11dfdd6 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -27,6 +27,7 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/variables/repository" util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" + "github.com/juju/errors" "go.uber.org/zap" "net/http" ) @@ -190,18 +191,20 @@ func (impl *DeploymentConfigurationServiceImpl) getDeploymentHistoryConfig(ctx c func (impl *DeploymentConfigurationServiceImpl) getPipelineStrategyConfigHistory(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams) (*bean2.DeploymentAndCmCsConfig, error) { pipelineStrategyJson := json.RawMessage{} + pipelineConfig := bean2.NewDeploymentAndCmCsConfig() pipelineStrategyHistory, err := impl.pipelineStrategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(ctx, configDataQueryParams.PipelineId, configDataQueryParams.WfrId) - if err != nil { + if err != nil && !util.IsErrNoRows(err) { impl.logger.Errorw("error in checking if history exists for pipelineId and wfrId", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err + } else if util.IsErrNoRows(err) { + return pipelineConfig, nil } err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategyHistory.Config)) if err != nil { impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "pipelineStrategyHistoryConfig", pipelineStrategyHistory.Config, "err", err) return nil, err } - pipelineConfig := bean2.NewDeploymentAndCmCsConfig(). - WithConfigData(pipelineStrategyJson). + pipelineConfig.WithConfigData(pipelineStrategyJson). WithResourceType(bean.PipelineStrategy). WithPipelineStrategyMetadata(pipelineStrategyHistory.PipelineTriggerType, string(pipelineStrategyHistory.Strategy)) return pipelineConfig, nil @@ -640,9 +643,11 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedPipelineStrategyConf return nil, err } pipelineStrategy, err := impl.deploymentConfigService.GetLatestPipelineStrategyConfig(pipeline) - if err != nil { + if err != nil && !errors.IsNotFound(err) { impl.logger.Errorw("error in GetLatestPipelineStrategyConfig", "pipelineId", pipeline.Id, "err", err) return nil, err + } else if errors.IsNotFound(err) { + return pipelineConfig, nil } err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategy.CodeEditorValue.Value)) if err != nil { diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index 2b98cd4c63..037f6dd774 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -335,7 +335,12 @@ func (impl DeploymentTemplateServiceImpl) fetchResolvedTemplateForPublishedEnvs( impl.Logger.Errorw("error in getting overridden values", "err", err) return nil, err } - _, _, version, _, err := impl.chartRefService.GetRefChart(override.EnvironmentConfig.ChartRefId) + // handle here for chart ref id in case + chartRefId := override.EnvironmentConfig.ChartRefId + if chartRefId == 0 { + chartRefId = override.GlobalChartRefId + } + _, _, version, _, err := impl.chartRefService.GetRefChart(chartRefId) if err != nil { impl.Logger.Errorw("error in getting chart ref by chartRefId ", "chartRefId", request.ChartRefId, "err", err) return nil, err From ae9c3aa74e4bcdcd2e9ad9172d4035bda64951ea Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 8 Oct 2024 13:47:41 +0530 Subject: [PATCH 22/32] revert defer cancel --- api/restHandler/DeploymentConfigurationRestHandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index e19dcb91ed..2e29d68cb6 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -103,9 +103,9 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp //RBAC END isSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*") userHasAdminAccess := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, object) - ctx, _ := context.WithTimeout(r.Context(), 60*time.Second) + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) ctx = util2.SetSuperAdminInContext(ctx, isSuperAdmin) - + defer cancel() res, err := handler.deploymentConfigurationService.GetAllConfigData(ctx, configDataQueryParams, userHasAdminAccess) if err != nil { handler.logger.Errorw("service err, GetAllConfigData ", "err", err) From a83f5ccbd9c18a9b9b3a74bc140b2e943bbdc270 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 8 Oct 2024 17:34:06 +0530 Subject: [PATCH 23/32] code review :- 1 --- api/restHandler/DeploymentConfigurationRestHandler.go | 2 +- pkg/configDiff/DeploymentConfigurationService.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index 2e29d68cb6..7a9f467629 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -104,8 +104,8 @@ func (handler *DeploymentConfigurationRestHandlerImpl) GetConfigData(w http.Resp isSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*") userHasAdminAccess := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, object) ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) - ctx = util2.SetSuperAdminInContext(ctx, isSuperAdmin) defer cancel() + ctx = util2.SetSuperAdminInContext(ctx, isSuperAdmin) res, err := handler.deploymentConfigurationService.GetAllConfigData(ctx, configDataQueryParams, userHasAdminAccess) if err != nil { handler.logger.Errorw("service err, GetAllConfigData ", "err", err) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 70c11dfdd6..3c1fe04cf4 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -263,7 +263,8 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context var configData []*bean.ConfigData configList := pipeline.ConfigsList{} secretList := bean.SecretsList{} - if configType == repository3.CONFIGMAP_TYPE { + switch configType { + case repository3.CONFIGMAP_TYPE: if len(history.Data) > 0 { err = json.Unmarshal([]byte(history.Data), &configList) if err != nil { @@ -273,7 +274,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context } resourceType = bean.CM configData = configList.ConfigData - } else if configType == repository3.SECRET_TYPE { + case repository3.SECRET_TYPE: if len(history.Data) > 0 { err = json.Unmarshal([]byte(history.Data), &secretList) if err != nil { @@ -283,6 +284,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context } resourceType = bean.CS configData = secretList.ConfigData + } resolvedDataMap, variableSnapshotMap, err := impl.scopedVariableManager.GetResolvedCMCSHistoryDtos(ctx, configType, adaptor.ReverseConfigListConvertor(configList), history, adaptor.ReverseSecretListConvertor(secretList)) @@ -632,7 +634,6 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx conte } func (impl *DeploymentConfigurationServiceImpl) getPublishedPipelineStrategyConfig(ctx context.Context, appId int, envId int) (*bean2.DeploymentAndCmCsConfig, error) { - pipelineStrategyJson := json.RawMessage{} pipelineConfig := bean2.NewDeploymentAndCmCsConfig() if envId == 0 { return pipelineConfig, nil @@ -649,6 +650,7 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedPipelineStrategyConf } else if errors.IsNotFound(err) { return pipelineConfig, nil } + pipelineStrategyJson := json.RawMessage{} err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategy.CodeEditorValue.Value)) if err != nil { impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "err", err) From 228e8e88bfad8d9dd774b5da0e2e49ac2259fc5f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 9 Oct 2024 12:56:11 +0530 Subject: [PATCH 24/32] remove pipelineStrategy --- pkg/configDiff/DeploymentConfigurationService.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 3c1fe04cf4..fadeeb0e23 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -229,7 +229,10 @@ func (impl *DeploymentConfigurationServiceImpl) getConfigDataForDeploymentHistor impl.logger.Errorw("getConfigDataForDeploymentHistory, error in getPipelineStrategyConfigHistory", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - configDataDto.WithPipelineConfigData(pipelineConfig) + if len(pipelineConfig.Data) > 0 { + configDataDto.WithPipelineConfigData(pipelineConfig) + } + // fetching for pipeline strategy config ends // fetching for cm config starts @@ -629,7 +632,9 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx conte impl.logger.Errorw("getPublishedConfigData, error in getting publishedOnly pipeline strategy ", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } - configData.WithPipelineConfigData(pipelineConfigData) + if len(pipelineConfigData.Data) > 0 { + configData.WithPipelineConfigData(pipelineConfigData) + } return configData, nil } From 57eb6b6300b154eb62197e80f1dcc59dc18c0612 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 10 Oct 2024 13:12:27 +0530 Subject: [PATCH 25/32] wire fix --- wire_gen.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wire_gen.go b/wire_gen.go index 436aac5874..0bee1a76cd 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -439,7 +439,7 @@ func InitializeApp() (*App, error) { ciWorkflowRepositoryImpl := pipelineConfig.NewCiWorkflowRepositoryImpl(db, sugaredLogger) ciPipelineMaterialRepositoryImpl := pipelineConfig.NewCiPipelineMaterialRepositoryImpl(db, sugaredLogger) ciArtifactRepositoryImpl := repository2.NewCiArtifactRepositoryImpl(db, sugaredLogger) - eventSimpleFactoryImpl := client2.NewEventSimpleFactoryImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, ciWorkflowRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, userRepositoryImpl, ciArtifactRepositoryImpl) + eventSimpleFactoryImpl := client2.NewEventSimpleFactoryImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, ciWorkflowRepositoryImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, userRepositoryImpl, environmentRepositoryImpl, ciArtifactRepositoryImpl) applicationServiceClientImpl := application.NewApplicationClientImpl(sugaredLogger, argoCDConnectionManagerImpl) configMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(sugaredLogger, db) chartRepositoryImpl := chartRepoRepository.NewChartRepository(db, transactionUtilImpl) @@ -722,7 +722,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - argoApplicationServiceExtendedImpl := argoApplication.NewArgoApplicationServiceExtendedServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationReadServiceImpl) + argoApplicationServiceExtendedImpl := argoApplication.NewArgoApplicationServiceExtendedServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, k8sApplicationServiceImpl, argoApplicationReadServiceImpl, applicationServiceClientImpl) installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl, argoApplicationServiceExtendedImpl) chartGroupEntriesRepositoryImpl := repository17.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository17.NewChartGroupReposotoryImpl(db, sugaredLogger) From 7f8b3d0e21709401eef344de9b88584158fbe60f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 10 Oct 2024 15:25:36 +0530 Subject: [PATCH 26/32] add wfr_id in template/list api --- internal/sql/repository/DeploymentTemplateRepository.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/sql/repository/DeploymentTemplateRepository.go b/internal/sql/repository/DeploymentTemplateRepository.go index 00116eb20b..053e7c8395 100644 --- a/internal/sql/repository/DeploymentTemplateRepository.go +++ b/internal/sql/repository/DeploymentTemplateRepository.go @@ -38,6 +38,7 @@ type DeploymentTemplateComparisonMetadata struct { EnvironmentId int `json:"environmentId,omitempty"` EnvironmentName string `json:"environmentName,omitempty"` DeploymentTemplateHistoryId int `json:"deploymentTemplateHistoryId,omitempty"` + WfrId int `json:"wfrId,omitempty"` StartedOn *time.Time `json:"startedOn,omitempty"` FinishedOn *time.Time `json:"finishedOn,omitempty"` Status string `json:"status,omitempty"` @@ -69,7 +70,7 @@ func (impl DeploymentTemplateRepositoryImpl) FetchDeploymentHistoryWithChartRefs limit := 15 query := "select p.id as pipeline_id, dth.id as deployment_template_history_id," + - " wfr.finished_on, wfr.status, c.chart_ref_id, c.chart_version FROM cd_workflow_runner wfr" + + " wfr.id as wfr_id, wfr.finished_on, wfr.status, c.chart_ref_id, c.chart_version FROM cd_workflow_runner wfr" + " JOIN cd_workflow wf ON wf.id = wfr.cd_workflow_id JOIN pipeline p ON p.id = wf.pipeline_id" + " JOIN deployment_template_history dth ON dth.deployed_on = wfr.started_on " + "JOIN pipeline_config_override pco ON pco.cd_workflow_id = wf.id " + From c56c4c9d3b90174e08b83aadb52548fc599895b7 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 10 Oct 2024 19:21:42 +0530 Subject: [PATCH 27/32] code review changes --- .../DeploymentTemplateService.go | 2 +- pkg/pipeline/adapter/adapter.go | 83 ++++++++++--------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index 037f6dd774..1a56931c71 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -347,7 +347,7 @@ func (impl DeploymentTemplateServiceImpl) fetchResolvedTemplateForPublishedEnvs( } resolvedTemplate, variableSnapshot, err := impl.resolveTemplateVariables(ctx, values, request) if err != nil { - impl.Logger.Errorw("error in resolving template variables for env override ", "values", values, "err", err) + impl.Logger.Errorw("error in resolving template variables for env override ", "deploymentTemplateRequest", request, "err", err) return nil, err } return &DeploymentTemplateResponse{ diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index feb6991b8c..e587a119bc 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -228,26 +228,29 @@ func GetSourceCiDownStreamResponse(linkedCIDetails []ciPipeline.LinkedCIDetails, } func ConvertConfigDataToPipelineConfigData(r *bean.ConfigData) *pipelineConfigBean.ConfigData { - return &pipelineConfigBean.ConfigData{ - Name: r.Name, - Type: r.Type, - External: r.External, - MountPath: r.MountPath, - Data: r.Data, - DefaultData: r.DefaultData, - DefaultMountPath: r.DefaultMountPath, - Global: r.Global, - ExternalSecretType: r.ExternalSecretType, - ESOSecretData: ConvertESOSecretDataToPipelineESOSecretData(r.ESOSecretData), - DefaultESOSecretData: ConvertESOSecretDataToPipelineESOSecretData(r.DefaultESOSecretData), - ExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.ExternalSecret), - DefaultExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.DefaultExternalSecret), - RoleARN: r.RoleARN, - SubPath: r.SubPath, - ESOSubPath: r.ESOSubPath, - FilePermission: r.FilePermission, - Overridden: r.Overridden, + if r != nil { + return &pipelineConfigBean.ConfigData{ + Name: r.Name, + Type: r.Type, + External: r.External, + MountPath: r.MountPath, + Data: r.Data, + DefaultData: r.DefaultData, + DefaultMountPath: r.DefaultMountPath, + Global: r.Global, + ExternalSecretType: r.ExternalSecretType, + ESOSecretData: ConvertESOSecretDataToPipelineESOSecretData(r.ESOSecretData), + DefaultESOSecretData: ConvertESOSecretDataToPipelineESOSecretData(r.DefaultESOSecretData), + ExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.ExternalSecret), + DefaultExternalSecret: ConvertExternalSecretToPipelineExternalSecret(r.DefaultExternalSecret), + RoleARN: r.RoleARN, + SubPath: r.SubPath, + ESOSubPath: r.ESOSubPath, + FilePermission: r.FilePermission, + Overridden: r.Overridden, + } } + return &pipelineConfigBean.ConfigData{} } func ConvertESOSecretDataToPipelineESOSecretData(r bean.ESOSecretData) pipelineConfigBean.ESOSecretData { @@ -289,26 +292,30 @@ func ConvertEsoDataToPipelineEsoData(r []bean.ESOData) []pipelineConfigBean.ESOD // reverse adapter for the above adapters func ConvertPipelineConfigDataToConfigData(r *pipelineConfigBean.ConfigData) *bean.ConfigData { - return &bean.ConfigData{ - Name: r.Name, - Type: r.Type, - External: r.External, - MountPath: r.MountPath, - Data: r.Data, - DefaultData: r.DefaultData, - DefaultMountPath: r.DefaultMountPath, - Global: r.Global, - ExternalSecretType: r.ExternalSecretType, - ESOSecretData: ConvertPipelineESOSecretDataToESOSecretData(r.ESOSecretData), - DefaultESOSecretData: ConvertPipelineESOSecretDataToESOSecretData(r.DefaultESOSecretData), - ExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.ExternalSecret), - DefaultExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.DefaultExternalSecret), - RoleARN: r.RoleARN, - SubPath: r.SubPath, - ESOSubPath: r.ESOSubPath, - FilePermission: r.FilePermission, - Overridden: r.Overridden, + if r != nil { + return &bean.ConfigData{ + Name: r.Name, + Type: r.Type, + External: r.External, + MountPath: r.MountPath, + Data: r.Data, + DefaultData: r.DefaultData, + DefaultMountPath: r.DefaultMountPath, + Global: r.Global, + ExternalSecretType: r.ExternalSecretType, + ESOSecretData: ConvertPipelineESOSecretDataToESOSecretData(r.ESOSecretData), + DefaultESOSecretData: ConvertPipelineESOSecretDataToESOSecretData(r.DefaultESOSecretData), + ExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.ExternalSecret), + DefaultExternalSecret: ConvertPipelineExternalSecretToExternalSecret(r.DefaultExternalSecret), + RoleARN: r.RoleARN, + SubPath: r.SubPath, + ESOSubPath: r.ESOSubPath, + FilePermission: r.FilePermission, + Overridden: r.Overridden, + } } + return &bean.ConfigData{} + } func ConvertPipelineESOSecretDataToESOSecretData(r pipelineConfigBean.ESOSecretData) bean.ESOSecretData { From aa670f1dda941d58995535bec08d1bf9e3f79f1c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 15 Oct 2024 02:57:00 +0530 Subject: [PATCH 28/32] scope var fix --- .../repository/EnvironmentRepository.go | 13 +++++++ .../DeploymentConfigurationService.go | 35 +++++++++++-------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/pkg/cluster/repository/EnvironmentRepository.go b/pkg/cluster/repository/EnvironmentRepository.go index 048aa83ae7..a5c43168ae 100644 --- a/pkg/cluster/repository/EnvironmentRepository.go +++ b/pkg/cluster/repository/EnvironmentRepository.go @@ -80,6 +80,7 @@ type EnvironmentRepository interface { FindAllActiveWithFilter() ([]*Environment, error) FindEnvClusterInfosByIds([]int) ([]*EnvCluserInfo, error) FindEnvLinkedWithCiPipelines(externalCi bool, ciPipelineIds []int) ([]*Environment, error) + FindEnvByNameWithClusterDetails(envName string) (*Environment, error) } func NewEnvironmentRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger, appStatusRepository appStatus.AppStatusRepository) *EnvironmentRepositoryImpl { @@ -160,6 +161,18 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindByName(name string) (*Enviro return environment, err } +func (repositoryImpl EnvironmentRepositoryImpl) FindEnvByNameWithClusterDetails(envName string) (*Environment, error) { + environment := &Environment{} + err := repositoryImpl.dbConnection. + Model(environment). + Column("environment.*", "Cluster"). + Where("environment.environment_name = ?", envName). + Where("environment.active = ?", true). + Limit(1). + Select() + return environment, err +} + func (repositoryImpl EnvironmentRepositoryImpl) FindIdByName(name string) (int, error) { environment := &Environment{} err := repositoryImpl.dbConnection. diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index fadeeb0e23..c946132f02 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -120,17 +120,21 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { var err error - var envId int - var appId int - var clusterId int + var envId, appId, clusterId int + systemMetadata := &resourceQualifiers.SystemMetadata{ + AppName: configDataQueryParams.AppName, + } if configDataQueryParams.IsEnvNameProvided() { - env, err := impl.environmentRepository.FindByName(configDataQueryParams.EnvName) + env, err := impl.environmentRepository.FindEnvByNameWithClusterDetails(configDataQueryParams.EnvName) if err != nil { impl.logger.Errorw("GetAllConfigData, error in getting environment model by envName", "envName", configDataQueryParams.EnvName, "err", err) return nil, err } envId = env.Id clusterId = env.ClusterId + systemMetadata.EnvironmentName = env.Name + systemMetadata.Namespace = env.Name + systemMetadata.ClusterName = env.Cluster.ClusterName } appId, err = impl.appRepository.FindAppIdByName(configDataQueryParams.AppName) if err != nil { @@ -145,7 +149,7 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) } // this would be the default case - return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess) + return impl.getConfigDataForAppConfiguration(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess, systemMetadata) } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { @@ -357,12 +361,12 @@ func (impl *DeploymentConfigurationServiceImpl) encodeSecretDataFromNonAdminUser } func (impl *DeploymentConfigurationServiceImpl) getConfigDataForAppConfiguration(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, - appId, envId, clusterId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { + appId, envId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.DeploymentAndCmCsConfigDto, error) { configDataDto := &bean2.DeploymentAndCmCsConfigDto{} var err error switch configDataQueryParams.ConfigType { default: // keeping default as PublishedOnly - configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess) + configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess, systemMetadata) if err != nil { impl.logger.Errorw("GetAllConfigData, error in config data for PublishedOnly", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err @@ -407,7 +411,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsEditDataForPublishedOnly( return configDataDto, nil } -func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { +func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.DeploymentAndCmCsConfigDto, error) { configDataDto := &bean2.DeploymentAndCmCsConfigDto{} secretData, err := impl.getSecretConfigResponse("", 0, envId, appId) @@ -435,7 +439,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(c return nil, err } - resolvedCmCsMetadataDto, err := impl.ResolveCmCs(ctx, envId, appId, clusterId, userHasAdminAccess) + resolvedCmCsMetadataDto, err := impl.ResolveCmCs(ctx, envId, appId, clusterId, userHasAdminAccess, systemMetadata) if err != nil { impl.logger.Errorw("error in resolving cm and cs for published only config only response", "appId", appId, "envId", envId, "err", err) return nil, err @@ -504,11 +508,12 @@ func (impl *DeploymentConfigurationServiceImpl) getMergedCmCs(envId, appId int) }, nil } -func (impl *DeploymentConfigurationServiceImpl) ResolveCmCs(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool) (*bean2.ResolvedCmCsMetadataDto, error) { +func (impl *DeploymentConfigurationServiceImpl) ResolveCmCs(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.ResolvedCmCsMetadataDto, error) { scope := resourceQualifiers.Scope{ - AppId: appId, - EnvId: envId, - ClusterId: clusterId, + AppId: appId, + EnvId: envId, + ClusterId: clusterId, + SystemMetadata: systemMetadata, } cmcsMetadataDto, err := impl.getMergedCmCs(envId, appId) if err != nil { @@ -609,13 +614,13 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedDeploymentConfig(ctx } func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, - appId, envId, clusterId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { + appId, envId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.DeploymentAndCmCsConfigDto, error) { if configDataQueryParams.IsRequestMadeForOneResource() { return impl.getCmCsEditDataForPublishedOnly(configDataQueryParams, envId, appId) } //ConfigMapsData and SecretsData are populated here - configData, err := impl.getCmCsPublishedConfigResponse(ctx, envId, appId, clusterId, userHasAdminAccess) + configData, err := impl.getCmCsPublishedConfigResponse(ctx, envId, appId, clusterId, userHasAdminAccess, systemMetadata) if err != nil { impl.logger.Errorw("getPublishedConfigData, error in getting cm cs for PublishedOnly state", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) return nil, err From 494d30bbe006c5cbdeeb2ed5424ad674672e6b43 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 15 Oct 2024 03:18:40 +0530 Subject: [PATCH 29/32] edit resource --- .../DeploymentConfigurationService.go | 32 ++++++++++++++-- pkg/configDiff/helper/helper.go | 38 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index c946132f02..93eceba890 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -375,7 +375,8 @@ func (impl *DeploymentConfigurationServiceImpl) getConfigDataForAppConfiguration return configDataDto, nil } -func (impl *DeploymentConfigurationServiceImpl) getCmCsEditDataForPublishedOnly(configDataQueryParams *bean2.ConfigDataQueryParams, envId, appId int) (*bean2.DeploymentAndCmCsConfigDto, error) { +func (impl *DeploymentConfigurationServiceImpl) getCmCsEditDataForPublishedOnly(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, envId, + appId int, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.DeploymentAndCmCsConfigDto, error) { configDataDto := &bean2.DeploymentAndCmCsConfigDto{} var resourceType bean.ResourceType @@ -401,11 +402,29 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsEditDataForPublishedOnly( impl.logger.Errorw("getCmCsEditDataForPublishedOnly, error in converting to json raw message", "configDataQueryParams", configDataQueryParams, "err", err) return nil, err } + resolvedCmCsMetadataDto, err := impl.ResolveCmCs(ctx, envId, appId, clusterId, userHasAdminAccess, configDataQueryParams.ResourceName, resourceType, systemMetadata) + if err != nil { + impl.logger.Errorw("error in resolving cm and cs for published only config only response", "appId", appId, "envId", envId, "err", err) + return nil, err + } cmCsConfig := bean2.NewDeploymentAndCmCsConfig().WithConfigData(respJson).WithResourceType(resourceType) + if resourceType == bean.CS { + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedCmCsMetadataDto.ResolvedSecretData) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage ", "err", err) + return nil, err + } + cmCsConfig.WithResolvedValue(resolvedConfigDataStringJson).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCS) configDataDto.WithSecretData(cmCsConfig) } else if resourceType == bean.CM { + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedCmCsMetadataDto.ResolvedConfigMapData) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "ResolvedConfigMapData", resolvedCmCsMetadataDto.ResolvedConfigMapData, "err", err) + return nil, err + } + cmCsConfig.WithResolvedValue(resolvedConfigDataStringJson).WithVariableSnapshot(resolvedCmCsMetadataDto.VariableMapCM) configDataDto.WithConfigMapData(cmCsConfig) } return configDataDto, nil @@ -439,7 +458,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsPublishedConfigResponse(c return nil, err } - resolvedCmCsMetadataDto, err := impl.ResolveCmCs(ctx, envId, appId, clusterId, userHasAdminAccess, systemMetadata) + resolvedCmCsMetadataDto, err := impl.ResolveCmCs(ctx, envId, appId, clusterId, userHasAdminAccess, "", "", systemMetadata) if err != nil { impl.logger.Errorw("error in resolving cm and cs for published only config only response", "appId", appId, "envId", envId, "err", err) return nil, err @@ -508,7 +527,8 @@ func (impl *DeploymentConfigurationServiceImpl) getMergedCmCs(envId, appId int) }, nil } -func (impl *DeploymentConfigurationServiceImpl) ResolveCmCs(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.ResolvedCmCsMetadataDto, error) { +func (impl *DeploymentConfigurationServiceImpl) ResolveCmCs(ctx context.Context, envId, appId, clusterId int, userHasAdminAccess bool, + resourceName string, resourceType bean.ResourceType, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.ResolvedCmCsMetadataDto, error) { scope := resourceQualifiers.Scope{ AppId: appId, EnvId: envId, @@ -520,6 +540,10 @@ func (impl *DeploymentConfigurationServiceImpl) ResolveCmCs(ctx context.Context, impl.logger.Errorw("error in getting merged cm cs", "appId", appId, "envId", envId, "err", err) return nil, err } + // if resourceName is provided then, resolve cmcs request is for single resource, then remove other data from merged cmCs + if len(resourceName) > 0 { + helper.FilterOutMergedCmCsForResourceName(cmcsMetadataDto, resourceName, resourceType) + } resolvedConfigList, resolvedSecretList, variableMapCM, variableMapCS, err := impl.scopedVariableManager.ResolveCMCS(ctx, scope, cmcsMetadataDto.ConfigAppLevelId, cmcsMetadataDto.ConfigEnvLevelId, cmcsMetadataDto.CmMap, cmcsMetadataDto.SecretMap) if err != nil { impl.logger.Errorw("error in resolving CM/CS", "scope", scope, "appId", appId, "envId", envId, "err", err) @@ -617,7 +641,7 @@ func (impl *DeploymentConfigurationServiceImpl) getPublishedConfigData(ctx conte appId, envId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.DeploymentAndCmCsConfigDto, error) { if configDataQueryParams.IsRequestMadeForOneResource() { - return impl.getCmCsEditDataForPublishedOnly(configDataQueryParams, envId, appId) + return impl.getCmCsEditDataForPublishedOnly(ctx, configDataQueryParams, envId, appId, clusterId, userHasAdminAccess, systemMetadata) } //ConfigMapsData and SecretsData are populated here configData, err := impl.getCmCsPublishedConfigResponse(ctx, envId, appId, clusterId, userHasAdminAccess, systemMetadata) diff --git a/pkg/configDiff/helper/helper.go b/pkg/configDiff/helper/helper.go index 70082a7bea..3bf5e5ffaa 100644 --- a/pkg/configDiff/helper/helper.go +++ b/pkg/configDiff/helper/helper.go @@ -1,7 +1,9 @@ package helper import ( + bean3 "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLevelMap map[string]*bean2.ConfigProperty) []*bean2.ConfigProperty { @@ -18,3 +20,39 @@ func GetCombinedPropertiesMap(cmcsKeyPropertyAppLevelMap, cmcsKeyPropertyEnvLeve } return combinedProperties } + +func GetKeysToDelete(cmcsData map[string]*bean3.ConfigData, resourceName string) []string { + keysToDelete := make([]string, 0, len(cmcsData)) + for key, _ := range cmcsData { + if key != resourceName { + keysToDelete = append(keysToDelete, key) + } + } + return keysToDelete +} + +func FilterOutMergedCmCsForResourceName(cmcsMerged *bean2.CmCsMetadataDto, resourceName string, resourceType bean.ResourceType) { + for _, key := range GetKeysToDelete(cmcsMerged.SecretMap, resourceName) { + delete(cmcsMerged.SecretMap, key) + } + for _, key := range GetKeysToDelete(cmcsMerged.CmMap, resourceName) { + delete(cmcsMerged.CmMap, key) + } + + // handle the case when a cm and a cs can have a same name, in that case, check from resource type if correct key is filtered out or not + if resourceType == bean.CS { + if len(cmcsMerged.CmMap) > 0 { + // delete all elements from cmMap as requested resource is of secret type + for key, _ := range cmcsMerged.CmMap { + delete(cmcsMerged.CmMap, key) + } + } + } else if resourceType == bean.CM { + if len(cmcsMerged.SecretMap) > 0 { + // delete all elements from secretMap as requested resource is of secret type + for key, _ := range cmcsMerged.SecretMap { + delete(cmcsMerged.SecretMap, key) + } + } + } +} From 5ffb3dd9acb0d918e8eeb61ad72dbff589447c4d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 15 Oct 2024 12:32:44 +0530 Subject: [PATCH 30/32] previous deployments stage added --- .../DeploymentConfigurationService.go | 121 +++++++++++++++- pkg/configDiff/adaptor/adaptor.go | 7 +- pkg/configDiff/bean/bean.go | 1 + pkg/pipeline/ConfigMapService.go | 32 ++--- pkg/pipeline/bean/ConfigMapBean.go | 4 + .../history/ConfigMapHistoryService.go | 133 ++++++++++++++++++ .../DeploymentTemplateHistoryService.go | 11 ++ .../repository/ConfigMapHistoryRepository.go | 19 +++ .../PipelineStrategyHistoryRepository.go | 9 ++ wire_gen.go | 2 +- 10 files changed, 315 insertions(+), 24 deletions(-) diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 93eceba890..7966efa159 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -20,6 +20,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/history" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" @@ -52,6 +53,8 @@ type DeploymentConfigurationServiceImpl struct { deploymentConfigService pipeline.PipelineDeploymentConfigService chartRefService chartRef.ChartRefService pipelineRepository pipelineConfig.PipelineRepository + deploymentTemplateHistoryService history.DeploymentTemplateHistoryService + configMapHistoryService history.ConfigMapHistoryService } func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, @@ -68,6 +71,8 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, deploymentConfigService pipeline.PipelineDeploymentConfigService, chartRefService chartRef.ChartRefService, pipelineRepository pipelineConfig.PipelineRepository, + deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, + configMapHistoryService history.ConfigMapHistoryService, ) (*DeploymentConfigurationServiceImpl, error) { deploymentConfigurationService := &DeploymentConfigurationServiceImpl{ logger: logger, @@ -84,6 +89,8 @@ func NewDeploymentConfigurationServiceImpl(logger *zap.SugaredLogger, deploymentConfigService: deploymentConfigService, chartRefService: chartRefService, pipelineRepository: pipelineRepository, + deploymentTemplateHistoryService: deploymentTemplateHistoryService, + configMapHistoryService: configMapHistoryService, } return deploymentConfigurationService, nil @@ -268,7 +275,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context return nil, err } var configData []*bean.ConfigData - configList := pipeline.ConfigsList{} + configList := bean.ConfigsList{} secretList := bean.SecretsList{} switch configType { case repository3.CONFIGMAP_TYPE: @@ -360,11 +367,123 @@ func (impl *DeploymentConfigurationServiceImpl) encodeSecretDataFromNonAdminUser } } +func (impl *DeploymentConfigurationServiceImpl) getCmCsDataForPreviousDeployments(ctx context.Context, deploymentTemplateHistoryId, pipelineId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { + + configDataDto := &bean2.DeploymentAndCmCsConfigDto{} + + deplTemplateHistory, err := impl.deploymentTemplateHistoryService.GetTemplateHistoryModelForDeployedTemplateById(deploymentTemplateHistoryId, pipelineId) + if err != nil { + impl.logger.Errorw("error in getting deployment template history", "err", err, "deploymentTemplateHistoryId", deploymentTemplateHistoryId, "pipelineId", pipelineId) + return nil, err + } + + secretConfigData, cmConfigData, err := impl.configMapHistoryService.GetConfigmapHistoryDataByDeployedOnAndPipelineId(ctx, pipelineId, deplTemplateHistory.DeployedOn, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("error in getting secretData and cmData", "err", err, "deploymentTemplateHistoryId", deploymentTemplateHistoryId, "pipelineId", pipelineId) + return nil, err + } + configDataDto.WithConfigMapData(cmConfigData).WithSecretData(secretConfigData) + return configDataDto, nil + +} +func (impl *DeploymentConfigurationServiceImpl) getPipelineStrategyForPreviousDeployments(ctx context.Context, deploymentTemplateHistoryId, pipelineId int) (*bean2.DeploymentAndCmCsConfig, error) { + pipelineStrategyJson := json.RawMessage{} + pipelineConfig := bean2.NewDeploymentAndCmCsConfig() + deplTemplateHistory, err := impl.deploymentTemplateHistoryService.GetTemplateHistoryModelForDeployedTemplateById(deploymentTemplateHistoryId, pipelineId) + if err != nil { + impl.logger.Errorw("error in getting deployment template history", "deploymentTemplateHistoryId", deploymentTemplateHistoryId, "pipelineId", pipelineId, "err", err) + return nil, err + } + pipelineStrategyHistory, err := impl.pipelineStrategyHistoryRepository.FindPipelineStrategyForDeployedOnAndPipelineId(pipelineId, deplTemplateHistory.DeployedOn) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in FindPipelineStrategyForDeployedOnAndPipelineId", "deploymentTemplateHistoryId", deploymentTemplateHistoryId, "deployedOn", deplTemplateHistory.DeployedOn, "pipelineId", pipelineId, "err", err) + return nil, err + } else if util.IsErrNoRows(err) { + return pipelineConfig, nil + } + err = pipelineStrategyJson.UnmarshalJSON([]byte(pipelineStrategyHistory.Config)) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in unmarshalling string pipelineStrategyHistory data into json Raw message", "err", err) + return nil, err + } + pipelineConfig.WithConfigData(pipelineStrategyJson). + WithResourceType(bean.PipelineStrategy). + WithPipelineStrategyMetadata(pipelineStrategyHistory.PipelineTriggerType, string(pipelineStrategyHistory.Strategy)) + return pipelineConfig, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getDeploymentsConfigForPreviousDeployments(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, + appId, envId int) (generateManifest.DeploymentTemplateResponse, error) { + deploymentTemplateRequest := generateManifest.DeploymentTemplateRequest{ + PipelineId: configDataQueryParams.PipelineId, + DeploymentTemplateHistoryId: configDataQueryParams.IdentifierId, + RequestDataMode: generateManifest.Values, + Type: repository2.DeployedOnSelfEnvironment, + } + var deploymentTemplateResponse generateManifest.DeploymentTemplateResponse + deploymentTemplateResponse, err := impl.deploymentTemplateService.GetDeploymentTemplate(ctx, deploymentTemplateRequest) + if err != nil { + impl.logger.Errorw("getDeploymentTemplateForEnvLevel, error in getting deployment template for ", "deploymentTemplateRequest", deploymentTemplateRequest, "err", err) + return deploymentTemplateResponse, err + } + + return deploymentTemplateResponse, nil +} + +func (impl *DeploymentConfigurationServiceImpl) getDeploymentAndCmCsConfigDataForPreviousDeployments(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, + appId, envId int, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { + + // getting DeploymentAndCmCsConfigDto obj with cm and cs data populated + configDataDto, err := impl.getCmCsDataForPreviousDeployments(ctx, configDataQueryParams.IdentifierId, configDataQueryParams.PipelineId, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("error in getting cm cs for PreviousDeployments state", "deploymentTemplateHistoryId", configDataQueryParams.IdentifierId, "pipelineId", configDataQueryParams.PipelineId, "err", err) + return nil, err + } + pipelineStrategy, err := impl.getPipelineStrategyForPreviousDeployments(ctx, configDataQueryParams.IdentifierId, configDataQueryParams.PipelineId) + if err != nil { + impl.logger.Errorw(" error in getting cm cs for PreviousDeployments state", "deploymentTemplateHistoryId", configDataQueryParams.IdentifierId, "pipelineId", configDataQueryParams.PipelineId, "err", err) + return nil, err + } + if len(pipelineStrategy.Data) > 0 { + configDataDto.WithPipelineConfigData(pipelineStrategy) + } + + deploymentTemplateData, err := impl.getDeploymentsConfigForPreviousDeployments(ctx, configDataQueryParams, appId, envId) + if err != nil { + impl.logger.Errorw("error in getting deployment config", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + deploymentJson := json.RawMessage{} + err = deploymentJson.UnmarshalJSON([]byte(deploymentTemplateData.Data)) + if err != nil { + impl.logger.Errorw("error in unmarshalling string deploymentTemplateResponse data into json Raw message", "appName", configDataQueryParams.AppName, "envName", configDataQueryParams.EnvName, "err", err) + return nil, err + } + variableSnapShotMap := map[string]map[string]string{bean.DeploymentTemplate.ToString(): deploymentTemplateData.VariableSnapshot} + + deploymentConfig := bean2.NewDeploymentAndCmCsConfig(). + WithDeploymentConfigMetadata(deploymentTemplateData.TemplateVersion, deploymentTemplateData.IsAppMetricsEnabled). + WithConfigData(deploymentJson). + WithResourceType(bean.DeploymentTemplate). + WithResolvedValue(json.RawMessage(deploymentTemplateData.ResolvedData)). + WithVariableSnapshot(variableSnapShotMap) + + configDataDto.WithDeploymentTemplateData(deploymentConfig) + + return configDataDto, nil +} + func (impl *DeploymentConfigurationServiceImpl) getConfigDataForAppConfiguration(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, appId, envId, clusterId int, userHasAdminAccess bool, systemMetadata *resourceQualifiers.SystemMetadata) (*bean2.DeploymentAndCmCsConfigDto, error) { configDataDto := &bean2.DeploymentAndCmCsConfigDto{} var err error switch configDataQueryParams.ConfigType { + case bean2.PreviousDeployments.ToString(): + configDataDto, err = impl.getDeploymentAndCmCsConfigDataForPreviousDeployments(ctx, configDataQueryParams, appId, envId, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("GetAllConfigData, error in config data for Previous Deployments", "configDataQueryParams", configDataQueryParams, "err", err) + return nil, err + } default: // keeping default as PublishedOnly configDataDto, err = impl.getPublishedConfigData(ctx, configDataQueryParams, appId, envId, clusterId, userHasAdminAccess, systemMetadata) if err != nil { diff --git a/pkg/configDiff/adaptor/adaptor.go b/pkg/configDiff/adaptor/adaptor.go index 6b93442ff4..6fd46129fe 100644 --- a/pkg/configDiff/adaptor/adaptor.go +++ b/pkg/configDiff/adaptor/adaptor.go @@ -3,7 +3,6 @@ package adaptor import ( bean3 "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) @@ -31,12 +30,12 @@ func GetCmCsAppAndEnvLevelMap(cMCSNamesAppLevel, cMCSNamesEnvLevel []bean.Config return cMCSNamesAppLevelMap, cMCSNamesEnvLevelMap } -func ConfigListConvertor(r bean3.ConfigList) pipeline.ConfigsList { +func ConfigListConvertor(r bean3.ConfigList) bean.ConfigsList { pipelineConfigData := make([]*bean.ConfigData, 0, len(r.ConfigData)) for _, item := range r.ConfigData { pipelineConfigData = append(pipelineConfigData, adapter.ConvertConfigDataToPipelineConfigData(item)) } - return pipeline.ConfigsList{ConfigData: pipelineConfigData} + return bean.ConfigsList{ConfigData: pipelineConfigData} } func SecretListConvertor(r bean3.SecretList) bean.SecretsList { @@ -47,7 +46,7 @@ func SecretListConvertor(r bean3.SecretList) bean.SecretsList { return bean.SecretsList{ConfigData: pipelineConfigData} } -func ReverseConfigListConvertor(r pipeline.ConfigsList) bean3.ConfigList { +func ReverseConfigListConvertor(r bean.ConfigsList) bean3.ConfigList { configData := make([]*bean3.ConfigData, 0, len(r.ConfigData)) for _, item := range r.ConfigData { configData = append(configData, adapter.ConvertPipelineConfigDataToConfigData(item)) diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index 94bc3ff7a4..b1ce554032 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -13,6 +13,7 @@ type ConfigState string const ( PublishedConfigState ConfigState = "PublishedOnly" + PreviousDeployments ConfigState = "PreviousDeployments" ) func (r ConfigState) ToString() string { diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index d2e0bb883c..bbfb74148b 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -47,10 +47,6 @@ const ( HashiCorpVault string = "HashiCorpVault" ) -type ConfigsList struct { - ConfigData []*bean.ConfigData `json:"maps"` -} - type ConfigMapService interface { CMGlobalAddUpdate(configMapRequest *bean.ConfigDataRequest) (*bean.ConfigDataRequest, error) CMGlobalFetch(appId int) (*bean.ConfigDataRequest, error) @@ -165,7 +161,7 @@ func (impl ConfigMapServiceImpl) CMGlobalAddUpdate(configMapRequest *bean.Config impl.logger.Errorw("error while fetching from db", "error", err) return nil, err } - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} found := false var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { @@ -208,7 +204,7 @@ func (impl ConfigMapServiceImpl) CMGlobalAddUpdate(configMapRequest *bean.Config } else { //creating config map record for first time - configsList := &ConfigsList{ + configsList := &bean.ConfigsList{ ConfigData: configMapRequest.ConfigData, } configDataByte, err := json.Marshal(configsList) @@ -254,7 +250,7 @@ func (impl ConfigMapServiceImpl) CMGlobalFetch(appId int) (*bean.ConfigDataReque impl.logger.Debugw("no config map data found for this request", "appId", appId) } - configMapGlobalList := &ConfigsList{} + configMapGlobalList := &bean.ConfigsList{} if len(configMapGlobal.ConfigMapData) > 0 { err = json.Unmarshal([]byte(configMapGlobal.ConfigMapData), configMapGlobalList) if err != nil { @@ -301,7 +297,7 @@ func (impl ConfigMapServiceImpl) CMEnvironmentAddUpdate(configMapRequest *bean.C return nil, err } if err == nil && model.Id > 0 { - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} found := false var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { @@ -345,7 +341,7 @@ func (impl ConfigMapServiceImpl) CMEnvironmentAddUpdate(configMapRequest *bean.C } else if err == pg.ErrNoRows { //creating config map record for first time - configsList := &ConfigsList{ + configsList := &bean.ConfigsList{ ConfigData: configMapRequest.ConfigData, } configDataByte, err := json.Marshal(configsList) @@ -391,7 +387,7 @@ func (impl ConfigMapServiceImpl) CMGlobalFetchForEdit(name string, id int) (*bea impl.logger.Debugw("no config map data found for this request", "id", id) } - configMapGlobalList := &ConfigsList{} + configMapGlobalList := &bean.ConfigsList{} if len(configMapGlobal.ConfigMapData) > 0 { err = json.Unmarshal([]byte(configMapGlobal.ConfigMapData), configMapGlobalList) if err != nil { @@ -439,7 +435,7 @@ func (impl ConfigMapServiceImpl) CMEnvironmentFetch(appId int, envId int) (*bean if pg.ErrNoRows == err { impl.logger.Debugw("no config map data found for this request", "appId", appId) } - configMapGlobalList := &ConfigsList{} + configMapGlobalList := &bean.ConfigsList{} if len(configMapGlobal.ConfigMapData) > 0 { err = json.Unmarshal([]byte(configMapGlobal.ConfigMapData), configMapGlobalList) if err != nil { @@ -454,7 +450,7 @@ func (impl ConfigMapServiceImpl) CMEnvironmentFetch(appId int, envId int) (*bean if pg.ErrNoRows == err { impl.logger.Debugw("no config map data found for this request", "appId", appId) } - configsListEnvLevel := &ConfigsList{} + configsListEnvLevel := &bean.ConfigsList{} if len(configMapEnvLevel.ConfigMapData) > 0 { err = json.Unmarshal([]byte(configMapEnvLevel.ConfigMapData), configsListEnvLevel) if err != nil { @@ -918,7 +914,7 @@ func (impl ConfigMapServiceImpl) CMGlobalDelete(name string, id int, userId int3 impl.logger.Errorw("error while fetching from db", "error", err) return false, err } - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} found := false var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { @@ -974,7 +970,7 @@ func (impl ConfigMapServiceImpl) CMEnvironmentDelete(name string, id int, userId impl.logger.Errorw("error while fetching from db", "error", err) return false, err } - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} found := false var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { @@ -1140,7 +1136,7 @@ func (impl ConfigMapServiceImpl) CMGlobalDeleteByAppId(name string, appId int, u impl.logger.Errorw("error while fetching from db", "error", err) return false, err } - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} found := false var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { @@ -1190,7 +1186,7 @@ func (impl ConfigMapServiceImpl) CMEnvironmentDeleteByAppIdAndEnvId(name string, impl.logger.Errorw("error while fetching from db", "error", err) return false, err } - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} found := false var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { @@ -1540,7 +1536,7 @@ func (impl ConfigMapServiceImpl) ConfigSecretGlobalBulkPatch(bulkPatchRequest *b continue } if bulkPatchRequest.Type == "CM" { - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { err = json.Unmarshal([]byte(model.ConfigMapData), configsList) @@ -1645,7 +1641,7 @@ func (impl ConfigMapServiceImpl) ConfigSecretEnvironmentBulkPatch(bulkPatchReque continue } if bulkPatchRequest.Type == "CM" { - configsList := &ConfigsList{} + configsList := &bean.ConfigsList{} var configs []*bean.ConfigData if len(model.ConfigMapData) > 0 { err = json.Unmarshal([]byte(model.ConfigMapData), configsList) diff --git a/pkg/pipeline/bean/ConfigMapBean.go b/pkg/pipeline/bean/ConfigMapBean.go index 5f194ebe1f..65cb8af637 100644 --- a/pkg/pipeline/bean/ConfigMapBean.go +++ b/pkg/pipeline/bean/ConfigMapBean.go @@ -119,6 +119,10 @@ type SecretsList struct { ConfigData []*ConfigData `json:"secrets"` } +type ConfigsList struct { + ConfigData []*ConfigData `json:"maps"` +} + type ConfigNameAndType struct { Id int Name string diff --git a/pkg/pipeline/history/ConfigMapHistoryService.go b/pkg/pipeline/history/ConfigMapHistoryService.go index 4f56673bd1..107f486dea 100644 --- a/pkg/pipeline/history/ConfigMapHistoryService.go +++ b/pkg/pipeline/history/ConfigMapHistoryService.go @@ -20,6 +20,11 @@ import ( "context" "encoding/json" "errors" + "github.com/devtron-labs/devtron/pkg/configDiff/adaptor" + bean2 "github.com/devtron-labs/devtron/pkg/configDiff/bean" + "github.com/devtron-labs/devtron/pkg/configDiff/utils" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" globalUtil "github.com/devtron-labs/devtron/util" "time" @@ -48,6 +53,8 @@ type ConfigMapHistoryService interface { CheckIfTriggerHistoryExistsForPipelineIdOnTime(pipelineId int, deployedOn time.Time) (cmId int, csId int, exists bool, err error) GetDeployedHistoryDetailForCMCSByPipelineIdAndWfrId(ctx context.Context, pipelineId, wfrId int, configType repository.ConfigType, userHasAdminAccess bool) ([]*ComponentLevelHistoryDetailDto, error) ConvertConfigDataToComponentLevelDto(config *bean.ConfigData, configType repository.ConfigType, userHasAdminAccess bool) (*ComponentLevelHistoryDetailDto, error) + + GetConfigmapHistoryDataByDeployedOnAndPipelineId(ctx context.Context, pipelineId int, deployedOn time.Time, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfig, *bean2.DeploymentAndCmCsConfig, error) } type ConfigMapHistoryServiceImpl struct { @@ -691,3 +698,129 @@ func (impl ConfigMapHistoryServiceImpl) CheckIfTriggerHistoryExistsForPipelineId } return cmId, csId, exists, nil } + +func (impl ConfigMapHistoryServiceImpl) GetConfigmapHistoryDataByDeployedOnAndPipelineId(ctx context.Context, pipelineId int, deployedOn time.Time, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfig, *bean2.DeploymentAndCmCsConfig, error) { + secretConfigData, err := impl.getResolvedConfigData(ctx, pipelineId, deployedOn, repository.SECRET_TYPE, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("error in getting resolved secret config data in case of previous deployments ", "pipelineId", pipelineId, "deployedOn", deployedOn, "err", err) + return nil, nil, err + } + cmConfigData, err := impl.getResolvedConfigData(ctx, pipelineId, deployedOn, repository.CONFIGMAP_TYPE, userHasAdminAccess) + if err != nil { + impl.logger.Errorw("error in getting resolved cm config data in case of previous deployments ", "pipelineId", pipelineId, "deployedOn", deployedOn, "err", err) + return nil, nil, err + } + + return secretConfigData, cmConfigData, nil +} + +func (impl *ConfigMapHistoryServiceImpl) getResolvedConfigData(ctx context.Context, pipelineId int, deployedOn time.Time, configType repository.ConfigType, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfig, error) { + configsList := &bean3.ConfigsList{} + secretsList := &bean3.SecretsList{} + var err error + history, err := impl.configMapHistoryRepository.GetDeployedHistoryByPipelineIdAndDeployedOn(pipelineId, deployedOn, configType) + if err != nil { + impl.logger.Errorw("error in getting deployed history by pipeline id and deployed on", "pipelineId", pipelineId, "deployedOn", deployedOn, "err", err) + return nil, err + } + if configType == repository.SECRET_TYPE { + _, secretsList, err = impl.getConfigDataRequestForHistory(history) + if err != nil { + impl.logger.Errorw("error in getting config data request for history", "err", err) + return nil, err + } + } else if configType == repository.CONFIGMAP_TYPE { + configsList, _, err = impl.getConfigDataRequestForHistory(history) + if err != nil { + impl.logger.Errorw("error in getting config data request for history", "cmCsHistory", history, "err", err) + return nil, err + } + } + + resolvedDataMap, variableSnapshotMap, err := impl.scopedVariableManager.GetResolvedCMCSHistoryDtos(ctx, configType, adaptor.ReverseConfigListConvertor(*configsList), history, adaptor.ReverseSecretListConvertor(*secretsList)) + if err != nil { + return nil, err + } + resolvedConfigDataList := make([]*bean3.ConfigData, 0, len(resolvedDataMap)) + for _, resolvedConfigData := range resolvedDataMap { + resolvedConfigDataList = append(resolvedConfigDataList, adapter.ConvertConfigDataToPipelineConfigData(&resolvedConfigData)) + } + configDataReq := &bean3.ConfigDataRequest{} + var resourceType bean3.ResourceType + if configType == repository.SECRET_TYPE { + impl.encodeSecretDataFromNonAdminUsers(secretsList.ConfigData, userHasAdminAccess) + impl.encodeSecretDataFromNonAdminUsers(resolvedConfigDataList, userHasAdminAccess) + configDataReq.ConfigData = secretsList.ConfigData + resourceType = bean3.CS + } else if configType == repository.CONFIGMAP_TYPE { + configDataReq.ConfigData = configsList.ConfigData + resourceType = bean3.CM + } + + configDataJson, err := utils.ConvertToJsonRawMessage(configDataReq) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", pipelineId, "deployedOn", deployedOn, "err", err) + return nil, err + } + resolvedConfigDataReq := &bean3.ConfigDataRequest{ConfigData: resolvedConfigDataList} + resolvedConfigDataString, err := utils.ConvertToString(resolvedConfigDataReq) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", pipelineId, "deployedOn", deployedOn, "err", err) + return nil, err + } + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedConfigDataString) + if err != nil { + impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "resolvedJson", resolvedConfigDataStringJson, "err", err) + return nil, err + } + return bean2.NewDeploymentAndCmCsConfig().WithConfigData(configDataJson).WithResourceType(resourceType). + WithVariableSnapshot(variableSnapshotMap).WithResolvedValue(resolvedConfigDataStringJson), nil +} + +func (impl *ConfigMapHistoryServiceImpl) encodeSecretDataFromNonAdminUsers(configDataList []*bean3.ConfigData, userHasAdminAccess bool) { + for _, config := range configDataList { + if config.Data != nil { + if !userHasAdminAccess { + //removing keys and sending + resultMap := make(map[string]string) + resultMapFinal := make(map[string]string) + err := json.Unmarshal(config.Data, &resultMap) + if err != nil { + impl.logger.Errorw("unmarshal failed", "error", err) + return + } + for key, _ := range resultMap { + //hard-coding values to show them as hidden to user + resultMapFinal[key] = "*****" + } + config.Data, err = utils.ConvertToJsonRawMessage(resultMapFinal) + if err != nil { + impl.logger.Errorw("error while marshaling request", "err", err) + return + } + } + } + } +} + +func (impl ConfigMapHistoryServiceImpl) getConfigDataRequestForHistory(history *repository.ConfigmapAndSecretHistory) (*bean3.ConfigsList, *bean3.SecretsList, error) { + + configsList := &bean3.ConfigsList{} + secretsList := &bean3.SecretsList{} + if history.IsConfigmapHistorySecretType() { + err := json.Unmarshal([]byte(history.Data), secretsList) + if err != nil { + impl.logger.Errorw("error while Unmarshal in secret history data", "error", err) + return configsList, secretsList, err + } + return configsList, secretsList, nil + } else if history.IsConfigmapHistoryConfigMapType() { + err := json.Unmarshal([]byte(history.Data), configsList) + if err != nil { + impl.logger.Errorw("error while Unmarshal in config history data", "historyData", history.Data, "error", err) + return configsList, secretsList, err + } + return configsList, secretsList, nil + } + return configsList, secretsList, nil +} diff --git a/pkg/pipeline/history/DeploymentTemplateHistoryService.go b/pkg/pipeline/history/DeploymentTemplateHistoryService.go index 276a090500..d2eac61f80 100644 --- a/pkg/pipeline/history/DeploymentTemplateHistoryService.go +++ b/pkg/pipeline/history/DeploymentTemplateHistoryService.go @@ -50,6 +50,8 @@ type DeploymentTemplateHistoryService interface { // used for rollback GetDeployedHistoryByPipelineIdAndWfrId(ctx context.Context, pipelineId, wfrId int) (*HistoryDetailDto, error) + + GetTemplateHistoryModelForDeployedTemplateById(deploymentTemplateHistoryId, pipelineId int) (*repository.DeploymentTemplateHistory, error) } type DeploymentTemplateHistoryServiceImpl struct { @@ -407,3 +409,12 @@ func (impl DeploymentTemplateHistoryServiceImpl) CheckIfTriggerHistoryExistsForP exists = true return deploymentTemplateHistoryId, exists, err } + +func (impl DeploymentTemplateHistoryServiceImpl) GetTemplateHistoryModelForDeployedTemplateById(deploymentTemplateHistoryId, pipelineId int) (*repository.DeploymentTemplateHistory, error) { + history, err := impl.deploymentTemplateHistoryRepository.GetHistoryForDeployedTemplateById(deploymentTemplateHistoryId, pipelineId) + if err != nil { + impl.logger.Errorw("error in getting deployment template history", "err", err, "deploymentTemplateHistoryId", deploymentTemplateHistoryId, "pipelineId", pipelineId) + return nil, err + } + return history, nil +} diff --git a/pkg/pipeline/history/repository/ConfigMapHistoryRepository.go b/pkg/pipeline/history/repository/ConfigMapHistoryRepository.go index e3a6918ee6..ebf45afe84 100644 --- a/pkg/pipeline/history/repository/ConfigMapHistoryRepository.go +++ b/pkg/pipeline/history/repository/ConfigMapHistoryRepository.go @@ -39,6 +39,7 @@ type ConfigMapHistoryRepository interface { GetHistoryByPipelineIdAndWfrId(pipelineId, wfrId int, configType ConfigType) (*ConfigmapAndSecretHistory, error) GetDeployedHistoryForPipelineIdOnTime(pipelineId int, deployedOn time.Time, configType ConfigType) (*ConfigmapAndSecretHistory, error) GetDeployedHistoryList(pipelineId, baseConfigId int, configType ConfigType, componentName string) ([]*ConfigmapAndSecretHistory, error) + GetDeployedHistoryByPipelineIdAndDeployedOn(pipelineId int, deployedOn time.Time, configType ConfigType) (*ConfigmapAndSecretHistory, error) } type ConfigMapHistoryRepositoryImpl struct { @@ -71,6 +72,13 @@ type ConfigmapAndSecretHistory struct { DeployedByEmailId string `sql:"-"` } +func (r *ConfigmapAndSecretHistory) IsConfigmapHistorySecretType() bool { + return r.DataType == SECRET_TYPE +} + +func (r *ConfigmapAndSecretHistory) IsConfigmapHistoryConfigMapType() bool { + return r.DataType == CONFIGMAP_TYPE +} func (impl ConfigMapHistoryRepositoryImpl) CreateHistory(tx *pg.Tx, model *ConfigmapAndSecretHistory) (*ConfigmapAndSecretHistory, error) { var err error if tx != nil { @@ -149,3 +157,14 @@ func (impl ConfigMapHistoryRepositoryImpl) GetDeployedHistoryForPipelineIdOnTime Select() return &history, err } + +func (impl ConfigMapHistoryRepositoryImpl) GetDeployedHistoryByPipelineIdAndDeployedOn(pipelineId int, deployedOn time.Time, configType ConfigType) (*ConfigmapAndSecretHistory, error) { + var history ConfigmapAndSecretHistory + err := impl.dbConnection.Model(&history). + Where("pipeline_id = ?", pipelineId). + Where("data_type = ?", configType). + Where("deployed_on = ?", deployedOn). + Where("deployed = ?", true). + Select() + return &history, err +} diff --git a/pkg/pipeline/history/repository/PipelineStrategyHistoryRepository.go b/pkg/pipeline/history/repository/PipelineStrategyHistoryRepository.go index 092a81239e..755d38686c 100644 --- a/pkg/pipeline/history/repository/PipelineStrategyHistoryRepository.go +++ b/pkg/pipeline/history/repository/PipelineStrategyHistoryRepository.go @@ -35,6 +35,7 @@ type PipelineStrategyHistoryRepository interface { GetHistoryByPipelineIdAndWfrId(ctx context.Context, pipelineId, wfrId int) (*PipelineStrategyHistory, error) CheckIfTriggerHistoryExistsForPipelineIdOnTime(pipelineId int, deployedOn time.Time) (bool, error) GetDeployedHistoryList(pipelineId, baseConfigId int) ([]*PipelineStrategyHistory, error) + FindPipelineStrategyForDeployedOnAndPipelineId(pipelineId int, deployedOn time.Time) (PipelineStrategyHistory, error) } type PipelineStrategyHistoryRepositoryImpl struct { @@ -145,3 +146,11 @@ func (impl PipelineStrategyHistoryRepositoryImpl) CheckIfTriggerHistoryExistsFor Exists() return exists, err } + +func (impl PipelineStrategyHistoryRepositoryImpl) FindPipelineStrategyForDeployedOnAndPipelineId(pipelineId int, deployedOn time.Time) (PipelineStrategyHistory, error) { + var history PipelineStrategyHistory + err := impl.dbConnection.Model(&history). + Where("pipeline_strategy_history.deployed_on = ?", deployedOn). + Where("pipeline_strategy_history.pipeline_id = ?", pipelineId).Select() + return history, err +} diff --git a/wire_gen.go b/wire_gen.go index 0bee1a76cd..a53fca7b97 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -951,7 +951,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl, chartRefServiceImpl, pipelineRepositoryImpl) + deploymentConfigurationServiceImpl, err := configDiff.NewDeploymentConfigurationServiceImpl(sugaredLogger, configMapServiceImpl, appRepositoryImpl, environmentRepositoryImpl, chartServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentTemplateHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, configMapRepositoryImpl, pipelineDeploymentConfigServiceImpl, chartRefServiceImpl, pipelineRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl) if err != nil { return nil, err } From ebcfd494c54cf510eb7077b3c7576ef2d5ec2418 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 17 Oct 2024 13:40:07 +0530 Subject: [PATCH 31/32] code review incorporation :- kartik --- pkg/cluster/repository/EnvironmentRepository.go | 1 - pkg/configDiff/DeploymentConfigurationService.go | 15 +++++++-------- pkg/configDiff/bean/bean.go | 4 +++- pkg/generateManifest/DeploymentTemplateService.go | 10 ++++------ pkg/generateManifest/adapter.go | 14 ++++++++++++++ pkg/pipeline/history/ConfigMapHistoryService.go | 7 +------ 6 files changed, 29 insertions(+), 22 deletions(-) create mode 100644 pkg/generateManifest/adapter.go diff --git a/pkg/cluster/repository/EnvironmentRepository.go b/pkg/cluster/repository/EnvironmentRepository.go index a5c43168ae..15e806e2ca 100644 --- a/pkg/cluster/repository/EnvironmentRepository.go +++ b/pkg/cluster/repository/EnvironmentRepository.go @@ -168,7 +168,6 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindEnvByNameWithClusterDetails( Column("environment.*", "Cluster"). Where("environment.environment_name = ?", envName). Where("environment.active = ?", true). - Limit(1). Select() return environment, err } diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 7966efa159..4e0f6b166f 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -31,6 +31,7 @@ import ( "github.com/juju/errors" "go.uber.org/zap" "net/http" + "strconv" ) type DeploymentConfigurationService interface { @@ -140,7 +141,7 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con envId = env.Id clusterId = env.ClusterId systemMetadata.EnvironmentName = env.Name - systemMetadata.Namespace = env.Name + systemMetadata.Namespace = env.Namespace systemMetadata.ClusterName = env.Cluster.ClusterName } appId, err = impl.appRepository.FindAppIdByName(configDataQueryParams.AppName) @@ -161,6 +162,9 @@ func (impl *DeploymentConfigurationServiceImpl) GetAllConfigData(ctx context.Con func (impl *DeploymentConfigurationServiceImpl) getConfigDataForCdRollback(ctx context.Context, configDataQueryParams *bean2.ConfigDataQueryParams, userHasAdminAccess bool) (*bean2.DeploymentAndCmCsConfigDto, error) { // wfrId is expected in this case to return the expected data + if configDataQueryParams.WfrId == 0 { + return nil, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: strconv.Itoa(http.StatusNotFound), InternalMessage: bean2.ExpectedWfrIdNotPassedInQueryParamErr, UserMessage: bean2.ExpectedWfrIdNotPassedInQueryParamErr} + } return impl.getConfigDataForDeploymentHistory(ctx, configDataQueryParams, userHasAdminAccess) } @@ -323,12 +327,7 @@ func (impl *DeploymentConfigurationServiceImpl) getCmCsConfigHistory(ctx context return nil, err } resolvedConfigDataReq := &bean.ConfigDataRequest{ConfigData: resolvedConfigDataList} - resolvedConfigDataString, err := utils.ConvertToString(resolvedConfigDataReq) - if err != nil { - impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) - return nil, err - } - resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedConfigDataString) + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedConfigDataReq) if err != nil { impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedConfigDataString", "pipelineId", configDataQueryParams.PipelineId, "wfrId", configDataQueryParams.WfrId, "err", err) return nil, err @@ -355,7 +354,7 @@ func (impl *DeploymentConfigurationServiceImpl) encodeSecretDataFromNonAdminUser } for key, _ := range resultMap { //hard-coding values to show them as hidden to user - resultMapFinal[key] = "*****" + resultMapFinal[key] = bean2.SecretMaskedValue } config.Data, err = utils.ConvertToJsonRawMessage(resultMapFinal) if err != nil { diff --git a/pkg/configDiff/bean/bean.go b/pkg/configDiff/bean/bean.go index b1ce554032..30e183a229 100644 --- a/pkg/configDiff/bean/bean.go +++ b/pkg/configDiff/bean/bean.go @@ -229,5 +229,7 @@ type DeploymentTemplateMetadata struct { } const ( - NoDeploymentDoneForSelectedImage = "there were no deployments done for the selected image" + NoDeploymentDoneForSelectedImage = "there were no deployments done for the selected image" + ExpectedWfrIdNotPassedInQueryParamErr = "wfrId is expected in the query param which was not passed" + SecretMaskedValue = "*****" ) diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index 1a56931c71..fadd192726 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -235,11 +235,7 @@ func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Cont result.ResolvedData = resolvedValue result.VariableSnapshot = variableSnapshot if response != nil { - result.Data = response.Data - result.ResolvedData = response.ResolvedData - result.VariableSnapshot = response.VariableSnapshot - result.TemplateVersion = response.TemplateVersion - result.IsAppMetricsEnabled = response.IsAppMetricsEnabled + result = ConvertPointerDeploymentTemplateResponseToNonPointer(response) } return result, nil } @@ -251,7 +247,9 @@ func (impl DeploymentTemplateServiceImpl) GetDeploymentTemplate(ctx context.Cont if err != nil { return result, err } - result.Data = *manifest.Manifest + if manifest != nil { + result.Data = *manifest.Manifest + } return result, nil } diff --git a/pkg/generateManifest/adapter.go b/pkg/generateManifest/adapter.go new file mode 100644 index 0000000000..ca5755aa64 --- /dev/null +++ b/pkg/generateManifest/adapter.go @@ -0,0 +1,14 @@ +package generateManifest + +func ConvertPointerDeploymentTemplateResponseToNonPointer(r *DeploymentTemplateResponse) DeploymentTemplateResponse { + if r != nil { + return DeploymentTemplateResponse{ + Data: r.Data, + ResolvedData: r.ResolvedData, + VariableSnapshot: r.VariableSnapshot, + TemplateVersion: r.TemplateVersion, + IsAppMetricsEnabled: r.IsAppMetricsEnabled, + } + } + return DeploymentTemplateResponse{} +} diff --git a/pkg/pipeline/history/ConfigMapHistoryService.go b/pkg/pipeline/history/ConfigMapHistoryService.go index 107f486dea..07c375db8d 100644 --- a/pkg/pipeline/history/ConfigMapHistoryService.go +++ b/pkg/pipeline/history/ConfigMapHistoryService.go @@ -763,12 +763,7 @@ func (impl *ConfigMapHistoryServiceImpl) getResolvedConfigData(ctx context.Conte return nil, err } resolvedConfigDataReq := &bean3.ConfigDataRequest{ConfigData: resolvedConfigDataList} - resolvedConfigDataString, err := utils.ConvertToString(resolvedConfigDataReq) - if err != nil { - impl.logger.Errorw("getCmCsPublishedConfigResponse, error in converting config data to json raw message", "pipelineId", pipelineId, "deployedOn", deployedOn, "err", err) - return nil, err - } - resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedConfigDataString) + resolvedConfigDataStringJson, err := utils.ConvertToJsonRawMessage(resolvedConfigDataReq) if err != nil { impl.logger.Errorw("getCmCsPublishedConfigResponse, error in ConvertToJsonRawMessage for resolvedJson", "resolvedJson", resolvedConfigDataStringJson, "err", err) return nil, err From 0078d8040ffb23b815386fc96f985fac7f806390 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 17 Oct 2024 15:09:53 +0530 Subject: [PATCH 32/32] code review incorporation :- kartik 2 --- pkg/generateManifest/adapter.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/generateManifest/adapter.go b/pkg/generateManifest/adapter.go index ca5755aa64..9528ec2d8b 100644 --- a/pkg/generateManifest/adapter.go +++ b/pkg/generateManifest/adapter.go @@ -2,13 +2,7 @@ package generateManifest func ConvertPointerDeploymentTemplateResponseToNonPointer(r *DeploymentTemplateResponse) DeploymentTemplateResponse { if r != nil { - return DeploymentTemplateResponse{ - Data: r.Data, - ResolvedData: r.ResolvedData, - VariableSnapshot: r.VariableSnapshot, - TemplateVersion: r.TemplateVersion, - IsAppMetricsEnabled: r.IsAppMetricsEnabled, - } + return *r } return DeploymentTemplateResponse{} }