diff --git a/.gitbook.yaml b/.gitbook.yaml index 2f04cbca95..777f5fc9a3 100644 --- a/.gitbook.yaml +++ b/.gitbook.yaml @@ -16,7 +16,7 @@ redirects: setup/upgrade/devtron-upgrade-0.2.x-0.3.x: getting-started/upgrade/devtron-upgrade-0.2.x-0.3.x setup/global-configurations: user-guide/global-configurations/README.md setup/global-configurations/gitops: user-guide/global-configurations/gitops.md - setup/global-configurations/custom-charts: user-guide/global-configurations/custom-charts.md + setup/global-configurations/custom-charts: user-guide/global-configurations/deployment-charts.md setup/global-configurations/user-access: user-guide/global-configurations/authorization/user-access.md setup/global-configurations/external-links: user-guide/global-configurations/external-links.md setup/global-configurations/projects: user-guide/global-configurations/projects.md @@ -109,7 +109,7 @@ redirects: getting-started/global-configurations/filter-condition: user-guide/global-configurations/filter-condition.md getting-started/global-configurations/build-infra: user-guide/global-configurations/build-infra.md getting-started/global-configurations/gitops: user-guide/global-configurations/gitops.md - getting-started/global-configurations/custom-charts: user-guide/global-configurations/custom-charts.md + getting-started/global-configurations/custom-charts: user-guide/global-configurations/deployment-charts.md getting-started/global-configurations/external-links: user-guide/global-configurations/external-links.md getting-started/global-configurations/projects: user-guide/global-configurations/projects.md getting-started/global-configurations/manage-notification: user-guide/global-configurations/manage-notification.md @@ -127,4 +127,5 @@ redirects: user-guide/clusters: user-guide/resource-browser.md usage/clusters: user-guide/resource-browser.md global-configurations/authorization/sso-login/okta: user-guide/global-configurations/authorization/sso/okta.md - usage/applications/creating-application/ci-pipeline/ci-build-pre-post-plugins: user-guide/creating-application/workflow/ci-build-pre-post-plugins.md \ No newline at end of file + usage/applications/creating-application/ci-pipeline/ci-build-pre-post-plugins: user-guide/creating-application/workflow/ci-build-pre-post-plugins.md + global-configurations/custom-charts: user-guide/global-configurations/deployment-charts.md \ No newline at end of file diff --git a/api/restHandler/DeploymentConfigurationRestHandler.go b/api/restHandler/DeploymentConfigurationRestHandler.go index a29776a6b6..144838da01 100644 --- a/api/restHandler/DeploymentConfigurationRestHandler.go +++ b/api/restHandler/DeploymentConfigurationRestHandler.go @@ -115,9 +115,11 @@ func (handler *DeploymentConfigurationRestHandlerImpl) enforceForAppAndEnv(appNa return false } - object = handler.enforcerUtil.GetEnvRBACNameByAppAndEnvName(appName, envName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, action, object); !ok { - return false + if len(envName) > 0 { + object = handler.enforcerUtil.GetEnvRBACNameByAppAndEnvName(appName, envName) + if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, action, object); !ok { + return false + } } return true } diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 47626841d9..1ae6d4f33d 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "golang.org/x/exp/maps" "io" "net/http" @@ -496,19 +495,13 @@ func (handler *PipelineConfigRestHandlerImpl) getCdPipelinesForCIPatchRbac(patch // find the workflow in which we are patching and use the workflow id to fetch all the workflow mappings using the workflow. // get cd pipeline ids from those workflows and fetch the cd pipelines. - // get the ciPipeline - switchFromPipelineId, switchFromType := patchRequest.SwitchSourceInfo() - - // in app workflow mapping all the build source types are 'CI_PIPELINE' type, except external -> WEBHOOK. - componentType := appWorkflow.CIPIPELINE - if switchFromType == CiPipeline.EXTERNAL { - componentType = appWorkflow.WEBHOOK - } + // get the ciPipeline patch source info + componentId, componentType := patchRequest.PatchSourceInfo() // the appWorkflowId can be taken from patchRequest.AppWorkflowId but doing this can make 2 sources of truth to find the workflow - sourceAppWorkflowMapping, err := handler.appWorkflowService.FindWFMappingByComponent(componentType, switchFromPipelineId) + sourceAppWorkflowMapping, err := handler.appWorkflowService.FindWFMappingByComponent(componentType, componentId) if err != nil { - handler.Logger.Errorw("error in finding the appWorkflowMapping using componentId and componentType", "componentType", componentType, "componentId", switchFromPipelineId, "err", err) + handler.Logger.Errorw("error in finding the appWorkflowMapping using componentId and componentType", "componentType", componentType, "componentId", componentId, "err", err) return nil, err } diff --git a/devtron-images.txt.source b/devtron-images.txt.source new file mode 100644 index 0000000000..778292ffed --- /dev/null +++ b/devtron-images.txt.source @@ -0,0 +1,39 @@ +quay.io/devtron/image-scanner:137872c2-141-23848 +quay.io/devtron/inception:473deaa4-185-21582 +quay.io/devtron/hyperion:291c4c75-280-23860 +public.ecr.aws/docker/library/redis:7.0.5-alpine +quay.io/argoproj/argocd:v2.5.2 +quay.io/argoproj/workflow-controller:v3.4.3 +quay.io/devtron/authenticator:e414faff-393-13273 +quay.io/devtron/bats:v1.4.1 +quay.io/devtron/busybox:1.31.1 +quay.io/devtron/chart-sync:5a1d0301-150-23845 +quay.io/devtron/curl:7.73.0 +quay.io/devtron/dashboard:5f95d187-690-23841 +quay.io/devtron/devtron-utils:dup-chart-repo-v1.1.0 +quay.io/devtron/devtron:291c4c75-434-23853 +quay.io/devtron/ci-runner:48aca9f4-138-23844 +quay.io/devtron/dex:v2.30.2 +quay.io/devtron/git-sensor:86e13283-200-23847 +quay.io/devtron/grafana:7.3.1 +quay.io/devtron/k8s-sidecar:1.1.0 +quay.io/devtron/k8s-utils:tutum-curl +quay.io/devtron/kubectl:latest +quay.io/devtron/kubelink:0dee6306-564-23843 +quay.io/devtron/kubewatch:850b40d5-419-23840 +quay.io/devtron/lens:56211042-333-23839 +quay.io/devtron/migrator:v4.16.2 +quay.io/devtron/nats-box +quay.io/devtron/nats-server-config-reloader:0.6.2 +quay.io/devtron/nats:2.9.3-alpine +quay.io/devtron/notifier:9639b1ab-372-23850 +quay.io/devtron/postgres:11.9 +quay.io/devtron/postgres_exporter:v0.10.1 +quay.io/devtron/prometheus-nats-exporter:0.9.0 +quay.io/devtron/minio:RELEASE.2021-02-14T04-01-33Z +quay.io/devtron/clair:4.3.6 +quay.io/devtron/postgres:11.9.0-debian-10-r26 +quay.io/devtron/postgres_exporter:v0.4.7 +quay.io/devtron/minio-mc:RELEASE.2021-02-14T04-28-06Z +quay.io/devtron/minideb:latest + diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 7412526613..e4ad4067a0 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -7,6 +7,7 @@ * [Install Devtron with CI/CD and GitOps (Argo CD)](setup/install/install-devtron-with-cicd-with-gitops.md) * [Install Devtron without Integrations](setup/install/install-devtron.md) * [Install Devtron on Minikube, Microk8s, K3s, Kind, Cloud VMs](setup/install/Install-devtron-on-Minikube-Microk8s-K3s-Kind.md) + * [Install Devtron on Airgapped Environment](setup/install/install-devtron-in-airgapped-environment.md) * [Demo on Popular Cloud Providers](setup/install/demo-tutorials.md) * [Backup for Disaster Recovery](setup/install/devtron-backup.md) * [Uninstall Devtron](setup/install/uninstall-devtron.md) @@ -24,7 +25,7 @@ * [Git Accounts](user-guide/global-configurations/git-accounts.md) * [Container/OCI Registry](user-guide/global-configurations/container-registries.md) * [Chart Repositories](user-guide/global-configurations/chart-repo.md) - * [Custom Charts](user-guide/global-configurations/custom-charts.md) + * [Deployment Charts](user-guide/global-configurations/deployment-charts.md) * [Authorization](user-guide/global-configurations/authorization/README.md) * [SSO Login Services](user-guide/global-configurations/sso-login.md) * [Google](user-guide/global-configurations/authorization/sso/google.md) @@ -135,9 +136,14 @@ * [Code-Scan](user-guide/plugins/code-scan.md) * [Copacetic](user-guide/plugins/copacetic.md) * [Copy Container Image](user-guide/plugins/copy-container-image.md) + * [Cosign](user-guide/plugins/cosign.md) + * [CraneCopy](user-guide/plugins/crane-copy.md) * [Dependency track - Maven & Gradle](user-guide/plugins/dependency-track-maven-gradle.md) * [Dependency track - NodeJS](user-guide/plugins/dependency-track-nodejs.md) * [Dependency track - Python](user-guide/plugins/dependency-track-python.md) + * [Devtron CD Trigger](user-guide/plugins/devtron-cd-trigger.md) + * [Devtron Job Trigger](user-guide/plugins/devtron-job-trigger.md) + * [DockerSlim](user-guide/plugins/docker-slim.md) * [GoLang-migrate](user-guide/plugins/golang-migrate.md) * [Jenkins](user-guide/plugins/jenkins.md) * [K6 Load Testing](user-guide/plugins/k6-load-testing.md) @@ -159,4 +165,4 @@ * [Pull Helm Charts from OCI Registry](user-guide/use-cases/oci-pull.md) * [Telemetry Overview](user-guide/telemetry.md) * [Devtron on Graviton](reference/graviton.md) -* [Release Notes](https://github.com/devtron-labs/devtron/releases) \ No newline at end of file +* [Release Notes](https://github.com/devtron-labs/devtron/releases) diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 69402187cb..c219ea5bd1 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -64,9 +64,9 @@ Temporarily marking a node as unschedulable, preventing new pods from being assi CronJob is used to create Jobs on a repeating schedule. It is commonly used for running periodic tasks with no manual intervention. In Devtron, you can view a list of cronjobs by going to Resource Browser → (choose a cluster) → Workloads → CronJob. [Read More...](../user-guide/creating-application/deployment-template/job-and-cronjob.md#2.-cronjob) -### Custom Charts +### Deployment Charts -Devtron offers a variety of ready-made Helm charts for common tasks and functions. If you have a specific need that isn't met by these preconfigured charts, super-admins have the permission to upload their own custom charts. Once uploaded, these custom charts become accessible for use by all users on the Devtron platform. [Read More...](../user-guide/global-configurations/custom-charts.md) +Devtron offers a variety of ready-made Helm charts for common tasks and functions. If you have a specific need that isn't met by these preconfigured charts, super-admins have the permission to upload their own charts. Once uploaded, these charts become accessible for use by all users on the Devtron platform. [Read More...](../user-guide/global-configurations/deployment-charts.md) ### DaemonSet diff --git a/docs/setup/install/install-devtron-in-airgapped-environment.md b/docs/setup/install/install-devtron-in-airgapped-environment.md new file mode 100644 index 0000000000..8d705fb77a --- /dev/null +++ b/docs/setup/install/install-devtron-in-airgapped-environment.md @@ -0,0 +1,245 @@ +# Devtron Installation in an Airgapped Environment + +## Introduction + +In certain scenarios, you may need to deploy Devtron to a Kubernetes cluster that isn’t connected to the internet. Such air-gapped environments are used for various reasons, particularly in industries with strict regulatory requirements like healthcare, banking, and finance. This is because air-gapped environments aren't exposed to the public internet; therefore, they create a controlled and secure space for handling sensitive data and operations. + +### Prerequisites + +1. Install `podman` or `docker` on the VM from where you're executing the installation commands. +2. Clone the Devtron Helm chart: + + ```bash + git clone https://github.com/devtron-labs/devtron.git + cd devtron + ``` + +3. Set the values of `TARGET_REGISTRY`, `TARGET_REGISTRY_USERNAME`, and `TARGET_REGISTRY_TOKEN`. This registry should be accessible from the VM where you are running the cloning script and the K8s cluster where you’re installing Devtron. + +{% hint style="warning" %} +### Note +If you are using Docker, the TARGET_REGISTRY should be in the format `docker.io/` +{% endhint %} + +--- + +## Docker Instructions + +### Platform Selection + +#### For Linux/amd64 + + ```bash + export PLATFORM="linux/amd64" + ``` +#### For Linux/arm64 + + ```bash + export PLATFORM="linux/arm64" + ``` + + + +1. Set the environment variables + + ```bash + # Set the source registry URL + export SOURCE_REGISTRY="quay.io/devtron" + + # Set the target registry URL, username, and token/password + export TARGET_REGISTRY="" + export TARGET_REGISTRY_USERNAME="" + export TARGET_REGISTRY_TOKEN="" + + # Set the source and target image file names with default values if not already set + SOURCE_IMAGES_LIST="${SOURCE_IMAGES_LIST:=devtron-images.txt.source}" + TARGET_IMAGES_LIST="${TARGET_IMAGES_LIST:=devtron-images.txt.target}" + ``` + +2. Log in to the target Docker registry + + ```bash + docker login -u $TARGET_REGISTRY_USERNAME -p $TARGET_REGISTRY_TOKEN $TARGET_REGISTRY + ``` + +3. Clone the images + + ```bash + while IFS= read -r source_image; do + # Check if the source image belongs to the quay.io/devtron registry + if [[ "$source_image" == quay.io/devtron/* ]]; then + # Replace the source registry with the target registry in the image name + target_image="${source_image/quay.io\/devtron/$TARGET_REGISTRY}" + + # Check if the source image belongs to the quay.io/argoproj registry + elif [[ "$source_image" == quay.io/argoproj/* ]]; then + # Replace the source registry with the target registry in the image name + target_image="${source_image/quay.io\/argoproj/$TARGET_REGISTRY}" + + # Check if the source image belongs to the public.ecr.aws/docker/library registry + elif [[ "$source_image" == public.ecr.aws/docker/library/* ]]; then + # Replace the source registry with the target registry in the image name + target_image="${source_image/public.ecr.aws\/docker\/library/$TARGET_REGISTRY}" + fi + + # Pull the image from the source registry + docker pull --platform $PLATFORM $source_image + + # Tag the image with the new target registry name + docker tag $source_image $target_image + + # Push the image to the target registry + docker push $target_image + + # Output the updated image name + echo "Updated image: $target_image" + + # Append the new image name to the target image file + echo "$target_image" >> "$TARGET_IMAGES_LIST" + + done < "$SOURCE_IMAGES_LIST" + ``` +--- + +## Podman Instructions + +### For Multi-arch + +1. Set the environment variables + + ```bash + export SOURCE_REGISTRY="quay.io/devtron" + export SOURCE_REGISTRY_TOKEN=#Enter token provided by Devtron team + export TARGET_REGISTRY=#Enter target registry url + export TARGET_REGISTRY_USERNAME=#Enter target registry username + export TARGET_REGISTRY_TOKEN=#Enter target registry token/password + ``` + +2. Log in to the target Podman registry + + ```bash + podman login -u $TARGET_REGISTRY_USERNAME -p $TARGET_REGISTRY_TOKEN $TARGET_REGISTRY + ``` + +3. Clone the images + + ```bash + SOURCE_REGISTRY="quay.io/devtron" + TARGET_REGISTRY=${TARGET_REGISTRY} + SOURCE_IMAGES_FILE_NAME="${SOURCE_IMAGES_FILE_NAME:=devtron-images.txt.source}" + TARGET_IMAGES_FILE_NAME="${TARGET_IMAGES_FILE_NAME:=devtron-images.txt.target}" + + cp $SOURCE_IMAGES_FILE_NAME $TARGET_IMAGES_FILE_NAME + while read source_image; do + if [[ "$source_image" == *"workflow-controller:"* || "$source_image" == *"argoexec:"* || "$source_image" == *"argocd:"* ]] + then + SOURCE_REGISTRY="quay.io/argoproj" + sed -i "s|${SOURCE_REGISTRY}|${TARGET_REGISTRY}|g" $TARGET_IMAGES_FILE_NAME + elif [[ "$source_image" == *"redis:"* ]] + then + SOURCE_REGISTRY="public.ecr.aws/docker/library" + sed -i "s|${SOURCE_REGISTRY}|${TARGET_REGISTRY}|g" $TARGET_IMAGES_FILE_NAME + else + SOURCE_REGISTRY="quay.io/devtron" + sed -i "s|${SOURCE_REGISTRY}|${TARGET_REGISTRY}|g" $TARGET_IMAGES_FILE_NAME + fi + done <$SOURCE_IMAGES_FILE_NAME + echo "Target Images file finalized" + + while read -r -u 3 source_image && read -r -u 4 target_image ; do + echo "Pushing $source_image $target_image" + podman manifest create $source_image + podman manifest add $source_image $source_image --all + podman manifest push $source_image $target_image --all + done 3<"$SOURCE_IMAGES_FILE_NAME" 4<"$TARGET_IMAGES_FILE_NAME" + ``` + +--- + +## Devtron Installation + +Before starting, ensure you have created an image pull secret for your registry if authentication is required. + +1. Create the namespace (if not already created) + ```bash + kubectl create ns devtroncd + ``` + +2. Create the Docker registry secret + ```bash + kubectl create secret docker-registry devtron-imagepull \ + --namespace devtroncd \ + --docker-server=$TARGET_REGISTRY \ + --docker-username=$TARGET_REGISTRY_USERNAME \ + --docker-password=$TARGET_REGISTRY_TOKEN + ``` + If you are installing Devtron with the CI/CD module or using Argo CD, create the secret in the following namespaces else, you can skip this step-: + ```bash + kubectl create secret docker-registry devtron-imagepull \ + --namespace devtron-cd \ + --docker-server=$TARGET_REGISTRY \ + --docker-username=$TARGET_REGISTRY_USERNAME \ + --docker-password=$TARGET_REGISTRY_TOKEN + kubectl create secret docker-registry devtron-imagepull \ + --namespace devtron-ci \ + --docker-server=$TARGET_REGISTRY \ + --docker-username=$TARGET_REGISTRY_USERNAME \ + --docker-password=$TARGET_REGISTRY_TOKEN + kubectl create secret docker-registry devtron-imagepull \ + --namespace argo \ + --docker-server=$TARGET_REGISTRY \ + --docker-username=$TARGET_REGISTRY_USERNAME \ + --docker-password=$TARGET_REGISTRY_TOKEN + ``` + +3. Navigate to the Devtron Helm chart directory + ```bash + cd charts/devtron + ``` + + +### Install Devtron without any Integration + +Use the below command to install Devtron without any Integrations + +1. Without `imagePullSecrets`: + ```bash + helm install devtron . -n devtroncd --set global.containerRegistry="$TARGET_REGISTRY" + ``` + +2. With `imagePullSecrets`: + ```bash + helm install devtron . -n devtroncd --set global.containerRegistry="$TARGET_REGISTRY" --set global.imagePullSecrets[0].name=devtron-imagepull + ``` + +### Installing Devtron with CI/CD Mode +Use the below command to install Devtron with only the CI/CD module + +1. Without `imagePullSecrets`: + ```bash + helm install devtron . -n devtroncd --set installer.modules={cicd} --set global.containerRegistry="$TARGET_REGISTRY" + ``` + +2. With `imagePullSecrets`: + ```bash + helm install devtron . -n devtroncd --set installer.modules={cicd} --set global.containerRegistry="$TARGET_REGISTRY" --set global.imagePullSecrets[0].name=devtron-imagepull + ``` + +### Install Devtron with CICD Mode including Argocd + +Use the below command to install Devtron with the CI/CD module and Argo CD + +1. Without `imagePullSecrets`: + ```bash + helm install devtron . --create-namespace -n devtroncd --set installer.modules={cicd} --set argo-cd.enabled=true --set global.containerRegistry="$TARGET_REGISTRY" --set argo-cd.global.image.repository="${TARGET_REGISTRY}/argocd" --set argo-cd.redis.image.repository="${TARGET_REGISTRY}/redis" + ``` + +2. With `imagePullSecrets`: + ```bash + helm install devtron . --create-namespace -n devtroncd --set installer.modules={cicd} --set argo-cd.enabled=true --set global.containerRegistry="$TARGET_REGISTRY" --set argo-cd.global.image.repository="${TARGET_REGISTRY}/argocd" --set argo-cd.redis.image.repository="${TARGET_REGISTRY}/redis" --set global.imagePullSecrets[0].name=devtron-imagepull + ``` + +--- + +## Next Steps +After installation, refer [Devtron installation documentation](https://docs.devtron.ai/install/install-devtron-with-cicd-with-gitops#devtron-dashboard) for further steps, including obtaining the dashboard URL and the admin password. diff --git a/docs/user-guide/creating-application/deployment-template.md b/docs/user-guide/creating-application/deployment-template.md index c3e7438236..ab117b9a3c 100644 --- a/docs/user-guide/creating-application/deployment-template.md +++ b/docs/user-guide/creating-application/deployment-template.md @@ -23,7 +23,7 @@ Users need to have [Admin role](../user-guide/global-configurations/authorizatio {% hint style="warning" %} ### Note -After you select and save a chart type for a given application, you won't be able to change it later. Make sure to choose the correct chart type before saving. You can select a chart from [Devtron Charts](#from-devtron-charts) or [Custom Charts](#from-custom-charts). +After you select and save a chart type for a given application, you won't be able to change it later. Make sure to choose the correct chart type before saving. You can select a chart from [Devtron Charts](#from-devtron-charts) or other [Deployment Charts](#from-deployment-charts). {% endhint %} ### From Devtron Charts @@ -37,10 +37,10 @@ You can select a default deployment chart from the following options: ![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/deployment-template/select-devtron-chart.gif) -### From Custom Charts +### From Deployment Charts {% hint style="warning" %} -This option will be available only if a custom chart exists. If it doesn't, a user with `super admin` permission may upload one in [Global Configurations → Custom Charts](../global-configurations/custom-charts.md). +This option will be available only if a custom chart exists. If it doesn't, a user with `super admin` permission may upload one in [Global Configurations → Deployment Charts](../global-configurations/deployment-charts.md). {% endhint %} You can select an available custom chart as shown below. You can also view the description of the custom charts in the list. @@ -110,20 +110,26 @@ Click **Save Changes**. If you want to do additional configurations, then click {% hint style="warning" %} ### Who Can Perform This Action? -Superadmin can define and apply custom deployment schema using API +Superadmin can define and apply custom deployment schema. {% endhint %} By default, the `Basic (GUI)` section comes with multiple predefined fields as seen earlier [in the table](#2-basic-configuration). However, if you wish to display a different set of fields to your team, you can modify the whole section as per your requirement. -{% embed url="https://www.youtube.com/watch?v=09VP1I-WvUs" caption="JSON-driven Deployment Schema" %} - This is useful in scenarios where: * Your team members find it difficult to understand and edit the [Advanced (YAML)](#3-advanced-yaml) section. * You frequently edit certain fields in Advanced (YAML), which you expect to remain easily accessible in Basic (GUI) section. * You don't require some fields in Basic (GUI) section. * You need the autonomy to keep the Basic (GUI) unique for applications/clusters/environments/charts, or display the same Basic (GUI) everywhere. -This is possible by passing a custom JSON (deployment schema) of your choice through the following API. You may need to run the API with the `POST` method if you are doing it for the first time. +{% hint style="info" %} +There are two ways you can customize the Basic GUI, use any one of the following: +1. From [Deployment Charts](../global-configurations/deployment-charts.md#editing-gui-schema-of-deployment-charts) section +2. Using APIs (explained below) +{% endhint %} + +{% embed url="https://www.youtube.com/watch?v=09VP1I-WvUs" caption="JSON-driven Deployment Schema" %} + +You can pass a custom JSON (deployment schema) of your choice through the following API. You may need to run the API with the `POST` method if you are doing it for the first time. ``` PUT {{DEVTRON_BASEURL}}/orchestrator/deployment/template/schema diff --git a/docs/user-guide/global-configurations/README.md b/docs/user-guide/global-configurations/README.md index 488ed2328b..6a7aa4cf29 100644 --- a/docs/user-guide/global-configurations/README.md +++ b/docs/user-guide/global-configurations/README.md @@ -18,7 +18,7 @@ Before you start creating an application, we recommend to provide basic informat [Chart Repositories](chart-repo.md) -[Custom Charts](custom-charts.md) +[Deployment Charts](deployment-charts.md) [Authorization](authorization/README.md) diff --git a/docs/user-guide/global-configurations/custom-charts.md b/docs/user-guide/global-configurations/deployment-charts.md similarity index 68% rename from docs/user-guide/global-configurations/custom-charts.md rename to docs/user-guide/global-configurations/deployment-charts.md index 18fefc8759..256e105cc3 100644 --- a/docs/user-guide/global-configurations/custom-charts.md +++ b/docs/user-guide/global-configurations/deployment-charts.md @@ -1,4 +1,4 @@ -# Custom Charts +# Deployment Charts Devtron includes predefined helm charts that cover the majority of use cases. For any use case not addressed by the default helm charts, you can upload your own helm chart and use it as a custom chart in Devtron. @@ -8,7 +8,7 @@ For any use case not addressed by the default helm charts, you can upload your o > A super admin can upload multiple versions of a custom helm chart. -![Custom charts](https://devtron-public-asset.s3.us-east-2.amazonaws.com/custom-charts/custom-charts-lists.png) +![Figure 1: Deployment Charts](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/gc-deployment-charts.jpg) ## Prerequisites @@ -99,7 +99,9 @@ helm package my-custom-chart The above command will create a `my-custom-chart-0.1.0.tgz` file. -## Uploading a custom chart +--- + +## Uploading a Deployment Chart > A custom chart can only be uploaded by a super admin. @@ -142,21 +144,73 @@ The following are the validation results: ![Already exists](https://devtron-public-asset.s3.us-east-2.amazonaws.com/custom-charts/List+-+Empty-1.png) -## View the custom charts +--- + +## Viewing Deployment Charts > All users can view the custom charts. -To view a list of available custom charts, go to **Global Configurations > Custom charts** page. +To view the list of available custom charts, go to **Global Configurations → Deployment Charts** page. * The charts can be searched with their name, version, or description. * New [custom charts can be uploaded](#uploading-a-custom-chart) by selecting **Upload chart**. -![Custom charts](https://devtron-public-asset.s3.us-east-2.amazonaws.com/custom-charts/custom-charts-lists.png) +![Custom charts](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/upload-custom-chart.jpg) + +--- -## Use the custom chart in an application +## Using Deployment Chart in Application The custom charts can be used from the [Deployment Template](../creating-application/deployment-template.md) section. > **Info**: > > The deployment strategy for a custom chart is fetched from the custom chart template and cannot be configured in the [CD pipeline](../creating-application/workflow/cd-pipeline.md#deployment-strategies). + +--- + +## Editing GUI Schema of Deployment Charts [![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/elements/EnterpriseTag.svg)](https://devtron.ai/pricing) + +{% hint style="warning" %} +### Who Can Perform This Action? +Only superadmins can edit the GUI schema of deployment charts. +{% endhint %} + +{% hint style="info" %} +### Reference +This section is an extension of [Customize Basic GUI](../creating-application/deployment-template.md#customize-basic-gui) feature within **App Configuration** → **Base Deployment Template**. Refer the document to know more about the significance of having a customizable GUI schema for your deployment templates. +{% endhint %} + +You can edit the GUI schema of both the deployment charts: +1. Charts provided by Devtron (*Deployment*, *Job & CronJob*, *Rollout Deployment*, and *StatefulSet*) +2. Custom charts uploaded by you + +### Tutorial + +{% embed url="https://www.youtube.com/watch?v=93tGIsM1qC8" caption="JSON-driven Deployment Schema" %} + +### Steps + +In this example, we will edit the Deployment chart type provided by Devtron. + +1. Click the edit button next to the chart as shown below. + + ![Edit GUI Schema Button](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/edit-chart-schema.jpg) + +2. A GUI schema is available for you to edit in case of Devtron charts. In case of custom charts, you may have to define a GUI schema yourself. To know how to create such GUI schema, refer [RJSF JSON Schema Tool](https://rjsf-team.github.io/react-jsonschema-form/). + + ![Editable Schema](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/gui-schema.jpg) + +3. You may start editing the schema by excluding existing fields/objects or including more of them. Click the **Refer YAML** button to view all the supported fields. + + ![Refer YAML Button](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/refer-yaml.gif) + +4. While editing the schema, you may use the **Preview GUI** option for a real-time preview of your changes. + + ![Preview GUI Button](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/preview-gui.gif) + +5. Click **Save Changes**. + + ![Save Changes](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/global-configurations/deployment-charts/save-changes.jpg) + +Next, if you go to **App Configuration** → **Base Deployment Template**, you will be able to see the deployment template fields (in Basic GUI) as per your customized schema. \ No newline at end of file diff --git a/docs/user-guide/plugins/code-scan.md b/docs/user-guide/plugins/code-scan.md index 18eafedf92..cbefde15af 100644 --- a/docs/user-guide/plugins/code-scan.md +++ b/docs/user-guide/plugins/code-scan.md @@ -4,7 +4,7 @@ The Code Scan plugin of Devtron allows you to perform the code scanning using Trivy. By integrating the **Code Scan** plugin into your workflow you can detect common Vulnerabilities, Misconfigurations, License Risks, and Exposed Secrets in your code. ### Prerequisites -No prerequisites are required for integrating **Code Scan** plugin. +Before integrating the **Code Scan** plugin, install the [Vulnerability Scanning (Trivy/Clair)](https://docs.devtron.ai/usage/integrations/clair) integration from Devtron Stack Manager. --- @@ -34,11 +34,11 @@ e.g., `Code Scanning` ### Description Add a brief explanation of the task and the reason for choosing the plugin. Include information for someone else to understand the purpose of the task. -e.g., `The Code Scan plugin is integrated for scanning the in-code vulnerablities.` +e.g., `The Code Scan plugin is integrated for scanning the in-code vulnerabilities.` ### Input Variables -No input variables are required for Code Scan plugin. +No input variables are required for the Code Scan plugin. ### Output Variables Code Scan will not be generating an output variable. diff --git a/docs/user-guide/plugins/cosign.md b/docs/user-guide/plugins/cosign.md new file mode 100644 index 0000000000..d91fbb2d3e --- /dev/null +++ b/docs/user-guide/plugins/cosign.md @@ -0,0 +1,59 @@ +# Cosign + +## Introduction +The **Cosign** plugin by Devtron enables secure signing of your container images, enhancing supply chain security. It authenticates your identity as the creator and ensures image integrity, allowing users to verify the source and detect any tampering. This provides greater assurance to developers incorporating your artifacts into their workflows. + +### Prerequisites +Before integrating the Cosign plugin, ensure that you have configured the [Cosign](https://github.com/sigstore/cosign) and have a set of private and public keys to sign the container images. + +--- + +## Steps +1. Go to **Applications** → **Devtron Apps**. +2. Click your application. +3. Go to **App Configuration** → **Workflow Editor**. +4. Click **New Workflow** and navigate to the **Build and Deploy from Source Code**. +5. Fill the required fields in the **Create build pipeline** window and navigate to the **Post-build stage**. + +{% hint style="warning" %} +If you have already configured workflow, edit the build pipeline, and navigate to **Post-build stage**. +{% endhint %} + +6. Under 'TASKS', click the **+ Add task** button. +7. Click the **Cosign** plugin. +8. Enter the following [user inputs](#user-inputs) with appropriate values. +--- + +## User Inputs + +### Task Name +Enter the name of your task + +e.g., `Signing of container images` + +### Description +Add a brief explanation of the task and the reason for choosing the plugin. Include information for someone else to understand the purpose of the task. + +e.g., `The Cosign plugin is integrated for ensuring the authenticity of container images.` + +### Input Variables + +| Variable | Format | Description | Sample Value | +| ------------------------ | ------------ | ----------- | ------------ | +| PrivateKeyFilePath | STRING | Path of private key file in Git repo | cosign/cosign.key | +| PostCommand | STRING | Command to run after image is signed by Cosign | cosign verify $DOCKER_IMAGE | +| ExtraArguments | STRING | Arguments for Cosign command | --certificate-identity=name@example.com | +| CosignPassword | STRING | Password for Cosign private key | S3cur3P@ssw0rd123! | +| VariableAsPrivateKey | STRING | base64 encoded private-key | @{{COSIGN_PRIVATE_KEY}} | +| PreCommand | STRING | Command to get the required conditions to execute Cosign command | curl -sLJO https://raw.githubusercontent.com/devtron-labs/sampleRepo/branchName/private | + +### Trigger/Skip Condition +Here you can set conditions to execute or skip the task. You can select `Set trigger conditions` for the execution of a task or `Set skip conditions` to skip the task. + +### Output Variables +Cosign will not be generating an output variable. + +Click **Update Pipeline**. + + + diff --git a/docs/user-guide/plugins/crane-copy.md b/docs/user-guide/plugins/crane-copy.md new file mode 100644 index 0000000000..7b7de1e0f2 --- /dev/null +++ b/docs/user-guide/plugins/crane-copy.md @@ -0,0 +1,58 @@ +# CraneCopy + +## Introduction +The **CraneCopy** plugin by Devtron facilitates the transfer of multi-architecture container images between registries. When integrated into Devtron's Post-build stage, this plugin allows you to efficiently copy and store your container images to a specified target repository. + +### Prerequisites +No prerequisites are required for integrating the **CraneCopy** plugin. + +--- + +## Steps +1. Go to **Applications** → **Devtron Apps**. +2. Click your application. +3. Go to **App Configuration** → **Workflow Editor**. +4. Click **New Workflow** and navigate to the **Build and Deploy from Source Code**. +5. Fill the required fields in the **Create build pipeline** window and navigate to the **Post-build stage**. + +{% hint style="warning" %} +If you have already configured workflow, edit the build pipeline, and navigate to **Post-build stage**. +{% endhint %} + +6. Under 'TASKS', click the **+ Add task** button. +7. Click the **CraneCopy** plugin. +8. Enter the following [user inputs](#user-inputs) with appropriate values. + +--- + +## User Inputs + +### Task Name +Enter the name of your task + +e.g., `Copy and store container images` + +### Description +Add a brief explanation of the task and the reason for choosing the plugin. Include information for someone else to understand the purpose of the task. + +e.g., `The CraneCopy plugin is integrated to copy the container images from one registry to another.` + +### Input Variables + +| Variable | Format | Description | Sample Value | +| ------------------------ | ------------ | ----------- | ------------ | +| RegistryUsername | STRING | Username of target registry for authentication | admin | +| RegistryPassword | STRING | Password for the target registry for authentication | Tr5$mH7p | +| TargetRegistry | STRING | The target registry to push to image | docker.io/dockertest | + + +### Trigger/Skip Condition +Here you can set conditions to execute or skip the task. You can select `Set trigger conditions` for the execution of a task or `Set skip conditions` to skip the task. + +### Output Variables +CraneCopy will not be generating an output variable. + +Click **Update Pipeline**. + + + diff --git a/docs/user-guide/plugins/devtron-cd-trigger.md b/docs/user-guide/plugins/devtron-cd-trigger.md new file mode 100644 index 0000000000..8f7cbfcd7d --- /dev/null +++ b/docs/user-guide/plugins/devtron-cd-trigger.md @@ -0,0 +1,61 @@ +# Devtron-CD-Trigger + +## Introduction +The **Devtron CD Trigger** plugin allows you to trigger the PRE-CD, CD, or POST-CD stages of target Devtron App from within your current application workflow. This plugin offers flexibility in managing application dependencies and deployment sequences. For example, by incorporating this plugin at the pre-deployment stage of your application workflow, you can deploy another application that contains dependencies required by your current application, ensuring a coordinated deployment process. + +### Prerequisites +Before integrating the Devtron CD Trigger plugin, you need to properly configure the target Devtron App to ensure smooth execution. + +--- + +## Steps +1. Go to **Applications** → **Devtron Apps**. +2. Click your application. +3. Go to **App Configuration** → **Workflow Editor**. +4. Click **New Workflow** and navigate to the **Build and Deploy from Source Code**. +5. Fill the required fields in the **Create build pipeline** window and navigate to the **Create deployment pipeline**. +6. Fill the required fields in the **Deployment Stage** window and navigate to the **Post-Deployment stage**. + +{% hint style="warning" %} +If you have already configured workflow, edit the deployment pipeline, and navigate to **Post-Deployment stage**. +{% endhint %} + +6. Under 'TASKS', click the **+ Add task** button. +7. Select the **Devtron CD Trigger** plugin. +8. Enter the following [user inputs](#user-inputs) with appropriate values. +--- + +## User Inputs + +### Task Name +Enter the name of your task + +e.g., `Triggers CD Pipeline` + +### Description +Add a brief explanation of the task and the reason for choosing the plugin. Include information for someone else to understand the purpose of the task. + +e.g., `The Devtron CD Trigger plugin is integrated for triggering the CD stage of another application.` + +### Input Variables + +| Variable | Format | Description | Sample Value | +| ------------------------ | ------------ | ----------- | ------------ | +| DevtronApiToken | STRING | Enter target Devtron API token. | abc123DEFxyz456token789 | +| DevtronEndpoint | STRING | Enter the target URL of Devtron. | https://devtron.example.com | +| DevtronApp | STRING | Enter the target Devtron Application name/ID | plugin-demo | +| DevtronEnv | STRING | Enter the target Environment name/ID. Required if JobPipeline is not given | preview | +| StatusTimeoutSeconds | STRING | Enter the maximum time (in seconds) a user can wait for the application to deploy. Enter a positive integer value | 120 | +| GitCommitHash | STRING | Enter the git hash from which user wants to deploy its application. By default it takes latest Artifact ID to deploy the application | cf19e4fd348589kjhsdjn092nfse01d2234235sdsg | +| TargetTriggerStage | STRING | Enter the Trigger Stage PRE/DEPLOY/POST. Default value is `Deploy`. | PRE | + +### Trigger/Skip Condition +Here you can set conditions to execute or skip the task. You can select `Set trigger conditions` for the execution of a task or `Set skip conditions` to skip the task. + +### Output Variables +Devtron CD Trigger will not be generating an output variable. + +Click **Update Pipeline**. + + + diff --git a/docs/user-guide/plugins/devtron-job-trigger.md b/docs/user-guide/plugins/devtron-job-trigger.md new file mode 100644 index 0000000000..19b38b7703 --- /dev/null +++ b/docs/user-guide/plugins/devtron-job-trigger.md @@ -0,0 +1,61 @@ +# Devtron-Job-Trigger + +## Introduction +The **Devtron Job Trigger** plugin enables you to trigger Devtron Jobs from your current application workflow. For example, by integrating this plugin at the pre-deployment stage of your application workflow, you can trigger jobs designed to run migration scripts in your database. This ensures that necessary migrations are executed before your application is deployed. + +### Prerequisites +Before integrating the Devtron Job Trigger plugin, you need to properly configure the target Devtron Job to ensure smooth execution. + +--- + +## Steps +1. Go to **Applications** → **Devtron Apps**. +2. Click your application. +3. Go to **App Configuration** → **Workflow Editor**. +4. Click **New Workflow** and navigate to the **Build and Deploy from Source Code**. +5. Fill the required fields in the **Create build pipeline** window and navigate to the **Create deployment pipeline**. +6. Fill the required fields in the **Deployment Stage** window and navigate to the **Pre-Deployment stage**. + +{% hint style="warning" %} +If you have already configured workflow, edit the deployment pipeline, and navigate to **Pre-Deployment stage**. +{% endhint %} + +6. Under 'TASKS', click the **+ Add task** button. +7. Select the **Devtron Job Trigger** plugin. +8. Enter the following [user inputs](#user-inputs) with appropriate values. +--- + +## User Inputs + +### Task Name +Enter the name of your task + +e.g., `Triggers Devtron Job ` + +### Description +Add a brief explanation of the task and the reason for choosing the plugin. Include information for someone else to understand the purpose of the task. + +e.g., `The Devtron Job Trigger plugin is integrated for triggering the Devtron Job.` + +### Input Variables + +| Variable | Format | Description | Sample Value | +| ------------------------ | ------------ | ----------- | ------------ | +| DevtronApiToken | STRING | Enter Devtron API token with required permissions. | abc123def456token789 | +| DevtronEndpoint | STRING | Enter the URL of Devtron dashboard. | https://devtron.example.com | +| DevtronJob | STRING | Enter the name or ID of Devtron Job to be triggered | plugin-test-job | +| DevtronEnv | STRING | Enter the name or ID of the Environment where the job is to be triggered. If JobPipeline is given, ignore this field and do not assign any value | prod | +| JobPipeline | STRING | Enter the name or ID of the Job pipeline to be triggered. If DevtronEnv is given, ignore this field and do not assign any value | hello-world | +| GitCommitHash | STRING | Enter the commit hash from which the job is to be triggered. If not given then, will pick the latest | cf19e4fd348589kjhsdjn092nfse01d2234235sdsg | +| StatusTimeoutSeconds | NUMBER | Enter the maximum time to wait for the job status | 120 | + +### Trigger/Skip Condition +Here you can set conditions to execute or skip the task. You can select `Set trigger conditions` for the execution of a task or `Set skip conditions` to skip the task. + +### Output Variables +Devtron Job Trigger will not be generating an output variable. + +Click **Update Pipeline**. + + + diff --git a/docs/user-guide/plugins/docker-slim.md b/docs/user-guide/plugins/docker-slim.md new file mode 100644 index 0000000000..897cae9d14 --- /dev/null +++ b/docs/user-guide/plugins/docker-slim.md @@ -0,0 +1,63 @@ +# DockerSlim + +## Introduction +The **DockerSlim** plugin by Devtron helps you to optimize your container deployments by reducing Docker image size. Now with these lighter Docker images, you can perform faster deployments and enhance overall system efficiency. + +{% hint style="warning" %} +Support for Docker buildx images will be added soon. +{% endhint %} + +### Prerequisites +No prerequisites are required for integrating the **DockerSlim** plugin. + +--- + +## Steps +1. Go to **Applications** → **Devtron Apps**. +2. Click your application. +3. Go to **App Configuration** → **Workflow Editor**. +4. Click **New Workflow** and navigate to the **Build and Deploy from Source Code**. +5. Fill the required fields in the **Create build pipeline** window and navigate to the **Post-build stage**. + +{% hint style="warning" %} +If you have already configured workflow, edit the build pipeline, and navigate to **Post-build stage**. +{% endhint %} + +6. Under 'TASKS', click the **+ Add task** button. +7. Click the **DockerSlim** plugin. +8. Enter the following [user inputs](#user-inputs) with appropriate values. +--- + +## User Inputs + +### Task Name +Enter the name of your task + +e.g., `Reduce Docker image size` + +### Description +Add a brief explanation of the task and the reason for choosing the plugin. Include information for someone else to understand the purpose of the task. + +e.g., `The DockerSlim plugin is integrated for reducing the size of Docker image.` + +### Input Variables + +{% hint style="warning" %} +At `IncludePathFile` input variable list down the file path of essential files from your Dockerfile. Files for which the path is not listed at `IncludePathFile` will may be excluded from the Docker image to reduce size. +{% endhint %} + +| Variable | Format | Description | Sample Value | +| ------------------------ | ------------ | ----------- | ------------ | +| HTTPProbe | BOOL | Indicates whether the port is exposed in Dockerfile or not | false | +| IncludePathFile | STRING | File path of required files | /etc/nginx/include.conf | + +### Trigger/Skip Condition +Here you can set conditions to execute or skip the task. You can select `Set trigger conditions` for the execution of a task or `Set skip conditions` to skip the task. + +### Output Variables +DockerSlim will not be generating an output variable. + +Click **Update Pipeline**. + + + diff --git a/env_gen.md b/env_gen.md index 2f4d37a3d2..d6c935a303 100644 --- a/env_gen.md +++ b/env_gen.md @@ -6,7 +6,7 @@ | ACD_NAMESPACE | devtroncd | | | ACD_PASSWORD | | | | ACD_USERNAME | admin | | - | APP | orchestrator | | + | APP | orchestrator | Application name | | APP_SYNC_IMAGE | quay.io/devtron/chart-sync:1227622d-132-3775 | | | APP_SYNC_JOB_RESOURCES_OBJ | | | | APP_SYNC_SERVICE_ACCOUNT | chart-sync | | @@ -259,6 +259,7 @@ | USE_CASBIN_V2 | false | | | USE_CUSTOM_HTTP_TRANSPORT | false | | | USE_DEPLOYMENT_CONFIG_DATA | false | | + | USE_DOCKER_API_TO_GET_DIGEST | false | | | USE_EXTERNAL_NODE | false | | | USE_GIT_CLI | false | | | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | false | | diff --git a/fetchAllEnv/fetchAllEnv.go b/fetchAllEnv/fetchAllEnv.go index 86585d4385..6df7dcc317 100644 --- a/fetchAllEnv/fetchAllEnv.go +++ b/fetchAllEnv/fetchAllEnv.go @@ -36,9 +36,10 @@ type EnvField struct { } const ( - envFieldTypeTag = "env" - envDefaultFieldTypeTag = "envDefault" - MARKDOWN_FILENAME = "env_gen.md" + envFieldTypeTag = "env" + envDefaultFieldTypeTag = "envDefault" + envDescriptionFieldTypeTag = "envDescription" + MARKDOWN_FILENAME = "env_gen.md" ) const MarkdownTemplate = ` @@ -97,14 +98,15 @@ func convertTagToStructTag(tag string) reflect.StructTag { return reflect.StructTag(strings.Split(tag, "`")[1]) } -func getEnvKeyAndValue(tag reflect.StructTag) (string, string) { +func getEnvKeyAndValue(tag reflect.StructTag) (string, string, string) { envKey := tag.Get(envFieldTypeTag) envValue := tag.Get(envDefaultFieldTypeTag) + envDescription := tag.Get(envDescriptionFieldTypeTag) // check if there exist any value provided in env for this field if value, ok := os.LookupEnv(envKey); ok { envValue = value } - return envKey, envValue + return envKey, envValue, envDescription } func processGoFile(filePath string, allFields *[]EnvField, uniqueKeys *map[string]bool) error { @@ -122,13 +124,14 @@ func processGoFile(filePath string, allFields *[]EnvField, uniqueKeys *map[strin for _, field := range structType.Fields.List { if field.Tag != nil { strippedTags := convertTagToStructTag(field.Tag.Value) - envKey, envValue := getEnvKeyAndValue(strippedTags) + envKey, envValue, envDescription := getEnvKeyAndValue(strippedTags) if len(envKey) == 0 || (*uniqueKeys)[envKey] { continue } *allFields = append(*allFields, EnvField{ - Env: envKey, - EnvValue: envValue, + Env: envKey, + EnvValue: envValue, + EnvDescription: envDescription, }) (*uniqueKeys)[envKey] = true } diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index bdf7d04751..da873dd50b 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -714,7 +714,7 @@ func (impl InstalledAppRepositoryImpl) GetDeploymentSuccessfulStatusCountForTele func (impl InstalledAppRepositoryImpl) GetGitOpsInstalledAppsWhereArgoAppDeletedIsTrue(installedAppId int, envId int) (InstalledApps, error) { var installedApps InstalledApps err := impl.dbConnection.Model(&installedApps). - Column("installed_apps.*", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name"). + Column("installed_apps.*", "App.id", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name"). Where("deployment_app_delete_request = ?", true). Where("installed_apps.active = ?", true). Where("installed_apps.id = ?", installedAppId). diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index 6276405607..72bf9555db 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -877,7 +877,7 @@ func (impl *AppStoreDeploymentServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoA apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo" return apiError } - deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedAppId, envId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedApp.App.Id, envId) if err != nil { impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", installedAppId, "envId", envId, "err", err) apiError.HttpStatusCode = http.StatusInternalServerError diff --git a/pkg/bean/app.go b/pkg/bean/app.go index bbab08513a..625bf8eef9 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -315,6 +315,24 @@ func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, CiPipeline2.Pipeli return switchFromPipelineId, switchFromType } +// PatchSourceInfo returns the CI component ID and component Type, which is being patched +func (ciPatchRequest CiPatchRequest) PatchSourceInfo() (int, string) { + // in app workflow mapping all the build source types are 'CI_PIPELINE' type, except external -> WEBHOOK. + componentType := appWorkflow.CIPIPELINE + var componentId int + // initialize componentId with ciPipeline id + if ciPatchRequest.CiPipeline != nil { + componentId = ciPatchRequest.CiPipeline.Id + } + if ciPatchRequest.SwitchFromExternalCiPipelineId != 0 { + componentType = appWorkflow.WEBHOOK + componentId = ciPatchRequest.SwitchFromExternalCiPipelineId + } else if ciPatchRequest.SwitchFromCiPipelineId != 0 { + componentId = ciPatchRequest.SwitchFromCiPipelineId + } + return componentId, componentType +} + func (ciPatchRequest CiPatchRequest) IsSwitchCiPipelineRequest() bool { return (ciPatchRequest.SwitchFromCiPipelineId != 0 || ciPatchRequest.SwitchFromExternalCiPipelineId != 0) } diff --git a/pkg/configDiff/DeploymentConfigurationService.go b/pkg/configDiff/DeploymentConfigurationService.go index 360de7f8b3..f64de5cd2f 100644 --- a/pkg/configDiff/DeploymentConfigurationService.go +++ b/pkg/configDiff/DeploymentConfigurationService.go @@ -63,6 +63,7 @@ func (impl *DeploymentConfigurationServiceImpl) ConfigAutoComplete(appId int, en if _, ok := cmcsKeyPropertyEnvLevelMap[key]; !ok { if envId > 0 { configProperty.ConfigStage = bean2.Inheriting + configProperty.Id = 0 } } diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index b48d8b5ab4..06e78162f3 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -19,7 +19,9 @@ package git import ( "context" "crypto/tls" + "errors" "fmt" + "github.com/devtron-labs/common-lib/utils/runTime" bean2 "github.com/devtron-labs/devtron/api/bean/gitOps" globalUtil "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/retryFunc" @@ -91,6 +93,15 @@ func (impl GitHubClient) DeleteRepository(config *bean2.GitOpsConfigDto) error { return nil } +func IsRepoNotFound(err error) bool { + if err == nil { + return false + } + var responseErr *github.ErrorResponse + ok := errors.As(err, &responseErr) + return ok && responseErr.Response.StatusCode == 404 +} + func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.GitOpsConfigDto) (url string, isNew bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) { var err error start := time.Now() @@ -100,15 +111,14 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error) repoExists := true - url, err = impl.GetRepoUrl(config) + url, err = impl.getRepoUrl(ctx, config, IsRepoNotFound) if err != nil { - responseErr, ok := err.(*github.ErrorResponse) - if !ok || responseErr.Response.StatusCode != 404 { + if IsRepoNotFound(err) { + repoExists = false + } else { impl.logger.Errorw("error in creating github repo", "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err return "", false, detailedErrorGitOpsConfigActions - } else { - repoExists = false } } if repoExists { @@ -251,12 +261,21 @@ func (impl GitHubClient) CommitValues(ctx context.Context, config *ChartConfig, } func (impl GitHubClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl string, err error) { + ctx := context.Background() + return impl.getRepoUrl(ctx, config, globalUtil.AllPublishableError()) +} + +func (impl GitHubClient) getRepoUrl(ctx context.Context, config *bean2.GitOpsConfigDto, + isNonPublishableError globalUtil.EvalIsNonPublishableErr) (repoUrl string, err error) { start := time.Now() defer func() { + if isNonPublishableError(err) { + impl.logger.Debugw("found non publishable error. skipping metrics publish!", "caller method", runTime.GetCallerFunctionName(), "err", err) + return + } globalUtil.TriggerGitOpsMetrics("GetRepoUrl", "GitHubClient", start, err) }() - ctx := context.Background() repo, _, err := impl.client.Repositories.Get(ctx, impl.org, config.GitRepoName) if err != nil { impl.logger.Errorw("error in getting repo url by repo name", "org", impl.org, "gitRepoName", config.GitRepoName, "err", err) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 791c32da73..72da885a3d 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -475,11 +475,15 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC if !cdWorkflow.BlobStorageEnabled { return nil, nil, errors.New("logs-not-stored-in-repository") } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == executors.WorkflowCancel { - impl.Logger.Debugw("pod is not live ", "err", err) + impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) } - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, err + if err != nil { + impl.Logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, err + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) + } } logReader := bufio.NewReader(logStream) return logReader, cleanUp, err diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index c9be42897e..722de6abb1 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -798,11 +798,15 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC if !ciWorkflow.BlobStorageEnabled { return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == executors.WorkflowCancel { - impl.Logger.Errorw("err", "err", err) + impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt) } - impl.Logger.Errorw("err", "err", err) - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} + if err != nil { + impl.Logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) + } } logReader := bufio.NewReader(logStream) return logReader, cleanUp, err diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index cb0a500bda..7d0d730270 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -68,9 +68,12 @@ func (impl *CiLogServiceImpl) FetchRunningWorkflowLogs(ciLogRequest types.BuildL } req := impl.k8sUtil.GetLogsForAPod(kubeClient, ciLogRequest.Namespace, ciLogRequest.PodName, CiPipeline.Main, true) podLogs, err := req.Stream(context.Background()) - if podLogs == nil || err != nil { - impl.logger.Errorw("error in opening stream", "name", ciLogRequest.PodName) + if err != nil { + impl.logger.Errorw("error in opening stream", "name", ciLogRequest.PodName, "err", err) return nil, nil, err + } else if podLogs == nil { + impl.logger.Warnw("no stream reader found", "name", ciLogRequest.PodName) + return nil, func() error { return nil }, err } cleanUpFunc := func() error { impl.logger.Info("closing running pod log stream") diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index d16dd55199..c960782585 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -743,6 +743,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. PluginArtifactStage: pluginArtifactStage, ImageScanMaxRetries: impl.config.ImageScanMaxRetries, ImageScanRetryDelay: impl.config.ImageScanRetryDelay, + UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, } if pipeline.App.AppType == helper.Job { workflowRequest.AppName = pipeline.App.DisplayName diff --git a/pkg/pipeline/ConfigMapService.go b/pkg/pipeline/ConfigMapService.go index 72cd8026da..906aaf3193 100644 --- a/pkg/pipeline/ConfigMapService.go +++ b/pkg/pipeline/ConfigMapService.go @@ -129,6 +129,17 @@ func NewConfigMapServiceImpl(chartRepository chartRepoRepository.ChartRepository } } +func (impl ConfigMapServiceImpl) checkIfConfigDataAlreadyExist(appId int) (int, error) { + config, err := impl.configMapRepository.GetByAppIdAppLevel(appId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error while checking if config data exist from db by appId", "appId", appId, "error", err) + return 0, err + } else if util.IsErrNoRows(err) { + return 0, nil + } + return config.Id, nil +} + func (impl ConfigMapServiceImpl) CMGlobalAddUpdate(configMapRequest *bean.ConfigDataRequest) (*bean.ConfigDataRequest, error) { if len(configMapRequest.ConfigData) != 1 { return nil, fmt.Errorf("invalid request multiple config found for add or update") @@ -140,6 +151,14 @@ func (impl ConfigMapServiceImpl) CMGlobalAddUpdate(configMapRequest *bean.Config return configMapRequest, err } var model *chartConfig.ConfigMapAppModel + requestId, err := impl.checkIfConfigDataAlreadyExist(configMapRequest.AppId) + if err != nil { + impl.logger.Errorw("error in checking if config map data already exists or not for appId", "appId", configMapRequest.AppId, "error", err) + return configMapRequest, err + } + if requestId > 0 { + configMapRequest.Id = requestId + } if configMapRequest.Id > 0 { model, err = impl.configMapRepository.GetByIdAppLevel(configMapRequest.Id) if err != nil { @@ -525,6 +544,14 @@ func (impl ConfigMapServiceImpl) CSGlobalAddUpdate(configMapRequest *bean.Config impl.logger.Errorw("error in validating", "error", err) return configMapRequest, err } + requestId, err := impl.checkIfConfigDataAlreadyExist(configMapRequest.AppId) + if err != nil { + impl.logger.Errorw("error in checking if config secret data already exists or not for appId", "appId", configMapRequest.AppId, "error", err) + return configMapRequest, err + } + if requestId > 0 { + configMapRequest.Id = requestId + } var model *chartConfig.ConfigMapAppModel if configMapRequest.Id > 0 { model, err = impl.configMapRepository.GetByIdAppLevel(configMapRequest.Id) diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 50e7d27271..df0f6ebaa5 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -141,6 +141,7 @@ type CiCdConfig struct { ExtBlobStorageSecretName string `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` UseArtifactListingQueryV2 bool `env:"USE_ARTIFACT_LISTING_QUERY_V2" envDefault:"true"` UseImageTagFromGitProviderForTagBasedBuild bool `env:"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD" envDefault:"false"` // this is being done for https://github.com/devtron-labs/devtron/issues/4263 + UseDockerApiToGetDigest bool `env:"USE_DOCKER_API_TO_GET_DIGEST" envDefault:"false"` } type CiConfig struct { diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index af15733e1c..cbecb3cd62 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -144,6 +144,7 @@ type WorkflowRequest struct { Scope resourceQualifiers.Scope BuildxCacheModeMin bool `json:"buildxCacheModeMin"` AsyncBuildxCacheExport bool `json:"asyncBuildxCacheExport"` + UseDockerApiToGetDigest bool `json:"useDockerApiToGetDigest"` } func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { diff --git a/pkg/sql/connection.go b/pkg/sql/connection.go index 70ef3b62be..88af6a963e 100644 --- a/pkg/sql/connection.go +++ b/pkg/sql/connection.go @@ -33,7 +33,7 @@ type Config struct { Password string `env:"PG_PASSWORD" envDefault:"" secretData:"-"` Database string `env:"PG_DATABASE" envDefault:"orchestrator"` CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin"` - ApplicationName string `env:"APP" envDefault:"orchestrator"` + ApplicationName string `env:"APP" envDefault:"orchestrator" envDescription:"Application name"` LogQuery bool `env:"PG_LOG_QUERY" envDefault:"true"` LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false"` ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"false"` diff --git a/scripts/devtron-reference-helm-charts/statefulset-chart_5-1-0/templates/servicemonitor.yaml b/scripts/devtron-reference-helm-charts/statefulset-chart_5-1-0/templates/servicemonitor.yaml index 9885733262..7368288e0c 100644 --- a/scripts/devtron-reference-helm-charts/statefulset-chart_5-1-0/templates/servicemonitor.yaml +++ b/scripts/devtron-reference-helm-charts/statefulset-chart_5-1-0/templates/servicemonitor.yaml @@ -38,6 +38,10 @@ spec: {{- if .servicemonitor.scrapeTimeout }} scrapeTimeout: {{ .servicemonitor.scrapeTimeout}} {{- end }} + {{- if .servicemonitor.basicAuth }} + basicAuth: + {{- toYaml .servicemonitor.basicAuth | nindent 8 }} + {{- end }} {{- if .servicemonitor.metricRelabelings}} metricRelabelings: {{toYaml .servicemonitor.metricRelabelings | indent 8 }} @@ -46,7 +50,16 @@ spec: {{- end }} {{- end }} {{- end }} + {{- if .Values.servicemonitor.namespaceSelector }} + namespaceSelector: + matchNames: + {{- toYaml .Values.servicemonitor.namespaceSelector | nindent 6 }} + {{- end }} selector: matchLabels: + {{- if .Values.servicemonitor.matchLabels }} + {{- toYaml .Values.servicemonitor.matchLabels | nindent 6 }} + {{- else }} app: {{ template ".Chart.Name .name" $ }} + {{- end }} {{- end }} diff --git a/scripts/sql/282_bitnami_chart_fix.down.sql b/scripts/sql/282_bitnami_chart_fix.down.sql new file mode 100644 index 0000000000..c1084856e7 --- /dev/null +++ b/scripts/sql/282_bitnami_chart_fix.down.sql @@ -0,0 +1 @@ +update chart_repo set allow_insecure_connection=true where name='bitnami'; diff --git a/scripts/sql/282_bitnami_chart_fix.up.sql b/scripts/sql/282_bitnami_chart_fix.up.sql new file mode 100644 index 0000000000..0729167a19 --- /dev/null +++ b/scripts/sql/282_bitnami_chart_fix.up.sql @@ -0,0 +1 @@ +update chart_repo set allow_insecure_connection=false where name='bitnami'; diff --git a/scripts/sql/282_user_group.down.sql b/scripts/sql/283_user_group.down.sql similarity index 100% rename from scripts/sql/282_user_group.down.sql rename to scripts/sql/283_user_group.down.sql diff --git a/scripts/sql/282_user_group.up.sql b/scripts/sql/283_user_group.up.sql similarity index 100% rename from scripts/sql/282_user_group.up.sql rename to scripts/sql/283_user_group.up.sql diff --git a/util/helper.go b/util/helper.go index 1e979a2d42..bd158ae788 100644 --- a/util/helper.go +++ b/util/helper.go @@ -278,6 +278,14 @@ func TriggerGitOpsMetrics(operation string, method string, startTime time.Time, middleware.GitOpsDuration.WithLabelValues(operation, method, status).Observe(time.Since(startTime).Seconds()) } +type EvalIsNonPublishableErr func(err error) bool + +func AllPublishableError() EvalIsNonPublishableErr { + return func(err error) bool { + return false + } +} + func InterfaceToString(resp interface{}) string { var dat string b, err := json.Marshal(resp)