Add configuration to pipeline stanza of DSC to enable InstructLab pip…

…eline (#1484)

* Add configuration to pipeline stanza of DSC to enable InstructLab pipeline

* Linter

* e2e tests

* Fix findings

README.md

@@ -319,6 +319,9 @@ spec:
       managementState: Managed
     datasciencepipelines:
       managementState: Managed
+      managedPipelines:
+        instructLab:
+          state: Managed
     kserve:
       managementState: Managed
       nim:

apis/components/v1alpha1/datasciencepipelines/types.go

@@ -0,0 +1,23 @@
+// +kubebuilder:object:generate=true
+
+// Package datasciencepipelines provides a set of types used for DataSciencePipelines component
+package datasciencepipelines
+
+import operatorv1 "github.com/openshift/api/operator/v1"
+
+type ManagedPipelinesSpec struct {
+	// Configures whether to automatically import the InstructLab pipeline.
+	// You must enable the trainingoperator component to run the InstructLab pipeline.
+	InstructLab ManagedPipelineOptions `json:"instructLab,omitempty"`
+}
+
+type ManagedPipelineOptions struct {
+	// Set to one of the following values:
+	//
+	// - "Managed" : This pipeline is automatically imported.
+	// - "Removed" : This pipeline is not automatically imported when a new pipeline server or DSPA is created. If previously set to "Managed", setting to "Removed" does not remove existing preloaded pipelines but does prevent future updates from being imported.
+	//
+	// +kubebuilder:validation:Enum=Managed;Removed
+	// +kubebuilder:default=Removed
+	State operatorv1.ManagementState `json:"state,omitempty"`
+}

apis/components/v1alpha1/datasciencepipelines_types.go

@@ -18,6 +18,7 @@ package v1alpha1
 
 import (
 	"github.com/opendatahub-io/opendatahub-operator/v2/apis/common"
+	"github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1alpha1/datasciencepipelines"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -51,6 +52,7 @@ type DataSciencePipelinesSpec struct {
 
 type DataSciencePipelinesCommonSpec struct {
 	common.DevFlagsSpec `json:",inline"`
+	PreloadedPipelines  datasciencepipelines.ManagedPipelinesSpec `json:"managedPipelines,omitempty"`
 }
 
 // DataSciencePipelinesCommonStatus defines the shared observed state of DataSciencePipelines

bundle/manifests/components.platform.opendatahub.io_datasciencepipelines.yaml

@@ -75,6 +75,27 @@ spec:
                       type: object
                     type: array
                 type: object
+              managedPipelines:
+                properties:
+                  instructLab:
+                    description: |-
+                      Configures whether to automatically import the InstructLab pipeline.
+                      You must enable the trainingoperator component to run the InstructLab pipeline.
+                    properties:
+                      state:
+                        default: Removed
+                        description: |-
+                          Set to one of the following values:
+
+                          - "Managed" : This pipeline is automatically imported.
+                          - "Removed" : This pipeline is not automatically imported when a new pipeline server or DSPA is created. If previously set to "Managed", setting to "Removed" does not remove existing preloaded pipelines but does prevent future updates from being imported.
+                        enum:
+                        - Managed
+                        - Removed
+                        pattern: ^(Managed|Unmanaged|Force|Removed)$
+                        type: string
+                    type: object
+                type: object
             type: object
           status:
             description: DataSciencePipelinesStatus defines the observed state of

bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml

@@ -172,6 +172,27 @@ spec:
                               type: object
                             type: array
                         type: object
+                      managedPipelines:
+                        properties:
+                          instructLab:
+                            description: |-
+                              Configures whether to automatically import the InstructLab pipeline.
+                              You must enable the trainingoperator component to run the InstructLab pipeline.
+                            properties:
+                              state:
+                                default: Removed
+                                description: |-
+                                  Set to one of the following values:
+
+                                  - "Managed" : This pipeline is automatically imported.
+                                  - "Removed" : This pipeline is not automatically imported when a new pipeline server or DSPA is created. If previously set to "Managed", setting to "Removed" does not remove existing preloaded pipelines but does prevent future updates from being imported.
+                                enum:
+                                - Managed
+                                - Removed
+                                pattern: ^(Managed|Unmanaged|Force|Removed)$
+                                type: string
+                            type: object
+                        type: object
                       managementState:
                         description: |-
                           Set to one of the following values:

config/crd/bases/components.platform.opendatahub.io_datasciencepipelines.yaml

@@ -75,6 +75,27 @@ spec:
                       type: object
                     type: array
                 type: object
+              managedPipelines:
+                properties:
+                  instructLab:
+                    description: |-
+                      Configures whether to automatically import the InstructLab pipeline.
+                      You must enable the trainingoperator component to run the InstructLab pipeline.
+                    properties:
+                      state:
+                        default: Removed
+                        description: |-
+                          Set to one of the following values:
+
+                          - "Managed" : This pipeline is automatically imported.
+                          - "Removed" : This pipeline is not automatically imported when a new pipeline server or DSPA is created. If previously set to "Managed", setting to "Removed" does not remove existing preloaded pipelines but does prevent future updates from being imported.
+                        enum:
+                        - Managed
+                        - Removed
+                        pattern: ^(Managed|Unmanaged|Force|Removed)$
+                        type: string
+                    type: object
+                type: object
             type: object
           status:
             description: DataSciencePipelinesStatus defines the observed state of

config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml

@@ -172,6 +172,27 @@ spec:
                               type: object
                             type: array
                         type: object
+                      managedPipelines:
+                        properties:
+                          instructLab:
+                            description: |-
+                              Configures whether to automatically import the InstructLab pipeline.
+                              You must enable the trainingoperator component to run the InstructLab pipeline.
+                            properties:
+                              state:
+                                default: Removed
+                                description: |-
+                                  Set to one of the following values:
+
+                                  - "Managed" : This pipeline is automatically imported.
+                                  - "Removed" : This pipeline is not automatically imported when a new pipeline server or DSPA is created. If previously set to "Managed", setting to "Removed" does not remove existing preloaded pipelines but does prevent future updates from being imported.
+                                enum:
+                                - Managed
+                                - Removed
+                                pattern: ^(Managed|Unmanaged|Force|Removed)$
+                                type: string
+                            type: object
+                        type: object
                       managementState:
                         description: |-
                           Set to one of the following values:

controllers/components/datasciencepipelines/datasciencepipelines.go

@@ -15,6 +15,7 @@ import (
 	componentApi "github.com/opendatahub-io/opendatahub-operator/v2/apis/components/v1alpha1"
 	dscv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/controllers/status"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster"
 	cr "github.com/opendatahub-io/opendatahub-operator/v2/pkg/componentsregistry"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/annotations"
@@ -38,15 +39,21 @@ func (s *componentHandler) GetManagementState(dsc *dscv1.DataScienceCluster) ope
 }
 
 func (s *componentHandler) Init(_ common.Platform) error {
-	if err := deploy.ApplyParams(paramsPath().String(), imageParamMap); err != nil {
-		return fmt.Errorf("failed to update images on path %s: %w", paramsPath(), err)
+	release := cluster.GetRelease()
+
+	extraParams := map[string]string{
+		platformVersionParamsKey: release.Version.String(),
+	}
+
+	if err := deploy.ApplyParams(paramsPath, imageParamMap, extraParams); err != nil {
+		return fmt.Errorf("failed to apply params on path %s: %w", paramsPath, err)
 	}
 
 	return nil
 }
 
 func (s *componentHandler) NewCRObject(dsc *dscv1.DataScienceCluster) common.PlatformObject {
-	return &componentApi.DataSciencePipelines{
+	obj := componentApi.DataSciencePipelines{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       componentApi.DataSciencePipelinesKind,
 			APIVersion: componentApi.GroupVersion.String(),
@@ -61,6 +68,14 @@ func (s *componentHandler) NewCRObject(dsc *dscv1.DataScienceCluster) common.Pla
 			DataSciencePipelinesCommonSpec: dsc.Spec.Components.DataSciencePipelines.DataSciencePipelinesCommonSpec,
 		},
 	}
+
+	// since the nested structures are not pointers, we must make sure
+	// any field respect the validation rules.
+	if obj.Spec.PreloadedPipelines.InstructLab.State == "" {
+		obj.Spec.PreloadedPipelines.InstructLab.State = operatorv1.Removed
+	}
+
+	return &obj
 }
 
 func (s *componentHandler) UpdateDSCStatus(dsc *dscv1.DataScienceCluster, obj client.Object) error {

controllers/components/datasciencepipelines/datasciencepipelines_controller_actions.go

@@ -70,7 +70,16 @@ func checkPreConditions(ctx context.Context, rr *odhtypes.ReconciliationRequest)
 }
 
 func initialize(_ context.Context, rr *odhtypes.ReconciliationRequest) error {
-	rr.Manifests = append(rr.Manifests, manifestPath(rr.Release.Name))
+	rr.Manifests = []odhtypes.ManifestInfo{manifestPath(rr.Release.Name)}
+
+	extraParamsMap, err := computeParamsMap(rr)
+	if err != nil {
+		return fmt.Errorf("computing extra params failed: %w", err)
+	}
+
+	if err := odhdeploy.ApplyParams(paramsPath, nil, extraParamsMap); err != nil {
+		return fmt.Errorf("failed to update params.env from %s : %w", paramsPath, err)
+	}
 
 	return nil
 }

controllers/components/datasciencepipelines/datasciencepipelines_support.go

@@ -1,6 +1,10 @@
 package datasciencepipelines
 
 import (
+	"encoding/json"
+	"fmt"
+	"path"
+
 	conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1"
 
 	"github.com/opendatahub-io/opendatahub-operator/v2/apis/common"
@@ -21,6 +25,9 @@ const (
 	// via Kustomize. Since a deployment selector is immutable, we can't upgrade existing
 	// deployment to the new component name, so keep it around till we figure out a solution.
 	LegacyComponentName = "data-science-pipelines-operator"
+
+	managedPipelineParamsKey = "MANAGEDPIPELINES"
+	platformVersionParamsKey = "PLATFORMVERSION"
 )
 
 var (
@@ -42,20 +49,37 @@ var (
 		cluster.OpenDataHub:      "overlays/odh",
 		cluster.Unknown:          "overlays/odh",
 	}
+
+	paramsPath = path.Join(odhdeploy.DefaultManifestPath, ComponentName, "base")
 )
 
-func paramsPath() types.ManifestInfo {
+func manifestPath(p common.Platform) types.ManifestInfo {
 	return types.ManifestInfo{
 		Path:       odhdeploy.DefaultManifestPath,
 		ContextDir: ComponentName,
-		SourcePath: "base",
+		SourcePath: overlaysSourcePaths[p],
 	}
 }
 
-func manifestPath(p common.Platform) types.ManifestInfo {
-	return types.ManifestInfo{
-		Path:       odhdeploy.DefaultManifestPath,
-		ContextDir: ComponentName,
-		SourcePath: overlaysSourcePaths[p],
+func computeParamsMap(rr *types.ReconciliationRequest) (map[string]string, error) {
+	dsp, ok := rr.Instance.(*componentApi.DataSciencePipelines)
+	if !ok {
+		return nil, fmt.Errorf("resource instance %v is not a componentApi.DataSciencePipelines", rr.Instance)
+	}
+
+	data, err := json.Marshal(dsp.Spec.PreloadedPipelines)
+	if err != nil {
+		return nil, fmt.Errorf("marshalling preloaded pipelines failed: %w", err)
+	}
+
+	data, err = json.Marshal(string(data))
+	if err != nil {
+		return nil, fmt.Errorf("marshalling preloaded pipelines failed: %w", err)
+	}
+
+	extraParamsMap := map[string]string{
+		managedPipelineParamsKey: string(data),
 	}
+
+	return extraParamsMap, nil
 }