diff --git a/internal/util/pathutil/pathutil.go b/internal/util/pathutil/pathutil.go index 479b60dd66..8c2eebd053 100644 --- a/internal/util/pathutil/pathutil.go +++ b/internal/util/pathutil/pathutil.go @@ -1,3 +1,17 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pathutil import ( diff --git a/internal/util/render/executor.go b/internal/util/render/executor.go index 1786c949aa..66c11e6e4d 100644 --- a/internal/util/render/executor.go +++ b/internal/util/render/executor.go @@ -47,9 +47,9 @@ type Renderer struct { // PkgPath is the absolute path to the root package PkgPath string - // Evaluator evaluates the function, if specified, this can override the default + // Runner evaluates the function, if specified, this can override the default // kpt function evaluator - Evaluator fn.Evaluator + Runner fn.FunctionRunner // ResultsDirPath is absolute path to the directory to write results ResultsDirPath string @@ -90,7 +90,7 @@ func (e *Renderer) Execute(ctx context.Context) error { imagePullPolicy: e.ImagePullPolicy, allowExec: e.AllowExec, fileSystem: e.FileSystem, - evaluator: e.Evaluator, + evaluator: e.Runner, } if _, err = hydrate(ctx, root, hctx); err != nil { @@ -205,7 +205,7 @@ type hydrationContext struct { fileSystem filesys.FileSystem - evaluator fn.Evaluator + evaluator fn.FunctionRunner } // @@ -486,7 +486,7 @@ func (pn *pkgNode) runValidators(ctx context.Context, hctx *hydrationContext, in hctx.dockerCheckDone = true } if hctx.evaluator != nil { - validator, err = hctx.evaluator.NewRunner(ctx, &function, fn.EvalOptions{ResultList: hctx.fnResults}) + validator, err = hctx.evaluator.NewRunner(ctx, &function, fn.RunnerOptions{ResultList: hctx.fnResults}) } else { validator, err = fnruntime.NewRunner(ctx, &function, pn.pkg.UniquePath, hctx.fnResults, hctx.imagePullPolicy, displayResourceCount) } @@ -604,7 +604,7 @@ func fnChain(ctx context.Context, hctx *hydrationContext, pkgPath types.UniquePa hctx.dockerCheckDone = true } if hctx.evaluator != nil { - runner, err = hctx.evaluator.NewRunner(ctx, &function, fn.EvalOptions{ResultList: hctx.fnResults}) + runner, err = hctx.evaluator.NewRunner(ctx, &function, fn.RunnerOptions{ResultList: hctx.fnResults}) } else { runner, err = fnruntime.NewRunner(ctx, &function, pkgPath, hctx.fnResults, hctx.imagePullPolicy, displayResourceCount) } diff --git a/pkg/fn/eval.go b/pkg/fn/eval.go index ddf6e6bb75..cbcb777425 100644 --- a/pkg/fn/eval.go +++ b/pkg/fn/eval.go @@ -19,16 +19,14 @@ import ( fnresult "github.com/GoogleContainerTools/kpt/pkg/api/fnresult/v1" v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" - "sigs.k8s.io/kustomize/kyaml/filesys" "sigs.k8s.io/kustomize/kyaml/kio" ) -type EvalOptions struct { +type RunnerOptions struct { // ResultList stores the result of the function evaluation ResultList *fnresult.ResultList } -type Evaluator interface { - Eval(ctx context.Context, pkg filesys.FileSystem, fn v1.Function, opts EvalOptions) error - NewRunner(ctx context.Context, fn *v1.Function, opts EvalOptions) (kio.Filter, error) +type FunctionRunner interface { + NewRunner(ctx context.Context, fn *v1.Function, opts RunnerOptions) (kio.Filter, error) } diff --git a/pkg/fn/render.go b/pkg/fn/render.go index 9a4f55902b..b8c8703b86 100644 --- a/pkg/fn/render.go +++ b/pkg/fn/render.go @@ -21,7 +21,7 @@ import ( ) type RenderOptions struct { - Eval Evaluator + Runner FunctionRunner PkgPath string } diff --git a/porch/engine/go.mod b/porch/engine/go.mod index cbc95914a6..b501d455d1 100644 --- a/porch/engine/go.mod +++ b/porch/engine/go.mod @@ -8,6 +8,7 @@ require ( github.com/GoogleContainerTools/kpt/porch/controllers v0.0.0-00010101000000-000000000000 github.com/GoogleContainerTools/kpt/porch/repository v0.0.0-00010101000000-000000000000 k8s.io/klog/v2 v2.40.1 + sigs.k8s.io/kustomize/api v0.8.11 sigs.k8s.io/kustomize/kyaml v0.13.1 ) @@ -93,7 +94,6 @@ require ( k8s.io/kubectl v0.22.2 // indirect k8s.io/utils v0.0.0-20211208161948-7d6a63dca704 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect - sigs.k8s.io/kustomize/api v0.8.11 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/porch/engine/pkg/engine/engine.go b/porch/engine/pkg/engine/engine.go index 91f47b5e99..c0e81b2eb4 100644 --- a/porch/engine/pkg/engine/engine.go +++ b/porch/engine/pkg/engine/engine.go @@ -24,6 +24,7 @@ import ( "reflect" "strings" + fnresultv1 "github.com/GoogleContainerTools/kpt/pkg/api/fnresult/v1" v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" "github.com/GoogleContainerTools/kpt/pkg/fn" api "github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1" @@ -31,6 +32,8 @@ import ( "github.com/GoogleContainerTools/kpt/porch/engine/pkg/kpt" "github.com/GoogleContainerTools/kpt/porch/repository/pkg/cache" "github.com/GoogleContainerTools/kpt/porch/repository/pkg/repository" + "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/kustomize/kyaml/kio" ) type CaDEngine interface { @@ -44,16 +47,16 @@ type CaDEngine interface { func NewCaDEngine(cache *cache.Cache) (CaDEngine, error) { return &cadEngine{ - cache: cache, - renderer: kpt.NewPlaceholderRenderer(), - evaluator: kpt.NewPlaceholderEvaluator(), + cache: cache, + renderer: kpt.NewPlaceholderRenderer(), + runner: kpt.NewPlaceholderFunctionRunner(), }, nil } type cadEngine struct { - cache *cache.Cache - renderer fn.Renderer - evaluator fn.Evaluator + cache *cache.Cache + renderer fn.Renderer + runner fn.FunctionRunner } var _ CaDEngine = &cadEngine{} @@ -101,8 +104,8 @@ func (cad *cadEngine) CreatePackageRevision(ctx context.Context, repositoryObj * return nil, fmt.Errorf("eval not set for task of type %q", task.Type) } mutations = append(mutations, &evalFunctionMutation{ - evaluator: cad.evaluator, - task: task, + runner: cad.runner, + task: task, }) default: @@ -112,8 +115,8 @@ func (cad *cadEngine) CreatePackageRevision(ctx context.Context, repositoryObj * // Render package after creation. mutations = append(mutations, &renderPackageMutation{ - renderer: cad.renderer, - evaluator: cad.evaluator, + renderer: cad.renderer, + runner: cad.runner, }) baseResources := repository.PackageResources{} @@ -164,8 +167,8 @@ func (cad *cadEngine) UpdatePackageRevision(ctx context.Context, repositoryObj * } mutations = append(mutations, &renderPackageMutation{ - renderer: cad.renderer, - evaluator: cad.evaluator, + renderer: cad.renderer, + runner: cad.runner, }) draft, err := repo.UpdatePackage(ctx, oldPackage) @@ -340,8 +343,8 @@ func loadResourcesFromDirectory(dir string) (repository.PackageResources, error) } type evalFunctionMutation struct { - evaluator fn.Evaluator - task *api.Task + runner fn.FunctionRunner + task *api.Task } func (m *evalFunctionMutation) Apply(ctx context.Context, resources repository.PackageResources) (repository.PackageResources, *api.Task, error) { @@ -353,12 +356,35 @@ func (m *evalFunctionMutation) Apply(ctx context.Context, resources repository.P return repository.PackageResources{}, nil, err } - if err := m.evaluator.Eval(ctx, fs, v1.Function{ + results := fnresultv1.ResultList{} + filter, err := m.runner.NewRunner(ctx, &v1.Function{ Image: e.Image, ConfigMap: e.ConfigMap, Selectors: []v1.Selector{}, - }, fn.EvalOptions{}); err != nil { - return repository.PackageResources{}, nil, err + }, fn.RunnerOptions{ + ResultList: &results, + }) + if err != nil { + return repository.PackageResources{}, nil, fmt.Errorf("failed to create function runner: %w", err) + } + + rw := &kio.LocalPackageReadWriter{ + PackagePath: "/", // TODO: Populate with the package directory. + IncludeSubpackages: true, + FileSystem: filesys.FileSystemOrOnDisk{ + FileSystem: fs, + }, + } + + pipeline := kio.Pipeline{ + Inputs: []kio.Reader{rw}, + Filters: []kio.Filter{filter}, + Outputs: []kio.Writer{rw}, + ContinueOnEmptyResult: false, + } + + if err := pipeline.Execute(); err != nil { + return repository.PackageResources{}, nil, fmt.Errorf("failed to evaluate function: %w", err) } result, err := readResources(fs) diff --git a/porch/engine/pkg/engine/render.go b/porch/engine/pkg/engine/render.go index 7f42f090b2..1e9301d609 100644 --- a/porch/engine/pkg/engine/render.go +++ b/porch/engine/pkg/engine/render.go @@ -28,8 +28,8 @@ import ( ) type renderPackageMutation struct { - renderer fn.Renderer - evaluator fn.Evaluator + renderer fn.Renderer + runner fn.FunctionRunner } var _ mutation = &renderPackageMutation{} @@ -42,7 +42,7 @@ func (m *renderPackageMutation) Apply(ctx context.Context, resources repository. } if err := m.renderer.Render(ctx, fs, fn.RenderOptions{ - Eval: m.evaluator, + Runner: m.runner, }); err != nil { return repository.PackageResources{}, nil, err } diff --git a/porch/engine/pkg/engine/render_test.go b/porch/engine/pkg/engine/render_test.go index b74a73f32f..a1dafab2d2 100644 --- a/porch/engine/pkg/engine/render_test.go +++ b/porch/engine/pkg/engine/render_test.go @@ -25,8 +25,8 @@ import ( func TestRender(t *testing.T) { render := &renderPackageMutation{ - renderer: kpt.NewPlaceholderRenderer(), - evaluator: kpt.NewPlaceholderEvaluator(), + renderer: kpt.NewPlaceholderRenderer(), + runner: kpt.NewPlaceholderFunctionRunner(), } const path = "bucket.yaml" diff --git a/porch/engine/pkg/internal/functions.go b/porch/engine/pkg/internal/functions.go index b0f5fdaf40..c671e04066 100644 --- a/porch/engine/pkg/internal/functions.go +++ b/porch/engine/pkg/internal/functions.go @@ -15,65 +15,16 @@ package internal import ( - "context" "fmt" - v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" - "github.com/GoogleContainerTools/kpt/pkg/fn" - "sigs.k8s.io/kustomize/kyaml/filesys" "sigs.k8s.io/kustomize/kyaml/fn/framework" - "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/yaml" ) var functions map[string]framework.ResourceListProcessorFunc = map[string]framework.ResourceListProcessorFunc{ "gcr.io/kpt-fn/set-labels:v0.1.5": setLabels, } -func Eval(ctx context.Context, pkg filesys.FileSystem, fn v1.Function, opts fn.EvalOptions) error { - rw := &kio.LocalPackageReadWriter{ - IncludeSubpackages: true, - PackagePath: "/", - FileSystem: filesys.FileSystemOrOnDisk{ - FileSystem: pkg, - }, - } - - rl := framework.ResourceList{} - - if fn.ConfigMap != nil { - if cm, err := NewConfigMap(fn.ConfigMap); err != nil { - return err - } else { - rl.FunctionConfig = cm - } - } - - // Read input - if items, err := rw.Read(); err != nil { - return fmt.Errorf("failed to read fn eval input: %w", err) - } else { - rl.Items = items - } - - if err := eval(fn.Image, &rl); err != nil { - return fmt.Errorf("function evaluation failed; %w", err) - } - - // Return error on error - if rl.Results.ExitCode() != 0 { - return rl.Results - } - - // Write Output - if err := rw.Write(rl.Items); err != nil { - return fmt.Errorf("failed to write fn eval output: %w", err) - } - - return nil -} - -func eval(image string, rl *framework.ResourceList) error { +func Eval(image string, rl *framework.ResourceList) error { // Evaluate if f, ok := functions[image]; ok { return f(rl) @@ -81,22 +32,3 @@ func eval(image string, rl *framework.ResourceList) error { return fmt.Errorf("unsupported kpt function %q", image) } } - -func NewConfigMap(data map[string]string) (*yaml.RNode, error) { - node := yaml.NewMapRNode(&data) - if node == nil { - return nil, nil - } - // create a ConfigMap only for configMap config - configMap := yaml.MustParse(` -apiVersion: v1 -kind: ConfigMap -metadata: - name: function-input -data: {} -`) - if err := configMap.PipeE(yaml.SetField("data", node)); err != nil { - return nil, err - } - return configMap, nil -} diff --git a/porch/engine/pkg/kpt/eval.go b/porch/engine/pkg/kpt/eval.go index 84371b096b..dbc90bc9a1 100644 --- a/porch/engine/pkg/kpt/eval.go +++ b/porch/engine/pkg/kpt/eval.go @@ -16,27 +16,83 @@ package kpt import ( "context" + "fmt" - v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" + fnresultv1 "github.com/GoogleContainerTools/kpt/pkg/api/fnresult/v1" + kptfilev1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" "github.com/GoogleContainerTools/kpt/pkg/fn" "github.com/GoogleContainerTools/kpt/porch/engine/pkg/internal" - "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/fn/framework" "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" ) -func NewPlaceholderEvaluator() fn.Evaluator { +func NewPlaceholderFunctionRunner() fn.FunctionRunner { return &evaluator{} } type evaluator struct { } -var _ fn.Evaluator = &evaluator{} +var _ fn.FunctionRunner = &evaluator{} -func (e *evaluator) Eval(ctx context.Context, pkg filesys.FileSystem, fn v1.Function, opts fn.EvalOptions) error { - return internal.Eval(ctx, pkg, fn, opts) +func (e *evaluator) NewRunner(ctx context.Context, fn *kptfilev1.Function, opts fn.RunnerOptions) (kio.Filter, error) { + return &runner{ + ctx: ctx, + fn: *fn, + rl: opts.ResultList, + }, nil } -func (e *evaluator) NewRunner(ctx context.Context, fn *v1.Function, opts fn.EvalOptions) (kio.Filter, error) { - return nil, nil +type runner struct { + ctx context.Context + fn kptfilev1.Function + rl *fnresultv1.ResultList +} + +var _ kio.Filter = &runner{} + +func (r *runner) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) { + rl := &framework.ResourceList{ + Items: items, + Results: []*framework.Result{}, + } + + if r.fn.ConfigMap != nil { + if cm, err := NewConfigMap(r.fn.ConfigMap); err != nil { + return nil, fmt.Errorf("cannot create config map: %w", err) + } else { + rl.FunctionConfig = cm + } + } + + if err := internal.Eval(r.fn.Image, rl); err != nil { + return nil, fmt.Errorf("function evaluation failed; %w", err) + } + + // Return error on error + if rl.Results.ExitCode() != 0 { + return nil, rl.Results + } + + return rl.Items, nil +} + +func NewConfigMap(data map[string]string) (*yaml.RNode, error) { + node := yaml.NewMapRNode(&data) + if node == nil { + return nil, nil + } + // create a ConfigMap only for configMap config + configMap := yaml.MustParse(` +apiVersion: v1 +kind: ConfigMap +metadata: + name: function-input +data: {} +`) + if err := configMap.PipeE(yaml.SetField("data", node)); err != nil { + return nil, err + } + return configMap, nil } diff --git a/porch/engine/pkg/kpt/eval_test.go b/porch/engine/pkg/kpt/eval_test.go index 1d431a220a..fe247dec27 100644 --- a/porch/engine/pkg/kpt/eval_test.go +++ b/porch/engine/pkg/kpt/eval_test.go @@ -16,11 +16,11 @@ package kpt import ( "context" + "strings" "testing" v1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1" "github.com/GoogleContainerTools/kpt/pkg/fn" - "sigs.k8s.io/kustomize/kyaml/filesys" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -28,7 +28,6 @@ import ( func TestSetLabels(t *testing.T) { k := &evaluator{} - const path = "bucket.yaml" const pkgYaml = `# Comment apiVersion: storage.cnrm.cloud.google.com/v1beta1 kind: StorageBucket @@ -39,32 +38,32 @@ spec: storageClass: standard ` - fs := &MemFS{} - fs.Mkdir("/") // TODO: Make this automatic. - fs.WriteFile(path, []byte(pkgYaml)) - - if err := k.Eval(context.Background(), fs, v1.Function{ + filter, err := k.NewRunner(context.Background(), &v1.Function{ Image: "gcr.io/kpt-fn/set-labels:v0.1.5", ConfigMap: map[string]string{ "label-key": "label-value", }, - }, fn.EvalOptions{}); err != nil { + }, fn.RunnerOptions{}) + + if err != nil { t.Errorf("Eval failed: %v", err) } - r := kio.LocalPackageReader{ - PackagePath: "/", - FileSystem: filesys.FileSystemOrOnDisk{ - FileSystem: fs, - }, + var result []*yaml.RNode + var writer kio.WriterFunc = func(r []*yaml.RNode) error { + result = r + return nil } - var result []*yaml.RNode + p := &kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(pkgYaml)}}, + Filters: []kio.Filter{filter}, + Outputs: []kio.Writer{writer}, + ContinueOnEmptyResult: false, + } - if nodes, err := r.Read(); err != nil { - t.Errorf("Result read failed: %v", err) - } else { - result = nodes + if err := p.Execute(); err != nil { + t.Errorf("Failed to evaluate function: %v", err) } if got, want := len(result), 1; got != want {