Skip to content

Commit

Permalink
feat: Add dependency filter
Browse files Browse the repository at this point in the history
- Pass TaskContext into TaskBuilder.Build
- Combine dependency graph for apply and prune objects.
  This is required to catch dependencies that would have been deleted.
- Replace graph.SortObjs into DependencyGraph + Sort + HydrateSetList
- Replace graph.ReverseSortObjs with ReverseSetList to perform on the
  combined (apply + prune) set list.
- Add planned pending applies and prune to the InventoryManager
  before executing the task queue.
  This allows the DependencyFilter to validate against the planned
  actuation strategy of objects that haven't been applied/pruned yet.
- Add the dependency graph to the TaskContext, for the
  DependencyFilter to use.
  This can be removed in the future if the filters are managed by the
  solver.
- Make Graph.Sort non-destructive, so the graph can be re-used by the
  DependencyFilter.
- Add Graph.EdgesFrom and EdgesTo for the DependencyFilter to use.
  This requires storing the reverse edge list.
- Add an e2e test for the DependencyFilter
- Add an e2e test for the LocalNamespaceFilter

Fixes kubernetes-sigs#526
Fixes kubernetes-sigs#528
Fixes kubernetes-sigs#554
  • Loading branch information
karlkfi committed Feb 25, 2022
1 parent e06456e commit 308cf9b
Show file tree
Hide file tree
Showing 15 changed files with 1,220 additions and 93 deletions.
19 changes: 14 additions & 5 deletions pkg/apply/applier.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
"k8s.io/klog/v2"
"sigs.k8s.io/cli-utils/pkg/apis/actuation"
"sigs.k8s.io/cli-utils/pkg/apply/cache"
"sigs.k8s.io/cli-utils/pkg/apply/event"
"sigs.k8s.io/cli-utils/pkg/apply/filter"
Expand Down Expand Up @@ -125,6 +126,10 @@ func (a *Applier) Run(ctx context.Context, invInfo inventory.Info, objects objec
}
klog.V(4).Infof("calculated %d apply objs; %d prune objs", len(applyObjs), len(pruneObjs))

// Build a TaskContext for passing info between tasks
resourceCache := cache.NewResourceCacheMap()
taskContext := taskrunner.NewTaskContext(eventChannel, resourceCache)

// Fetch the queue (channel) of tasks that should be executed.
klog.V(4).Infoln("applier building task queue...")
// Build list of apply validation filters.
Expand All @@ -135,6 +140,10 @@ func (a *Applier) Run(ctx context.Context, invInfo inventory.Info, objects objec
Inv: invInfo,
InvPolicy: options.InventoryPolicy,
},
filter.DependencyFilter{
TaskContext: taskContext,
Strategy: actuation.ActuationStrategyApply,
},
}
// Build list of prune validation filters.
pruneFilters := []filter.ValidationFilter{
Expand All @@ -146,9 +155,12 @@ func (a *Applier) Run(ctx context.Context, invInfo inventory.Info, objects objec
filter.LocalNamespacesFilter{
LocalNamespaces: localNamespaces(invInfo, object.UnstructuredSetToObjMetadataSet(objects)),
},
filter.DependencyFilter{
TaskContext: taskContext,
Strategy: actuation.ActuationStrategyDelete,
},
}
// Build list of apply mutators.
resourceCache := cache.NewResourceCacheMap()
applyMutators := []mutator.Interface{
&mutator.ApplyTimeMutator{
Client: a.client,
Expand Down Expand Up @@ -184,7 +196,7 @@ func (a *Applier) Run(ctx context.Context, invInfo inventory.Info, objects objec
WithApplyObjects(applyObjs).
WithPruneObjects(pruneObjs).
WithInventory(invInfo).
Build(opts)
Build(taskContext, opts)

klog.V(4).Infof("validation errors: %d", len(vCollector.Errors))
klog.V(4).Infof("invalid objects: %d", len(vCollector.InvalidIds))
Expand All @@ -206,9 +218,6 @@ func (a *Applier) Run(ctx context.Context, invInfo inventory.Info, objects objec
return
}

// Build a TaskContext for passing info between tasks
taskContext := taskrunner.NewTaskContext(eventChannel, resourceCache)

// Register invalid objects to be retained in the inventory, if present.
for _, id := range vCollector.InvalidIds {
taskContext.AddInvalidObject(id)
Expand Down
15 changes: 10 additions & 5 deletions pkg/apply/destroyer.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"
cmdutil "k8s.io/kubectl/pkg/cmd/util"
"sigs.k8s.io/cli-utils/pkg/apis/actuation"
"sigs.k8s.io/cli-utils/pkg/apply/cache"
"sigs.k8s.io/cli-utils/pkg/apply/event"
"sigs.k8s.io/cli-utils/pkg/apply/filter"
Expand Down Expand Up @@ -129,6 +130,10 @@ func (d *Destroyer) Run(ctx context.Context, invInfo inventory.Info, options Des
}
validator.Validate(deleteObjs)

// Build a TaskContext for passing info between tasks
resourceCache := cache.NewResourceCacheMap()
taskContext := taskrunner.NewTaskContext(eventChannel, resourceCache)

klog.V(4).Infoln("destroyer building task queue...")
dynamicClient, err := d.factory.DynamicClient()
if err != nil {
Expand All @@ -141,6 +146,10 @@ func (d *Destroyer) Run(ctx context.Context, invInfo inventory.Info, options Des
Inv: invInfo,
InvPolicy: options.InventoryPolicy,
},
filter.DependencyFilter{
TaskContext: taskContext,
Strategy: actuation.ActuationStrategyDelete,
},
}
taskBuilder := &solver.TaskQueueBuilder{
Pruner: d.pruner,
Expand All @@ -165,7 +174,7 @@ func (d *Destroyer) Run(ctx context.Context, invInfo inventory.Info, options Des
taskQueue := taskBuilder.
WithPruneObjects(deleteObjs).
WithInventory(invInfo).
Build(opts)
Build(taskContext, opts)

klog.V(4).Infof("validation errors: %d", len(vCollector.Errors))
klog.V(4).Infof("invalid objects: %d", len(vCollector.InvalidIds))
Expand All @@ -187,10 +196,6 @@ func (d *Destroyer) Run(ctx context.Context, invInfo inventory.Info, options Des
return
}

// Build a TaskContext for passing info between tasks
resourceCache := cache.NewResourceCacheMap()
taskContext := taskrunner.NewTaskContext(eventChannel, resourceCache)

// Register invalid objects to be retained in the inventory, if present.
for _, id := range vCollector.InvalidIds {
taskContext.AddInvalidObject(id)
Expand Down
113 changes: 113 additions & 0 deletions pkg/apply/filter/dependency-filter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package filter

import (
"fmt"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/cli-utils/pkg/apis/actuation"
"sigs.k8s.io/cli-utils/pkg/apply/taskrunner"
"sigs.k8s.io/cli-utils/pkg/object"
)

// DependencyFilter implements ValidationFilter interface to determine if an
// object can be applied or deleted based on the status of it's dependencies.
type DependencyFilter struct {
TaskContext *taskrunner.TaskContext
Strategy actuation.ActuationStrategy
}

const DependencyFilterName = "DependencyFilter"

// Name returns the name of the filter for logs and events.
func (dnrf DependencyFilter) Name() string {
return DependencyFilterName
}

// Filter returns true if the specified object should be skipped because at
// least one of its dependencies is Not Found or Not Reconciled.
func (dnrf DependencyFilter) Filter(obj *unstructured.Unstructured) (bool, string, error) {
id := object.UnstructuredToObjMetadata(obj)

switch dnrf.Strategy {
case actuation.ActuationStrategyApply:
// For apply, check dependencies (outgoing)
relationship := "dependency"
for _, depID := range dnrf.TaskContext.Graph().EdgesFrom(id) {
filter, reason, err := dnrf.filterByRelationStatus(depID, relationship)
if err != nil {
return false, "", err
}
if filter {
return filter, reason, nil
}
}
case actuation.ActuationStrategyDelete:
// For delete, check dependents (incoming)
relationship := "dependent"
for _, depID := range dnrf.TaskContext.Graph().EdgesTo(id) {
filter, reason, err := dnrf.filterByRelationStatus(depID, relationship)
if err != nil {
return false, "", err
}
if filter {
return filter, reason, nil
}
}
default:
panic(fmt.Sprintf("invalid filter strategy: %q", dnrf.Strategy))
}
return false, "", nil
}

func (dnrf DependencyFilter) filterByRelationStatus(id object.ObjMetadata, relationship string) (bool, string, error) {
// Skip invalid objects.
// This shouldn't happen, because invalid objects are excluded by the solver.
if dnrf.TaskContext.IsInvalidObject(id) {
return true, fmt.Sprintf("%s invalid: %q", relationship, id), nil
}

status, found := dnrf.TaskContext.InventoryManager().ObjectStatus(id)
if !found {
// Status is registered during planning.
// So if status is not found, the object is external (NYI) or invalid.
return false, "", fmt.Errorf("unknown %s actuation strategy: %v", relationship, id)
}

// Dependencies must have the same actuation strategy.
// If there is a mismatch, skip both.
if status.Strategy != dnrf.Strategy {
return true, fmt.Sprintf("%s skipped because %s is scheduled for %s: %q", dnrf.Strategy, relationship, status.Strategy, id), nil
}

switch status.Actuation {
case actuation.ActuationPending:
// If actuation is still pending, dependency sorting is probably broken.
return false, "", fmt.Errorf("premature actuation: %s %s %s: %q", relationship, dnrf.Strategy, status.Actuation, id)
case actuation.ActuationSkipped, actuation.ActuationFailed:
// Skip!
return true, fmt.Sprintf("%s %s %s: %q", relationship, dnrf.Strategy, status.Actuation, id), nil
case actuation.ActuationSucceeded:
// Don't skip!
default:
return false, "", fmt.Errorf("invalid %s apply status %q: %q", relationship, status.Actuation, id)
}

switch status.Reconcile {
case actuation.ReconcilePending:
// If reconcile is still pending, dependency sorting is probably broken.
return false, "", fmt.Errorf("premature reconciliation: %s reconcile %s: %q", relationship, status.Reconcile, id)
case actuation.ReconcileSkipped, actuation.ReconcileFailed, actuation.ReconcileTimeout:
// Skip!
return true, fmt.Sprintf("%s reconcile %s: %q", relationship, status.Reconcile, id), nil
case actuation.ReconcileSucceeded:
// Don't skip!
default:
return false, "", fmt.Errorf("invalid dependency reconcile status %q: %q", status.Reconcile, id)
}

// Don't skip!
return false, "", nil
}
63 changes: 42 additions & 21 deletions pkg/apply/solver/solver.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,48 @@ func (t *TaskQueueBuilder) WithPruneObjects(pruneObjs object.UnstructuredSet) *T
}

// Build returns the queue of tasks that have been created
func (t *TaskQueueBuilder) Build(o Options) *TaskQueue {
func (t *TaskQueueBuilder) Build(taskContext *taskrunner.TaskContext, o Options) *TaskQueue {
var tasks []taskrunner.Task

// reset counters
t.applyCounter = 0
t.pruneCounter = 0
t.waitCounter = 0

// Filter objects that failed earlier validation
applyObjs := t.Collector.FilterInvalidObjects(t.applyObjs)
pruneObjs := t.Collector.FilterInvalidObjects(t.pruneObjs)

// Merge applyObjs & pruneObjs and graph them together.
// This detects implicit and explicit dependencies.
// Invalid dependency annotations will be treated as validation errors.
allObjs := make(object.UnstructuredSet, 0, len(applyObjs)+len(pruneObjs))
allObjs = append(allObjs, applyObjs...)
allObjs = append(allObjs, pruneObjs...)
g, err := graph.DependencyGraph(allObjs)
if err != nil {
t.Collector.Collect(err)
}
// Store graph for use by DependencyFilter
taskContext.SetGraph(g)
// Sort objects into phases (apply order).
// Cycles will be treated as validation errors.
idSetList, err := g.Sort()
if err != nil {
t.Collector.Collect(err)
}

// Filter objects with cycles or invalid dependency annotations
applyObjs = t.Collector.FilterInvalidObjects(applyObjs)
pruneObjs = t.Collector.FilterInvalidObjects(pruneObjs)

if len(applyObjs) > 0 {
klog.V(2).Infoln("adding inventory add task (%d objects)", len(applyObjs))
// Register actuation plan in the inventory
for _, id := range object.UnstructuredSetToObjMetadataSet(applyObjs) {
taskContext.InventoryManager().AddPendingApply(id)
}

klog.V(2).Infof("adding inventory add task (%d objects)", len(applyObjs))
tasks = append(tasks, &task.InvAddTask{
TaskName: "inventory-add-0",
InvClient: t.InvClient,
Expand All @@ -140,18 +169,10 @@ func (t *TaskQueueBuilder) Build(o Options) *TaskQueue {
DryRun: o.DryRunStrategy,
})

// Create a dependency graph, sort, and flatten into phases.
applySets, err := graph.SortObjs(applyObjs)
if err != nil {
t.Collector.Collect(err)
}
// Filter idSetList down to just apply objects
applySets := graph.HydrateSetList(idSetList, applyObjs)

for _, applySet := range applySets {
// filter again, because sorting may have added more invalid objects.
applySet = t.Collector.FilterInvalidObjects(applySet)
if len(applySet) == 0 {
continue
}
tasks = append(tasks,
t.newApplyTask(applySet, t.ApplyFilters, t.ApplyMutators, o))
// dry-run skips wait tasks
Expand All @@ -164,18 +185,18 @@ func (t *TaskQueueBuilder) Build(o Options) *TaskQueue {
}

if o.Prune && len(pruneObjs) > 0 {
// Create a dependency graph, sort (in reverse), and flatten into phases.
pruneSets, err := graph.ReverseSortObjs(pruneObjs)
if err != nil {
t.Collector.Collect(err)
// Register actuation plan in the inventory
for _, id := range object.UnstructuredSetToObjMetadataSet(pruneObjs) {
taskContext.InventoryManager().AddPendingDelete(id)
}

// Filter idSetList down to just prune objects
pruneSets := graph.HydrateSetList(idSetList, pruneObjs)

// Reverse apply order to get prune order
graph.ReverseSetList(pruneSets)

for _, pruneSet := range pruneSets {
// filter again, because sorting may have added more invalid objects.
pruneSet = t.Collector.FilterInvalidObjects(pruneSet)
if len(pruneSet) == 0 {
continue
}
tasks = append(tasks,
t.newPruneTask(pruneSet, t.PruneFilters, o))
// dry-run skips wait tasks
Expand Down
6 changes: 4 additions & 2 deletions pkg/apply/solver/solver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -629,9 +629,10 @@ func TestTaskQueueBuilder_ApplyBuild(t *testing.T) {
InvClient: fakeInvClient,
Collector: vCollector,
}
taskContext := taskrunner.NewTaskContext(nil, nil)
tq := tqb.WithInventory(invInfo).
WithApplyObjects(tc.applyObjs).
Build(tc.options)
Build(taskContext, tc.options)
err := vCollector.ToError()
if tc.expectedError != nil {
assert.EqualError(t, err, tc.expectedError.Error())
Expand Down Expand Up @@ -1084,9 +1085,10 @@ func TestTaskQueueBuilder_PruneBuild(t *testing.T) {
InvClient: fakeInvClient,
Collector: vCollector,
}
taskContext := taskrunner.NewTaskContext(nil, nil)
tq := tqb.WithInventory(invInfo).
WithPruneObjects(tc.pruneObjs).
Build(tc.options)
Build(taskContext, tc.options)
err := vCollector.ToError()
if tc.expectedError != nil {
assert.EqualError(t, err, tc.expectedError.Error())
Expand Down
3 changes: 2 additions & 1 deletion pkg/apply/task/inv_set_task_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,10 @@ func TestInvSetTask(t *testing.T) {
},
"one apply objs, one prune failures; one inventory": {
// aritifical use case: applies and prunes are mutually exclusive
// failed delete replaces successful apply in the task context
appliedObjs: object.ObjMetadataSet{id3},
failedDeletes: object.ObjMetadataSet{id3},
expectedObjs: object.ObjMetadataSet{id3},
expectedObjs: object.ObjMetadataSet{},
},
"two apply objs, two prune failures; three inventory": {
// aritifical use case: applies and prunes are mutually exclusive
Expand Down
Loading

0 comments on commit 308cf9b

Please sign in to comment.