Main

feat: enable migrating from Kptfile & CM to resourcegroup inventories (kptdev#2705)

This commit enables `kpt live migrate` to migrate inventory information
from either a ConfigMap or Kptfile to a separate ResourceGroup file.

This functionality is currently behind a feature gate and is not exposed
to the user via any CLI flags. Enabling of this feature to users will be
done later.

make kpt binary optional in test harness (kptdev#2758)

feat: enable STDIN apply and destroy using RG inventory (kptdev#2709)

This commit enables actuation from STDIN using inventory information
that is stored in a ResourceGroup file.

This feature is currently behind a feature gate and will be exposed to
users as a CLI flag in a future commit/PR.

updated the version to 1.0.0-beta.13 (kptdev#2806)

Merge main into porch

Correct Apache License Text (kptdev#2675)

LICENSE file is supposed to be an exact copy of
https://www.apache.org/licenses/LICENSE-2.0.txt.

Formula/kpt.rb

@@ -15,8 +15,8 @@
 class Kpt < Formula
   desc "Toolkit to manage,and apply Kubernetes Resource config data files"
   homepage "https://googlecontainertools.github.io/kpt"
-  url "https://github.com/GoogleContainerTools/kpt/archive/v1.0.0-beta.7.tar.gz"
-  sha256 "e31e7ee63006150f730ed8a08e9063a2fb2faf9e617353e3649ef893bdf7c156"
+  url "https://github.com/GoogleContainerTools/kpt/archive/v1.0.0-beta.13.tar.gz"
+  sha256 "d98a95f1de38fc8c7dee861ec249874ea3ddf03f5a28ae7fc4a0364d3578667b"
 
   depends_on "go" => :build
 

LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2019 Google LLC
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,4 +199,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

internal/cmdapply/cmdapply.go

@@ -108,6 +108,7 @@ type Runner struct {
 	pruneTimeout                 time.Duration
 	inventoryPolicyString        string
 	dryRun                       bool
+	rgFile                       string
 	printStatusEvents            bool
 
 	inventoryPolicy inventory.InventoryPolicy
@@ -166,7 +167,7 @@ func (r *Runner) runE(c *cobra.Command, args []string) error {
 		}
 	}
 
-	objs, inv, err := live.Load(r.factory, path, c.InOrStdin())
+	objs, inv, err := live.Load(r.factory, path, r.rgFile, c.InOrStdin())
 	if err != nil {
 		return err
 	}

internal/cmddestroy/cmddestroy.go

@@ -83,6 +83,7 @@ type Runner struct {
 	output                string
 	inventoryPolicyString string
 	dryRun                bool
+	rgFile                string
 	printStatusEvents     bool
 
 	inventoryPolicy inventory.InventoryPolicy
@@ -128,7 +129,7 @@ func (r *Runner) runE(c *cobra.Command, args []string) error {
 		}
 	}
 
-	_, inv, err := live.Load(r.factory, path, c.InOrStdin())
+	_, inv, err := live.Load(r.factory, path, r.rgFile, c.InOrStdin())
 	if err != nil {
 		return err
 	}

internal/cmdmigrate/migratecmd.go

@@ -6,17 +6,21 @@ package cmdmigrate
 import (
 	"bytes"
 	"context"
-	"errors"
+	goerrors "errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	"github.com/GoogleContainerTools/kpt/internal/cmdliveinit"
 	"github.com/GoogleContainerTools/kpt/internal/docs/generated/livedocs"
+	"github.com/GoogleContainerTools/kpt/internal/errors"
 	"github.com/GoogleContainerTools/kpt/internal/pkg"
+	"github.com/GoogleContainerTools/kpt/internal/types"
 	"github.com/GoogleContainerTools/kpt/internal/util/argutil"
 	"github.com/GoogleContainerTools/kpt/internal/util/pathutil"
+	"github.com/GoogleContainerTools/kpt/pkg/kptfile/kptfileutil"
 	"github.com/GoogleContainerTools/kpt/pkg/live"
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -41,10 +45,12 @@ type MigrateRunner struct {
 	dir             string
 	dryRun          bool
 	name            string
+	rgFile          string
 	force           bool
 	rgInvClientFunc func(util.Factory) (inventory.InventoryClient, error)
 	cmInvClientFunc func(util.Factory) (inventory.InventoryClient, error)
 	cmLoader        manifestreader.ManifestLoader
+	cmNotMigrated   bool // flag to determine if migration from ConfigMap has occurred
 }
 
 // NewRunner returns a pointer to an initial MigrateRunner structure.
@@ -98,6 +104,15 @@ func NewCommand(ctx context.Context, f util.Factory, cmLoader manifestreader.Man
 // Run executes the migration from the ConfigMap based inventory to the ResourceGroup
 // based inventory.
 func (mr *MigrateRunner) Run(reader io.Reader, args []string) error {
+	// Use ResourceGroup file for inventory logic if the resourcegroup file
+	// is set directly. For this feature gate, the resourcegroup must be directly set
+	// through our tests since we are not exposing this through the command surface as a
+	// flag, currently. When we promote this, the resourcegroup filename can be empty and
+	// the default filename value will be inferred/used.
+	if mr.rgFile != "" {
+		return mr.runLiveMigrateWithRGFile(reader, args)
+	}
+
 	// Validate the number of arguments.
 	if len(args) > 1 {
 		return fmt.Errorf("too many arguments; migrate requires one directory argument (or stdin)")
@@ -292,7 +307,7 @@ func (mr *MigrateRunner) migrateObjs(rgInvClient inventory.InventoryClient,
 		}
 	}
 
-	_, inv, err := live.Load(mr.factory, path, reader)
+	_, inv, err := live.Load(mr.factory, path, mr.rgFile, reader)
 	if err != nil {
 		return err
 	}
@@ -390,3 +405,176 @@ func rgInvClient(factory util.Factory) (inventory.InventoryClient, error) {
 func cmInvClient(factory util.Factory) (inventory.InventoryClient, error) {
 	return inventory.NewInventoryClient(factory, inventory.WrapInventoryObj, inventory.InvInfoToConfigMap)
 }
+
+// func runLiveMigrateWithRGFile is a modified version of MigrateRunner.Run that stores the
+// package inventory information in a separate resourcegroup file. The logic for this is branched into
+// a separate function to enable feature gating.
+func (mr *MigrateRunner) runLiveMigrateWithRGFile(reader io.Reader, args []string) error {
+	// Validate the number of arguments.
+	if len(args) > 1 {
+		return fmt.Errorf("too many arguments; migrate requires one directory argument (or stdin)")
+	}
+	// Validate argument is a directory.
+	if len(args) == 1 {
+		var err error
+		mr.dir, err = config.NormalizeDir(args[0])
+		if err != nil {
+			return err
+		}
+	}
+	// Store the stdin bytes if necessary so they can be used twice.
+	var stdinBytes []byte
+	var err error
+	if len(args) == 0 {
+		stdinBytes, err = ioutil.ReadAll(reader)
+		if err != nil {
+			return err
+		}
+		if len(stdinBytes) == 0 {
+			return fmt.Errorf("no arguments means stdin has data; missing bytes on stdin")
+		}
+	}
+
+	// Apply the ResourceGroup CRD to the cluster, ignoring if it already exists.
+	if err := mr.applyCRD(); err != nil {
+		return err
+	}
+
+	// Check if we need to migrate from ConfigMap to ResourceGroup.
+	if err := mr.migrateCMToRG(stdinBytes, args); err != nil {
+		return err
+	}
+
+	// Migrate from Kptfile instead.
+	if mr.cmNotMigrated {
+		return mr.migrateKptfileToRG(args)
+	}
+
+	return nil
+}
+
+// migrateKptfileToRG extracts inventory information from a package's Kptfile
+// into an external resourcegroup file.
+func (mr *MigrateRunner) migrateKptfileToRG(args []string) error {
+	const op errors.Op = "migratecmd.migrateKptfileToRG"
+	klog.V(4).Infoln("attempting to migrate from Kptfile inventory")
+	fmt.Fprint(mr.ioStreams.Out, "  reading existing Kptfile...")
+	if !mr.dryRun {
+		dir := args[0]
+		p, err := pkg.New(filesys.FileSystemOrOnDisk{}, dir)
+		if err != nil {
+			return err
+		}
+		kf, err := p.Kptfile()
+		if err != nil {
+			return err
+		}
+
+		if _, err := kptfileutil.ValidateInventory(kf.Inventory); err != nil {
+			// Invalid Kptfile.
+			return err
+		}
+
+		// Make sure resourcegroup file does not exist.
+		_, rgFileErr := os.Stat(filepath.Join(dir, mr.rgFile))
+		switch {
+		case rgFileErr == nil:
+			return errors.E(op, errors.IO, types.UniquePath(dir), "the resourcegroup file already exists and inventory information cannot be migrated")
+		case err != nil && !goerrors.Is(err, os.ErrNotExist):
+			return errors.E(op, errors.IO, types.UniquePath(dir), err)
+		}
+
+		err = (&cmdliveinit.ConfigureInventoryInfo{
+			Pkg:         p,
+			Factory:     mr.factory,
+			Quiet:       true,
+			Name:        kf.Inventory.Name,
+			InventoryID: kf.Inventory.InventoryID,
+			RGFileName:  mr.rgFile,
+			Force:       true,
+		}).Run(mr.ctx)
+
+		if err != nil {
+			return err
+		}
+	}
+	fmt.Fprint(mr.ioStreams.Out, "success\n")
+	return nil
+}
+
+// migrateCMToRG migrates from ConfigMap to resourcegroup object.
+func (mr *MigrateRunner) migrateCMToRG(stdinBytes []byte, args []string) error {
+	// Create the inventory clients for reading inventories based on RG and
+	// ConfigMap.
+	rgInvClient, err := mr.rgInvClientFunc(mr.factory)
+	if err != nil {
+		return err
+	}
+	cmInvClient, err := mr.cmInvClientFunc(mr.factory)
+	if err != nil {
+		return err
+	}
+	// Retrieve the current ConfigMap inventory objects.
+	cmInvObj, err := mr.retrieveConfigMapInv(bytes.NewReader(stdinBytes), args)
+	if err != nil {
+		if _, ok := err.(inventory.NoInventoryObjError); ok {
+			// No ConfigMap inventory means the migration has already run before.
+			klog.V(4).Infoln("swallowing no ConfigMap inventory error")
+			mr.cmNotMigrated = true
+			return nil
+		}
+		klog.V(4).Infof("error retrieving ConfigMap inventory object: %s", err)
+		return err
+	}
+	cmInventoryID := cmInvObj.ID()
+	klog.V(4).Infof("previous inventoryID: %s", cmInventoryID)
+	// Create ResourceGroup object file locallly (e.g. namespace, name, id).
+	if err := mr.createRGfile(mr.ctx, args, cmInventoryID); err != nil {
+		return err
+	}
+	cmObjs, err := mr.retrieveInvObjs(cmInvClient, cmInvObj)
+	if err != nil {
+		return err
+	}
+	if len(cmObjs) > 0 {
+		// Migrate the ConfigMap inventory objects to a ResourceGroup custom resource.
+		if err = mr.migrateObjs(rgInvClient, cmObjs, bytes.NewReader(stdinBytes), args); err != nil {
+			return err
+		}
+		// Delete the old ConfigMap inventory object.
+		if err = mr.deleteConfigMapInv(cmInvClient, cmInvObj); err != nil {
+			return err
+		}
+	}
+	return mr.deleteConfigMapFile()
+}
+
+// createRGfile writes the inventory information into the resourcegroup object.
+func (mr *MigrateRunner) createRGfile(ctx context.Context, args []string, prevID string) error {
+	fmt.Fprint(mr.ioStreams.Out, "  creating ResourceGroup object file...")
+	if !mr.dryRun {
+		p, err := pkg.New(filesys.FileSystemOrOnDisk{}, args[0])
+		if err != nil {
+			return err
+		}
+		err = (&cmdliveinit.ConfigureInventoryInfo{
+			Pkg:         p,
+			Factory:     mr.factory,
+			Quiet:       true,
+			InventoryID: prevID,
+			RGFileName:  mr.rgFile,
+			Force:       mr.force,
+		}).Run(ctx)
+
+		if err != nil {
+			var invExistsError *cmdliveinit.InvExistsError
+			if errors.As(err, &invExistsError) {
+				fmt.Fprint(mr.ioStreams.Out, "values already exist...")
+			} else {
+				return err
+			}
+		}
+	}
+	fmt.Fprint(mr.ioStreams.Out, "success\n")
+	return nil
+}