Merged PR 5137763: update cri to c7bd367

Related work items: jterry75#104, jterry75#107

pkg/config/config.go

@@ -163,6 +163,8 @@ type PluginConfig struct {
 	RestrictOOMScoreAdj bool `toml:"restrict_oom_score_adj" json:"restrictOOMScoreAdj"`
 	// Sets GODEBUG=http2client=0 if enabled.
 	DisableHTTP2Client bool `toml:"disable_http2_client" json:"disableHTTP2Client"`
+	// Determines whether any running containers should be terminated when CRI shuts down or starts up.
+	TerminateContainersOnRestart bool `toml:"terminate_containers_on_restart" json:"terminateContainersOnRestart"`
 }
 
 // X509KeyPairStreaming contains the x509 configuration for streaming

pkg/server/container_create_windows.go

@@ -20,7 +20,6 @@ package server
 
 import (
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"net/url"
 	"path/filepath"
 	"regexp"
@@ -35,20 +34,20 @@ import (
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/snapshots"
-	"github.com/davecgh/go-spew/spew"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/pkg/errors"
-	"golang.org/x/net/context"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-
 	"github.com/containerd/cri/pkg/annotations"
 	criconfig "github.com/containerd/cri/pkg/config"
 	customopts "github.com/containerd/cri/pkg/containerd/opts"
 	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
 	cio "github.com/containerd/cri/pkg/server/io"
 	containerstore "github.com/containerd/cri/pkg/store/container"
 	"github.com/containerd/cri/pkg/util"
+	"github.com/davecgh/go-spew/spew"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
 // CreateContainer creates a new container in the given PodSandbox.
@@ -171,9 +170,9 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	}
 
 	log.G(ctx).WithFields(logrus.Fields{
-		"id": id,
+		"id":             id,
 		"runtimeHandler": sandbox.RuntimeHandler,
-		"spec": spew.NewFormatter(spec),
+		"spec":           spew.NewFormatter(spec),
 	}).Debug("Container creation")
 
 	// If the config field is specified, set the snapshotter label to reuse the pods
@@ -401,7 +400,8 @@ func (c *criService) generateContainerSpec(id string, sandboxID string, sandboxP
 			userstr = image.Config.User
 		}
 		if userstr != "" {
-			g.AddAnnotation("io.microsoft.lcow.userstr", userstr)
+			// For LCOW set the runtime specs Username field so we can take the user string and use this to find the uid:gid pair in the guest.
+			g.SetProcessUsername(userstr)
 		}
 		for _, group := range securityContext.GetSupplementalGroups() {
 			g.AddProcessAdditionalGid(uint32(group))
@@ -509,9 +509,15 @@ func (c *criService) addOCIMounts(ctx context.Context, g *generator, platform im
 				return errors.Errorf(`pipe mount.HostPath '%s' not supported for LCOW`, src)
 			}
 		} else if strings.HasPrefix(src, "sandbox://") {
-			// mount source prefix sandbox:// is only supported with lcow
+			// Sandbox mounts are supported for both Windows and Linux guests, but the type (bind) isn't needed to be set for Windows. This
+			// is just to tell runc what kind of mount to make for the mount provided in the runtime spec.
+			if platform.OS == "linux" {
+				mountType = "bind"
+			}
+		} else if strings.HasPrefix(src, "hugepages://") {
+			// mount source prefix hugepages:// is only supported with lcow
 			if platform.OS != "linux" || platform.Architecture != "amd64" {
-				return errors.Errorf(`sandbox://%s' mounts are only supported for LCOW`, src)
+				return errors.Errorf(`hugepages://%s mounts are only supported for LCOW`, src)
 			}
 			mountType = "bind"
 		} else if strings.Contains(src, "kubernetes.io~empty-dir") && platform.OS == "linux" && platform.Architecture == "amd64" {

pkg/server/restart.go

@@ -58,12 +58,14 @@ func (c *criService) recover(ctx context.Context) error {
 		return errors.Wrap(err, "failed to list sandbox containers")
 	}
 	for _, sandbox := range sandboxes {
+		l := log.G(ctx).WithField("containerID", sandbox.ID())
+		l.Debug("Loading sandbox")
 		sb, err := c.loadSandbox(ctx, sandbox)
 		if err != nil {
-			log.G(ctx).WithError(err).Errorf("Failed to load sandbox %q", sandbox.ID())
+			l.WithError(err).Errorf("Failed to load sandbox")
 			continue
 		}
-		log.G(ctx).Debugf("Loaded sandbox %+v", sb)
+		l.Debugf("Loaded sandbox")
 		if err := c.sandboxStore.Add(sb); err != nil {
 			return errors.Wrapf(err, "failed to add sandbox %q to store", sandbox.ID())
 		}
@@ -78,12 +80,14 @@ func (c *criService) recover(ctx context.Context) error {
 		return errors.Wrap(err, "failed to list containers")
 	}
 	for _, container := range containers {
+		l := log.G(ctx).WithField("containerID", container.ID())
+		l.Debug("Loading container")
 		cntr, err := c.loadContainer(ctx, container)
 		if err != nil {
-			log.G(ctx).WithError(err).Errorf("Failed to load container %q", container.ID())
+			l.WithError(err).Errorf("Failed to load container")
 			continue
 		}
-		log.G(ctx).Debugf("Loaded container %+v", cntr)
+		l.Debugf("Loaded container")
 		if err := c.containerStore.Add(cntr); err != nil {
 			return errors.Wrapf(err, "failed to add container %q to store", container.ID())
 		}
@@ -251,6 +255,11 @@ func (c *criService) loadContainer(ctx context.Context, cntr containerd.Containe
 				// Container is in exited/unknown state, return the status as it is.
 			}
 		} else {
+			// If we need to terminate any running containers, mark it as stopped here.
+			// This will cause it to be stopped via WithProcessKill in the switch case below.
+			if c.config.TerminateContainersOnRestart {
+				s.Status = containerd.Stopped
+			}
 			// Task status is found. Update container status based on the up-to-date task status.
 			switch s.Status {
 			case containerd.Created:
@@ -293,6 +302,7 @@ func (c *criService) loadContainer(ctx context.Context, cntr containerd.Containe
 					c.eventMonitor.startExitMonitor(context.Background(), id, status.Pid, exitCh)
 				}
 			case containerd.Stopped:
+				log.G(ctx).WithField("containerID", cntr.ID()).Info("Deleting dead container task")
 				// Task is stopped. Updata status and delete the task.
 				if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
 					return errors.Wrap(err, "failed to delete task")
@@ -374,7 +384,9 @@ func (c *criService) loadSandbox(ctx context.Context, cntr containerd.Container)
 			// Task does not exist, set sandbox state as NOTREADY.
 			status.State = sandboxstore.StateNotReady
 		} else {
-			if taskStatus.Status == containerd.Running {
+			// If we need to terminate running containers, treat it as stopped, so the else condition
+			// will clean it up via WithProcessKill.
+			if taskStatus.Status == containerd.Running && !c.config.TerminateContainersOnRestart {
 				// Wait for the task for sandbox monitor.
 				// wait is a long running background request, no timeout needed.
 				exitCh, err := t.Wait(ctrdutil.NamespacedContext())
@@ -390,6 +402,7 @@ func (c *criService) loadSandbox(ctx context.Context, cntr containerd.Container)
 					c.eventMonitor.startExitMonitor(context.Background(), meta.ID, status.Pid, exitCh)
 				}
 			} else {
+				log.G(ctx).WithField("sandboxID", cntr.ID()).Info("Deleting dead sandbox task")
 				// Task is not running. Delete the task and set sandbox state as NOTREADY.
 				if _, err := t.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) {
 					return status, errors.Wrap(err, "failed to delete task")

pkg/server/sandbox_run_windows.go

@@ -19,7 +19,6 @@ limitations under the License.
 package server
 
 import (
-	"github.com/sirupsen/logrus"
 	"strconv"
 
 	runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
@@ -30,20 +29,20 @@ import (
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/snapshots"
-	"github.com/davecgh/go-spew/spew"
-	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/pkg/errors"
-	"golang.org/x/net/context"
-	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-
 	"github.com/containerd/cri/pkg/annotations"
 	criconfig "github.com/containerd/cri/pkg/config"
 	customopts "github.com/containerd/cri/pkg/containerd/opts"
 	ctrdutil "github.com/containerd/cri/pkg/containerd/util"
 	"github.com/containerd/cri/pkg/netns"
 	sandboxstore "github.com/containerd/cri/pkg/store/sandbox"
 	"github.com/containerd/cri/pkg/util"
+	"github.com/davecgh/go-spew/spew"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
 // RunPodSandbox creates and starts a pod-level sandbox. Runtimes should ensure
@@ -174,9 +173,9 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		return nil, errors.Wrap(err, "failed to generate sandbox container spec")
 	}
 	log.G(ctx).WithFields(logrus.Fields{
-		"id": id,
+		"id":             id,
 		"runtimeHandler": runtimeHandler,
-		"spec": spew.NewFormatter(spec),
+		"spec":           spew.NewFormatter(spec),
 	}).Debug("Sandbox container creation")
 
 	sandboxLabels := buildLabels(config.Labels, containerKindSandbox)
@@ -377,7 +376,8 @@ func (c *criService) generateSandboxContainerSpec(id string, config *runtime.Pod
 			userstr = imageConfig.User
 		}
 		if userstr != "" {
-			g.AddAnnotation("io.microsoft.lcow.userstr", userstr)
+			// For LCOW set the runtime specs Username field so we can take the user string and use this to find the uid:gid pair in the guest.
+			g.SetProcessUsername(userstr)
 		}
 
 		for _, group := range securityContext.GetSupplementalGroups() {

pkg/server/service.go

@@ -17,6 +17,7 @@ limitations under the License.
 package server
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -26,6 +27,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/plugin"
 	cni "github.com/containerd/go-cni"
 	"github.com/pkg/errors"
@@ -272,6 +274,15 @@ func (c *criService) Run() error {
 // TODO(random-liu): Make close synchronous.
 func (c *criService) Close() error {
 	logrus.Info("Stop CRI service")
+	if c.config.TerminateContainersOnRestart {
+		// We need to stop any running containers. Do this by stopping all pods.
+		ctx := context.Background()
+		for _, sandbox := range c.sandboxStore.List() {
+			if err := c.stopPodSandbox(ctx, sandbox); err != nil {
+				log.G(ctx).WithField("sandboxID", sandbox.Metadata.ID).Error("Failed to stop sandbox on shutdown")
+			}
+		}
+	}
 	c.eventMonitor.stop()
 	if err := c.streamServer.Stop(); err != nil {
 		return errors.Wrap(err, "failed to stop stream server")

vendor.conf

@@ -3,7 +3,7 @@ github.com/blang/semver v3.1.0
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/containerd/cgroups caf71576c8b19daf80ab4685916e4d5b4c74887e
 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23
-github.com/containerd/containerd a828a5ec1b35001bcf770651c1aada6167717411 https://github.com/kevpar/containerd.git # fork/release/1.4
+github.com/containerd/containerd 908bdfd1381b8755779ed2a876d7330c37b47b1a https://github.com/kevpar/containerd.git # fork/release/1.4
 github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
 github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90
@@ -34,7 +34,7 @@ github.com/json-iterator/go 1.1.5
 github.com/konsorten/go-windows-terminal-sequences v1.0.1
 github.com/matttproud/golang_protobuf_extensions v1.0.0
 github.com/Microsoft/go-winio v0.4.16
-github.com/Microsoft/hcsshim e7fd842da715873202253f232212a7fa925be63c
+github.com/Microsoft/hcsshim 057bebe3caf7811ed63efae5b83bf893f21648d4
 github.com/modern-go/concurrent 1.0.3
 github.com/modern-go/reflect2 1.0.1
 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7