diff --git a/.golangci.yml b/.golangci.yml index a23703091e..b4af24858a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -64,6 +64,7 @@ linters: - nestif - nilerr - nilnesserr + - nlreturn - noctx - nolintlint - nosprintfhostport @@ -97,6 +98,7 @@ linters: - usetesting - wastedassign - whitespace + - wsl - zerologlint # - depguard # - err113 @@ -110,12 +112,10 @@ linters: # - lll # - mnd # - nilnil - # - nlreturn # - nonamedreturns # - testpackage # - varnamelen # - wrapcheck - # - wsl linters-settings: stylecheck: checks: diff --git a/cmd/crictl/attach.go b/cmd/crictl/attach.go index 9b41d99a14..b0f8de656e 100644 --- a/cmd/crictl/attach.go +++ b/cmd/crictl/attach.go @@ -102,6 +102,7 @@ var runtimeAttachCommand = &cli.Command{ if err = Attach(ctx, runtimeClient, opts); err != nil { return fmt.Errorf("attaching running container failed: %w", err) } + return nil }, } @@ -111,6 +112,7 @@ func Attach(ctx context.Context, client internalapi.RuntimeService, opts attachO if opts.id == "" { return errors.New("ID cannot be empty") } + request := &pb.AttachRequest{ ContainerId: opts.id, Tty: opts.tty, @@ -119,13 +121,16 @@ func Attach(ctx context.Context, client internalapi.RuntimeService, opts attachO Stderr: !opts.tty, } logrus.Debugf("AttachRequest: %v", request) + r, err := InterruptableRPC(ctx, func(ctx context.Context) (*pb.AttachResponse, error) { return client.Attach(ctx, request) }) logrus.Debugf("AttachResponse: %v", r) + if err != nil { return err } + attachURL := r.Url URL, err := url.Parse(attachURL) @@ -136,10 +141,12 @@ func Attach(ctx context.Context, client internalapi.RuntimeService, opts attachO if URL.Host == "" { URL.Host = kubeletURLHost } + if URL.Scheme == "" { URL.Scheme = kubeletURLSchema } logrus.Debugf("Attach URL: %v", URL) + return stream(ctx, opts.stdin, opts.tty, opts.transport, URL, opts.tlsConfig) } diff --git a/cmd/crictl/completion.go b/cmd/crictl/completion.go index 85b672de5a..06139673f7 100644 --- a/cmd/crictl/completion.go +++ b/cmd/crictl/completion.go @@ -37,10 +37,12 @@ complete -F _crictl crictl` func bashCompletion(c *cli.Context) error { subcommands := []string{} + for _, command := range c.App.Commands { if command.Hidden { continue } + subcommands = append(subcommands, command.Names()...) } @@ -50,6 +52,7 @@ func bashCompletion(c *cli.Context) error { } fmt.Fprintln(c.App.Writer, fmt.Sprintf(bashCompletionTemplate, strings.Join(subcommands, "\n"))) + return nil } @@ -71,10 +74,12 @@ compdef _crictl crictl` func zshCompletion(c *cli.Context) error { subcommands := []string{} + for _, command := range c.App.Commands { if command.Hidden { continue } + for _, name := range command.Names() { subcommands = append(subcommands, name+":"+command.Usage) } @@ -87,6 +92,7 @@ func zshCompletion(c *cli.Context) error { } fmt.Fprintln(c.App.Writer, fmt.Sprintf(zshCompletionTemplate, strings.Join(subcommands, "' '"), strings.Join(opts, "' '"))) + return nil } @@ -95,7 +101,9 @@ func fishCompletion(c *cli.Context) error { if err != nil { return err } + fmt.Fprintln(c.App.Writer, completion) + return nil } diff --git a/cmd/crictl/config.go b/cmd/crictl/config.go index c229b63c87..9870e54e3d 100644 --- a/cmd/crictl/config.go +++ b/cmd/crictl/config.go @@ -98,6 +98,7 @@ CRICTL OPTIONS: default: return fmt.Errorf("no configuration option named %s", get) } + return nil } else if c.IsSet("set") { settings := c.StringSlice("set") @@ -115,6 +116,7 @@ CRICTL OPTIONS: } } } + return common.WriteConfig(config, configFile) } else if c.Bool("list") { display := newDefaultTableDisplay() @@ -141,6 +143,7 @@ CRICTL OPTIONS: if err := setValue(key, value, config); err != nil { return fmt.Errorf("set %q to %q: %w", key, value, err) } + return common.WriteConfig(config, configFile) }, } @@ -156,27 +159,32 @@ func setValue(key, value string, config *common.Config) error { if err != nil { return fmt.Errorf("parse timeout value '%s': %w", value, err) } + config.Timeout = n case common.Debug: debug, err := strconv.ParseBool(value) if err != nil { return fmt.Errorf("parse debug value '%s': %w", value, err) } + config.Debug = debug case common.PullImageOnCreate: pi, err := strconv.ParseBool(value) if err != nil { return fmt.Errorf("parse pull-image-on-create value '%s': %w", value, err) } + config.PullImageOnCreate = pi case common.DisablePullOnRun: pi, err := strconv.ParseBool(value) if err != nil { return fmt.Errorf("parse disable-pull-on-run value '%s': %w", value, err) } + config.DisablePullOnRun = pi default: return fmt.Errorf("no configuration option named %s", key) } + return nil } diff --git a/cmd/crictl/container.go b/cmd/crictl/container.go index 05d708a9c9..cab769a8fc 100644 --- a/cmd/crictl/container.go +++ b/cmd/crictl/container.go @@ -253,6 +253,7 @@ var createContainerCommand = &cli.Command{ return fmt.Errorf("creating container: %w", err) } fmt.Println(ctrID) + return nil }, } @@ -276,6 +277,7 @@ var startContainerCommand = &cli.Command{ return fmt.Errorf("starting the container %q: %w", containerID, err) } } + return nil }, } @@ -344,6 +346,7 @@ var updateContainerCommand = &cli.Command{ return fmt.Errorf("updating container resources for %q: %w", containerID, err) } } + return nil }, } @@ -375,6 +378,7 @@ var stopContainerCommand = &cli.Command{ return fmt.Errorf("stopping the container %q: %w", containerID, err) } } + return nil }, } @@ -424,8 +428,10 @@ var removeContainerCommand = &cli.Command{ if len(ids) == 0 { if ctx.Bool("all") { logrus.Info("No containers to remove") + return nil } + return cli.ShowSubcommandHelp(ctx) } @@ -578,6 +584,7 @@ var containerStatusCommand = &cli.Command{ if len(ids) == 0 { logrus.Error("No IDs provided or nothing found per filter") + return cli.ShowSubcommandHelp(c) } @@ -710,6 +717,7 @@ var listContainersCommand = &cli.Command{ if err = OutputContainers(runtimeClient, imageClient, opts); err != nil { return fmt.Errorf("listing containers: %w", err) } + return nil }, } @@ -759,6 +767,7 @@ var runContainerCommand = &cli.Command{ if err = RunContainer(imageClient, runtimeClient, opts, c.String("runtime")); err != nil { return fmt.Errorf("running container: %w", err) } + return nil }, } @@ -801,6 +810,7 @@ var checkpointContainerCommand = &cli.Command{ return fmt.Errorf("checkpointing the container %q failed: %w", containerID, err) } } + return nil }, } @@ -826,6 +836,7 @@ func RunContainer( // Create the container containerOptions := createOptions{podID, &opts} + ctrID, err := CreateContainer(iClient, rClient, containerOptions) if err != nil { return fmt.Errorf("creating container failed: %w", err) @@ -836,6 +847,7 @@ func RunContainer( if err != nil { return fmt.Errorf("starting the container %q: %w", ctrID, err) } + return nil } @@ -850,6 +862,7 @@ func CreateContainer( if err != nil { return "", err } + var podConfig *pb.PodSandboxConfig if opts.podConfig != "" { podConfig, err = loadPodSandboxConfig(opts.podConfig) @@ -899,13 +912,16 @@ func CreateContainer( SandboxConfig: podConfig, } logrus.Debugf("CreateContainerRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (string, error) { return rClient.CreateContainer(ctx, opts.podID, config, podConfig) }) logrus.Debugf("CreateContainerResponse: %v", r) + if err != nil { return "", err } + return r, nil } @@ -915,12 +931,15 @@ func StartContainer(client internalapi.RuntimeService, id string) error { if id == "" { return errors.New("ID cannot be empty") } + if _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.StartContainer(ctx, id) }); err != nil { return err } + fmt.Println(id) + return nil } @@ -951,6 +970,7 @@ func UpdateContainerResources(client internalapi.RuntimeService, id string, opts if id == "" { return errors.New("ID cannot be empty") } + request := &pb.UpdateContainerResourcesRequest{ ContainerId: id, } @@ -972,14 +992,18 @@ func UpdateContainerResources(client internalapi.RuntimeService, id string, opts MemoryLimitInBytes: opts.MemoryLimitInBytes, } } + logrus.Debugf("UpdateContainerResourcesRequest: %v", request) resources := &pb.ContainerResources{Linux: request.Linux, Windows: request.Windows} + if _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.UpdateContainerResources(ctx, id, resources) }); err != nil { return err } + fmt.Println(id) + return nil } @@ -989,13 +1013,17 @@ func StopContainer(client internalapi.RuntimeService, id string, timeout int64) if id == "" { return errors.New("ID cannot be empty") } + logrus.Debugf("Stopping container: %s (timeout = %v)", id, timeout) + if _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.StopContainer(ctx, id, timeout) }); err != nil { return err } + fmt.Println(id) + return nil } @@ -1008,18 +1036,22 @@ func CheckpointContainer( if id == "" { return errors.New("ID cannot be empty") } + request := &pb.CheckpointContainerRequest{ ContainerId: id, Location: export, } logrus.Debugf("CheckpointContainerRequest: %v", request) + _, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.ImageFsInfoResponse, error) { return nil, rClient.CheckpointContainer(ctx, request) }) if err != nil { return err } + fmt.Println(id) + return nil } @@ -1029,13 +1061,17 @@ func RemoveContainer(client internalapi.RuntimeService, id string) error { if id == "" { return errors.New("ID cannot be empty") } + logrus.Debugf("Removing container: %s", id) + if _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.RemoveContainer(ctx, id) }); err != nil { return err } + fmt.Println(id) + return nil } @@ -1046,27 +1082,33 @@ func marshalContainerStatus(cs *pb.ContainerStatus) (string, error) { if err != nil { return "", err } + jsonMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(statusStr), &jsonMap) if err != nil { return "", err } jsonMap["createdAt"] = time.Unix(0, cs.CreatedAt).Format(time.RFC3339Nano) + var startedAt, finishedAt time.Time if cs.State != pb.ContainerState_CONTAINER_CREATED { // If container is not in the created state, we have tried and // started the container. Set the startedAt. startedAt = time.Unix(0, cs.StartedAt) } + if cs.State == pb.ContainerState_CONTAINER_EXITED || (cs.State == pb.ContainerState_CONTAINER_UNKNOWN && cs.FinishedAt > 0) { // If container is in the exit state, set the finishedAt. // Or if container is in the unknown state and FinishedAt > 0, set the finishedAt finishedAt = time.Unix(0, cs.FinishedAt) } + jsonMap["startedAt"] = startedAt.Format(time.RFC3339Nano) jsonMap["finishedAt"] = finishedAt.Format(time.RFC3339Nano) + return marshalMapInOrder(jsonMap, *cs) } @@ -1076,24 +1118,29 @@ func marshalContainerStatus(cs *pb.ContainerStatus) (string, error) { //nolint:dupl // pods and containers are similar, but still different func containerStatus(client internalapi.RuntimeService, ids []string, output, tmplStr string, quiet bool) error { verbose := !(quiet) + if output == "" { // default to json output output = outputTypeJSON } + if len(ids) == 0 { return errors.New("ID cannot be empty") } statuses := []statusData{} + for _, id := range ids { request := &pb.ContainerStatusRequest{ ContainerId: id, Verbose: verbose, } logrus.Debugf("ContainerStatusRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.ContainerStatusResponse, error) { return client.ContainerStatus(ctx, id, verbose) }) logrus.Debugf("ContainerStatusResponse: %v", r) + if err != nil { return fmt.Errorf("get container status: %w", err) } @@ -1115,40 +1162,51 @@ func containerStatus(client internalapi.RuntimeService, ids []string, output, tm func outputContainerStatusTable(r *pb.ContainerStatusResponse, verbose bool) { fmt.Printf("ID: %s\n", r.Status.Id) + if r.Status.Metadata != nil { if r.Status.Metadata.Name != "" { fmt.Printf("Name: %s\n", r.Status.Metadata.Name) } + if r.Status.Metadata.Attempt != 0 { fmt.Printf("Attempt: %v\n", r.Status.Metadata.Attempt) } } + fmt.Printf("State: %s\n", r.Status.State) ctm := time.Unix(0, r.Status.CreatedAt) fmt.Printf("Created: %v\n", units.HumanDuration(time.Now().UTC().Sub(ctm))+" ago") + if r.Status.State != pb.ContainerState_CONTAINER_CREATED { stm := time.Unix(0, r.Status.StartedAt) fmt.Printf("Started: %v\n", units.HumanDuration(time.Now().UTC().Sub(stm))+" ago") } + if r.Status.State == pb.ContainerState_CONTAINER_EXITED { if r.Status.FinishedAt > 0 { ftm := time.Unix(0, r.Status.FinishedAt) fmt.Printf("Finished: %v\n", units.HumanDuration(time.Now().UTC().Sub(ftm))+" ago") } + fmt.Printf("Exit Code: %v\n", r.Status.ExitCode) } + if r.Status.Labels != nil { fmt.Println("Labels:") + for _, k := range getSortedKeys(r.Status.Labels) { fmt.Printf("\t%s -> %s\n", k, r.Status.Labels[k]) } } + if r.Status.Annotations != nil { fmt.Println("Annotations:") + for _, k := range getSortedKeys(r.Status.Annotations) { fmt.Printf("\t%s -> %s\n", k, r.Status.Annotations[k]) } } + if verbose { fmt.Printf("Info: %v\n", r.GetInfo()) } @@ -1161,16 +1219,20 @@ func ListContainers(runtimeClient internalapi.RuntimeService, imageClient intern if opts.id != "" { filter.Id = opts.id } + if opts.podID != "" { filter.PodSandboxId = opts.podID } + st := &pb.ContainerStateValue{} if !opts.all && opts.state == "" { st.State = pb.ContainerState_CONTAINER_RUNNING filter.State = st } + if opts.state != "" { st.State = pb.ContainerState_CONTAINER_UNKNOWN + switch strings.ToLower(opts.state) { case "created": st.State = pb.ContainerState_CONTAINER_CREATED @@ -1188,20 +1250,25 @@ func ListContainers(runtimeClient internalapi.RuntimeService, imageClient intern log.Fatalf("--state should be one of created, running, exited or unknown") } } + if opts.latest || opts.last > 0 { // Do not filter by state if latest/last is specified. filter.State = nil } + if opts.labels != nil { filter.LabelSelector = opts.labels } + r, err := InterruptableRPC(nil, func(ctx context.Context) ([]*pb.Container, error) { return runtimeClient.ListContainers(ctx, filter) }) logrus.Debugf("ListContainerResponse: %v", r) + if err != nil { return nil, fmt.Errorf("call list containers RPC: %w", err) } + return getContainersList(imageClient, r, opts) } @@ -1228,22 +1295,27 @@ func OutputContainers(runtimeClient internalapi.RuntimeService, imageClient inte if !opts.verbose && !opts.quiet { display.AddRow([]string{columnContainer, columnImage, columnCreated, columnState, columnName, columnAttempt, columnPodID, columnPodName, columnNamespace}) } + for _, c := range r { if opts.quiet { fmt.Printf("%s\n", c.Id) + continue } createdAt := time.Unix(0, c.CreatedAt) ctm := units.HumanDuration(time.Now().UTC().Sub(createdAt)) + " ago" podNamespace := getPodNamespaceFromLabels(c.Labels) + if !opts.verbose { id := c.Id podID := c.PodSandboxId + var image string if c.Image != nil { image = c.Image.Image } + if !opts.noTrunc { id = getTruncatedID(id, "") podID = getTruncatedID(podID, "") @@ -1252,51 +1324,66 @@ func OutputContainers(runtimeClient internalapi.RuntimeService, imageClient inte image = getTruncatedID(digest.String(), string(digest.Algorithm())+":") } } + if opts.resolveImagePath { orig, err := getRepoImage(imageClient, image) if err != nil { return fmt.Errorf("failed to fetch repo image %w", err) } + image = orig } + podName := getPodNameFromLabels(c.Labels) display.AddRow([]string{ id, image, ctm, convertContainerState(c.State), c.Metadata.Name, strconv.FormatUint(uint64(c.Metadata.Attempt), 10), podID, podName, podNamespace, }) + continue } fmt.Printf("ID: %s\n", c.Id) fmt.Printf("PodID: %s\n", c.PodSandboxId) fmt.Printf("Namespace: %s\n", podNamespace) + if c.Metadata != nil { if c.Metadata.Name != "" { fmt.Printf("Name: %s\n", c.Metadata.Name) } + fmt.Printf("Attempt: %v\n", c.Metadata.Attempt) } + fmt.Printf("State: %s\n", convertContainerState(c.State)) + if c.Image != nil { fmt.Printf("Image: %s\n", c.Image.Image) } + fmt.Printf("Created: %v\n", ctm) + if c.Labels != nil { fmt.Println("Labels:") + for _, k := range getSortedKeys(c.Labels) { fmt.Printf("\t%s -> %s\n", k, c.Labels[k]) } } + if c.Annotations != nil { fmt.Println("Annotations:") + for _, k := range getSortedKeys(c.Annotations) { fmt.Printf("\t%s -> %s\n", k, c.Annotations[k]) } } + fmt.Println() } display.Flush() + return nil } @@ -1312,6 +1399,7 @@ func convertContainerState(state pb.ContainerState) string { return "Unknown" default: log.Fatalf("Unknown container state %q", state) + return "" } } @@ -1329,11 +1417,13 @@ func getFromLabels(labels map[string]string, label string) string { if ok { return value } + return "unknown" } func getContainersList(imageClient internalapi.ImageManagerService, containersList []*pb.Container, opts *listOptions) ([]*pb.Container, error) { filtered := []*pb.Container{} + for _, c := range containersList { if match, err := matchesImage(imageClient, opts.image, c.GetImage().GetImage()); err != nil { return nil, fmt.Errorf("check image match: %w", err) @@ -1351,17 +1441,21 @@ func getContainersList(imageClient internalapi.ImageManagerService, containersLi } sort.Sort(containerByCreated(filtered)) + n := len(filtered) if opts.latest { n = 1 } + if opts.last > 0 { n = opts.last } + n = func(a, b int) int { if a < b { return a } + return b }(n, len(filtered)) diff --git a/cmd/crictl/container_stats.go b/cmd/crictl/container_stats.go index aa2eb9676b..06435fb7de 100644 --- a/cmd/crictl/container_stats.go +++ b/cmd/crictl/container_stats.go @@ -120,6 +120,7 @@ var statsCommand = &cli.Command{ if err = ContainerStats(runtimeClient, opts); err != nil { return fmt.Errorf("get container stats: %w", err) } + return nil }, } @@ -161,17 +162,21 @@ func (d containerStatsDisplayer) displayStats(ctx context.Context, client intern if err != nil { return err } + switch d.opts.output { case outputTypeJSON: return outputProtobufObjAsJSON(r) case outputTypeYAML: return outputProtobufObjAsYAML(r) } + oldStats := make(map[string]*pb.ContainerStats) + for _, s := range r.GetStats() { if ctx.Err() != nil { return ctx.Err() } + oldStats[s.Attributes.Id] = s } @@ -183,10 +188,12 @@ func (d containerStatsDisplayer) displayStats(ctx context.Context, client intern } d.display.AddRow([]string{columnContainer, columnName, columnCPU, columnMemory, columnDisk, columnInodes, columnSwap}) + for _, s := range r.GetStats() { if ctx.Err() != nil { return ctx.Err() } + id := getTruncatedID(s.Attributes.Id, "") name := s.GetAttributes().GetMetadata().GetName() cpu := s.GetCpu().GetUsageCoreNanoSeconds().GetValue() @@ -194,29 +201,36 @@ func (d containerStatsDisplayer) displayStats(ctx context.Context, client intern disk := s.GetWritableLayer().GetUsedBytes().GetValue() inodes := s.GetWritableLayer().GetInodesUsed().GetValue() swap := s.GetSwap().GetSwapUsageBytes().GetValue() + if !d.opts.all && cpu == 0 && mem == 0 { // Skip non-running container continue } + old, ok := oldStats[s.Attributes.Id] if !ok { // Skip new container continue } + var cpuPerc float64 + if cpu != 0 { // Only generate cpuPerc for running container duration := s.GetCpu().GetTimestamp() - old.GetCpu().GetTimestamp() if duration == 0 { return errors.New("cpu stat is not updated during sample") } + cpuPerc = float64(cpu-old.GetCpu().GetUsageCoreNanoSeconds().GetValue()) / float64(duration) * 100 } + d.display.AddRow([]string{ id, name, fmt.Sprintf("%.2f", cpuPerc), units.HumanSize(float64(mem)), units.HumanSize(float64(disk)), strconv.FormatUint(inodes, 10), units.HumanSize(float64(swap)), }) } + d.display.ClearScreen() d.display.Flush() @@ -225,13 +239,17 @@ func (d containerStatsDisplayer) displayStats(ctx context.Context, client intern func getContainerStats(ctx context.Context, client internalapi.RuntimeService, request *pb.ListContainerStatsRequest) (*pb.ListContainerStatsResponse, error) { logrus.Debugf("ListContainerStatsRequest: %v", request) + r, err := InterruptableRPC(ctx, func(ctx context.Context) ([]*pb.ContainerStats, error) { return client.ListContainerStats(ctx, request.Filter) }) logrus.Debugf("ListContainerResponse: %v", r) + if err != nil { return nil, err } + sort.Sort(containerStatsByID(r)) + return &pb.ListContainerStatsResponse{Stats: r}, nil } diff --git a/cmd/crictl/container_test.go b/cmd/crictl/container_test.go index 69e33209e6..2e1aa2c5ca 100644 --- a/cmd/crictl/container_test.go +++ b/cmd/crictl/container_test.go @@ -29,10 +29,12 @@ import ( func fakeContainersWithCreatedAtDesc(names ...string) []*pb.Container { containers := make([]*pb.Container, len(names)) creationTime := time.Date(2023, 1, 1, 12, 0o0, 0o0, 0o0, time.UTC) + for i, name := range names { containers[i] = fakeContainer(name, creationTime.UnixNano()) creationTime = creationTime.Truncate(time.Hour) } + return containers } diff --git a/cmd/crictl/display.go b/cmd/crictl/display.go index 18e3019783..b6e2735e59 100644 --- a/cmd/crictl/display.go +++ b/cmd/crictl/display.go @@ -60,6 +60,7 @@ func newDefaultTableDisplay() *display { // newTableDisplay creates a display instance, and uses to format output with table. func newTableDisplay(minwidth, tabwidth, padding int, padchar byte, flags uint) *display { w := tabwriter.NewWriter(os.Stdout, minwidth, tabwidth, padding, padchar, flags) + return &display{w} } diff --git a/cmd/crictl/events.go b/cmd/crictl/events.go index 4cf0df70d9..22fd12f6d7 100644 --- a/cmd/crictl/events.go +++ b/cmd/crictl/events.go @@ -80,13 +80,16 @@ func Events(cliContext *cli.Context, client internalapi.RuntimeService) error { errCh := make(chan error, 1) containerEventsCh := make(chan *pb.ContainerEventResponse) + go func() { logrus.Debug("getting container events") + _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.GetContainerEvents(ctx, containerEventsCh, nil) }) if errors.Is(err, io.EOF) { errCh <- nil + return } errCh <- err diff --git a/cmd/crictl/exec.go b/cmd/crictl/exec.go index 725b86cf71..4a5e12656c 100644 --- a/cmd/crictl/exec.go +++ b/cmd/crictl/exec.go @@ -209,6 +209,7 @@ var runtimeExecCommand = &cli.Command{ if len(ids) == 0 { logrus.Error("No containers found per filter flags") + return cli.ShowSubcommandHelp(c) } } else if c.NArg() < 2 { @@ -262,6 +263,7 @@ var runtimeExecCommand = &cli.Command{ if ignoreErrors { logrus.Debugf("Ignoring errors: %v", errs) + return nil } @@ -286,6 +288,7 @@ func tlsConfigFromFlags(ctx *cli.Context) (*rest.TLSClientConfig, error) { if cfg.CAFile == "" && cfg.CertFile == "" && cfg.KeyFile == "" { return &rest.TLSClientConfig{Insecure: true}, nil } + if cfg.CertFile == "" || cfg.KeyFile == "" { return nil, fmt.Errorf( "all two flags --%s and --%s are required for TLS streaming, only --%s is optional", @@ -306,22 +309,28 @@ func ExecSync(client internalapi.RuntimeService, opts *execOptions) (int, error) Timeout: opts.timeout, } logrus.Debugf("ExecSyncRequest: %v", request) + timeoutDuration := time.Duration(opts.timeout) * time.Second + type stdio struct { stdout, stderr []byte } + io, err := InterruptableRPC(nil, func(ctx context.Context) (*stdio, error) { stdout, stderr, err := client.ExecSync(ctx, opts.id, opts.cmd, timeoutDuration) if err != nil { return nil, err } + return &stdio{stdout, stderr}, nil }) if err != nil { return 1, err } + fmt.Println(string(io.stdout)) fmt.Println(string(io.stderr)) + return 0, nil } @@ -337,13 +346,16 @@ func Exec(ctx context.Context, client internalapi.RuntimeService, opts *execOpti } logrus.Debugf("ExecRequest: %v", request) + r, err := InterruptableRPC(ctx, func(ctx context.Context) (*pb.ExecResponse, error) { return client.Exec(ctx, request) }) logrus.Debugf("ExecResponse: %v", r) + if err != nil { return err } + execURL := r.Url URL, err := url.Parse(execURL) @@ -360,6 +372,7 @@ func Exec(ctx context.Context, client internalapi.RuntimeService, opts *execOpti } logrus.Debugf("Exec URL: %v", URL) + return stream(ctx, opts.stdin, opts.tty, opts.transport, URL, opts.tlsConfig) } @@ -375,22 +388,29 @@ func stream(ctx context.Context, in, tty bool, transport string, parsedURL *url. Stderr: stderr, Tty: tty, } + if in { streamOptions.Stdin = stdin } + logrus.Debugf("StreamOptions: %v", streamOptions) + if !tty { return executor.StreamWithContext(ctx, streamOptions) } + detachKeys, err := mobyterm.ToBytes(detachSequence) if err != nil { return errors.New("could not bind detach keys") } + pr := mobyterm.NewEscapeProxy(streamOptions.Stdin, detachKeys) streamOptions.Stdin = pr + if !in { return errors.New("tty=true must be specified with interactive=true") } + t := term.TTY{ In: stdin, Out: stdout, @@ -399,7 +419,9 @@ func stream(ctx context.Context, in, tty bool, transport string, parsedURL *url. if !t.IsTerminalIn() { return errors.New("input is not a terminal") } + streamOptions.TerminalSizeQueue = t.MonitorSize(t.GetSize()) + return t.Safe(func() error { return executor.StreamWithContext(ctx, streamOptions) }) } diff --git a/cmd/crictl/image.go b/cmd/crictl/image.go index faea7c85e4..5ba1a264d9 100644 --- a/cmd/crictl/image.go +++ b/cmd/crictl/image.go @@ -45,9 +45,11 @@ func (a imageByRef) Less(i, j int) bool { if len(a[i].RepoTags) > 0 && len(a[j].RepoTags) > 0 { return a[i].RepoTags[0] < a[j].RepoTags[0] } + if len(a[i].RepoDigests) > 0 && len(a[j].RepoDigests) > 0 { return a[i].RepoDigests[0] < a[j].RepoDigests[0] } + return a[i].Id < a[j].Id } @@ -138,6 +140,7 @@ var pullImageCommand = &cli.Command{ return fmt.Errorf("pulling image: %w", err) } fmt.Printf("Image is up to date for %s\n", r.ImageRef) + return nil }, } @@ -225,6 +228,7 @@ var listImageCommand = &cli.Command{ for _, image := range r.Images { if quiet { fmt.Printf("%s\n", image.Id) + continue } if !verbose { @@ -247,6 +251,7 @@ var listImageCommand = &cli.Command{ } display.AddRow(row) } + continue } fmt.Printf("ID: %s\n", image.Id) @@ -271,6 +276,7 @@ var listImageCommand = &cli.Command{ fmt.Printf("\n") } display.Flush() + return nil }, } @@ -332,6 +338,7 @@ var imageStatusCommand = &cli.Command{ if len(ids) == 0 { logrus.Error("No IDs provided or nothing found per filter") + return cli.ShowSubcommandHelp(c) } @@ -365,14 +372,18 @@ var imageStatusCommand = &cli.Command{ func outputImageStatusTable(r *pb.ImageStatusResponse, verbose bool) { // otherwise output in table format fmt.Printf("ID: %s\n", r.Image.Id) + for _, tag := range r.Image.RepoTags { fmt.Printf("Tag: %s\n", tag) } + for _, digest := range r.Image.RepoDigests { fmt.Printf("Digest: %s\n", digest) } + size := units.HumanSizeWithPrecision(float64(r.Image.GetSize_()), 3) fmt.Printf("Size: %s\n", size) + if verbose { fmt.Printf("Info: %v\n", r.GetInfo()) } @@ -422,6 +433,7 @@ var removeImageCommand = &cli.Command{ // Pinned images should not be removed on prune. if prune && img.Pinned { logrus.Debugf("Excluding pinned container image: %v", img.GetId()) + continue } logrus.Debugf("Adding container image to be removed: %v", img.GetId()) @@ -451,6 +463,7 @@ var removeImageCommand = &cli.Command{ "image status request for %q failed: %v", img, err, ) + continue } id := imageStatus.GetImage().GetId() @@ -462,8 +475,10 @@ var removeImageCommand = &cli.Command{ if len(ids) == 0 { if all || prune { logrus.Info("No images to remove") + return nil } + return cli.ShowSubcommandHelp(cliCtx) } @@ -487,6 +502,7 @@ var removeImageCommand = &cli.Command{ if !prune { return fmt.Errorf("error of removing image %q: %w", id, err) } + return nil } if len(status.Image.RepoTags) == 0 { @@ -495,11 +511,13 @@ var removeImageCommand = &cli.Command{ for _, repoDigest := range status.Image.RepoDigests { fmt.Printf("Deleted: %s\n", repoDigest) } + return nil } for _, repoTag := range status.Image.RepoTags { fmt.Printf("Deleted: %s\n", repoTag) } + return nil }) } @@ -557,6 +575,7 @@ var imageFsInfoCommand = &cli.Command{ func outputImageFsInfoTable(r *pb.ImageFsInfoResponse) { tablePrintFileSystem := func(fileLabel string, filesystem []*pb.FilesystemUsage) { fmt.Printf("%s Filesystem \n", fileLabel) + for i, val := range filesystem { fmt.Printf("TimeStamp[%d]: %d\n", i, val.Timestamp) fmt.Printf("Disk[%d]: %s\n", i, units.HumanSize(float64(val.UsedBytes.GetValue()))) @@ -573,48 +592,61 @@ func parseCreds(creds string) (username, password string, err error) { if creds == "" { return "", "", errors.New("credentials can't be empty") } + up := strings.SplitN(creds, ":", 2) if len(up) == 1 { return up[0], "", nil } + if up[0] == "" { return "", "", errors.New("username can't be empty") } + return up[0], up[1], nil } func getAuth(creds, auth, username string) (*pb.AuthConfig, error) { if username != "" { fmt.Print("Enter Password:") + bytePassword, err := term.ReadPassword(int(syscall.Stdin)) //nolint:unconvert // required for windows + fmt.Print("\n") + if err != nil { return nil, err } + password := string(bytePassword) + return &pb.AuthConfig{ Username: username, Password: password, }, nil } + if creds != "" && auth != "" { return nil, errors.New("both `--creds` and `--auth` are specified") } + if creds != "" { username, password, err := parseCreds(creds) if err != nil { return nil, err } + return &pb.AuthConfig{ Username: username, Password: password, }, nil } + if auth != "" { return &pb.AuthConfig{ Auth: auth, }, nil } + return nil, nil } @@ -624,20 +656,26 @@ func normalizeRepoTagPair(repoTags []string, imageName string) (repoTagPairs [][ const none = "" if len(repoTags) == 0 { repoTagPairs = append(repoTagPairs, []string{imageName, none}) + return } + for _, repoTag := range repoTags { idx := strings.LastIndex(repoTag, ":") if idx == -1 { repoTagPairs = append(repoTagPairs, []string{"errorRepoTag", "errorRepoTag"}) + continue } + name := repoTag[:idx] if name == none { name = imageName } + repoTagPairs = append(repoTagPairs, []string{name, repoTag[idx+1:]}) } + return } @@ -645,10 +683,12 @@ func normalizeRepoDigest(repoDigests []string) (repo, digest string) { if len(repoDigests) == 0 { return "", "" } + repoDigestPair := strings.Split(repoDigests[0], "@") if len(repoDigestPair) != 2 { return "errorName", "errorRepoDigest" } + return repoDigestPair[0], repoDigestPair[1] } @@ -674,6 +714,7 @@ func PullImageWithSandbox(client internalapi.ImageManagerService, image string, if timeout > 0 { logrus.Debugf("Using pull context with timeout of %s", timeout) + ctx, cancel = context.WithTimeout(ctx, timeout) defer cancel() } @@ -684,8 +725,10 @@ func PullImageWithSandbox(client internalapi.ImageManagerService, image string, if err != nil { return nil, err } + resp := &pb.PullImageResponse{ImageRef: res} logrus.Debugf("PullImageResponse: %v", resp) + return resp, nil } @@ -694,12 +737,14 @@ func PullImageWithSandbox(client internalapi.ImageManagerService, image string, func ListImages(client internalapi.ImageManagerService, nameFilter string, conditionFilters []string) (*pb.ListImagesResponse, error) { request := &pb.ListImagesRequest{Filter: &pb.ImageFilter{Image: &pb.ImageSpec{Image: nameFilter}}} logrus.Debugf("ListImagesRequest: %v", request) + res, err := InterruptableRPC(nil, func(ctx context.Context) ([]*pb.Image, error) { return client.ListImages(ctx, request.Filter) }) if err != nil { return nil, err } + resp := &pb.ListImagesResponse{Images: res} logrus.Debugf("ListImagesResponse: %v", resp) @@ -719,6 +764,7 @@ func ListImages(client internalapi.ImageManagerService, nameFilter string, condi func filterImagesList(imageList []*pb.Image, filters []string) ([]*pb.Image, error) { filtered := []*pb.Image{} filtered = append(filtered, imageList...) + for _, filter := range filters { switch { case strings.HasPrefix(filter, "before="): @@ -739,19 +785,23 @@ func filterImagesList(imageList []*pb.Image, filters []string) ([]*pb.Image, err return []*pb.Image{}, fmt.Errorf("unknown filter flag: %s", filter) } } + return filtered, nil } func filterByBeforeSince(filterValue string, imageList []*pb.Image) []*pb.Image { filtered := []*pb.Image{} + for _, img := range imageList { // Filter by [:] if strings.Contains(filterValue, ":") && !strings.Contains(filterValue, "@") { imageName, _ := normalizeRepoDigest(img.RepoDigests) + repoTagPairs := normalizeRepoTagPair(img.RepoTags, imageName) if strings.Join(repoTagPairs[0], ":") == filterValue { break } + filtered = append(filtered, img) } // Filter by @@ -759,6 +809,7 @@ func filterByBeforeSince(filterValue string, imageList []*pb.Image) []*pb.Image if strings.HasPrefix(img.Id, filterValue) { break } + filtered = append(filtered, img) } // Filter by @@ -767,6 +818,7 @@ func filterByBeforeSince(filterValue string, imageList []*pb.Image) []*pb.Image if strings.HasPrefix(img.RepoDigests[0], filterValue) { break } + filtered = append(filtered, img) } } @@ -777,10 +829,12 @@ func filterByBeforeSince(filterValue string, imageList []*pb.Image) []*pb.Image func filterByReference(filterValue string, imageList []*pb.Image) ([]*pb.Image, error) { filtered := []*pb.Image{} + re, err := regexp.Compile(filterValue) if err != nil { return filtered, err } + for _, img := range imageList { imgName, _ := normalizeRepoDigest(img.RepoDigests) if re.MatchString(imgName) || imgName == filterValue { @@ -793,10 +847,12 @@ func filterByReference(filterValue string, imageList []*pb.Image) ([]*pb.Image, func filterByDangling(filterValue string, imageList []*pb.Image) []*pb.Image { filtered := []*pb.Image{} + for _, img := range imageList { if filterValue == "true" && len(img.RepoTags) == 0 { filtered = append(filtered, img) } + if filterValue == "false" && len(img.RepoTags) > 0 { filtered = append(filtered, img) } @@ -813,13 +869,16 @@ func ImageStatus(client internalapi.ImageManagerService, image string, verbose b Verbose: verbose, } logrus.Debugf("ImageStatusRequest: %v", request) + res, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.ImageStatusResponse, error) { return client.ImageStatus(ctx, request.Image, request.Verbose) }) if err != nil { return nil, err } + logrus.Debugf("ImageStatusResponse: %v", res) + return res, nil } @@ -829,11 +888,14 @@ func RemoveImage(client internalapi.ImageManagerService, image string) error { if image == "" { return errors.New("ImageID cannot be empty") } + request := &pb.RemoveImageRequest{Image: &pb.ImageSpec{Image: image}} logrus.Debugf("RemoveImageRequest: %v", request) + _, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.RemoveImageResponse, error) { return nil, client.RemoveImage(ctx, request.Image) }) + return err } @@ -846,10 +908,12 @@ func ImageFsInfo(client internalapi.ImageManagerService) (*pb.ImageFsInfoRespons if err != nil { return nil, err } + resp := &pb.ImageFsInfoResponse{ ImageFilesystems: res.GetImageFilesystems(), ContainerFilesystems: res.GetContainerFilesystems(), } logrus.Debugf("ImageFsInfoResponse: %v", resp) + return resp, nil } diff --git a/cmd/crictl/image_test.go b/cmd/crictl/image_test.go index f6704abb01..e78b0ba2d2 100644 --- a/cmd/crictl/image_test.go +++ b/cmd/crictl/image_test.go @@ -29,9 +29,11 @@ func fakeImage(id string, digest, tags []string) *pb.Image { func assert(input []*pb.Image, options, images []string) { actual, _ := filterImagesList(input, options) expected := []string{} + for _, img := range actual { expected = append(expected, img.Id) } + Expect(images).To(Equal(expected)) } diff --git a/cmd/crictl/info.go b/cmd/crictl/info.go index 38066a11e3..1864a04a44 100644 --- a/cmd/crictl/info.go +++ b/cmd/crictl/info.go @@ -62,6 +62,7 @@ var runtimeStatusCommand = &cli.Command{ if err = Info(c, runtimeClient); err != nil { return fmt.Errorf("getting status of runtime: %w", err) } + return nil }, } @@ -70,10 +71,12 @@ var runtimeStatusCommand = &cli.Command{ func Info(cliContext *cli.Context, client internalapi.RuntimeService) error { request := &pb.StatusRequest{Verbose: !cliContext.Bool("quiet")} logrus.Debugf("StatusRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.StatusResponse, error) { return client.Status(ctx, request.Verbose) }) logrus.Debugf("StatusResponse: %v", r) + if err != nil { return err } @@ -82,10 +85,13 @@ func Info(cliContext *cli.Context, client internalapi.RuntimeService) error { if err != nil { return fmt.Errorf("create status JSON: %w", err) } + handlers, err := json.Marshal(r.RuntimeHandlers) // protobufObjectToJSON cannot be used if err != nil { return err } + data := []statusData{{json: statusJSON, runtimeHandlers: string(handlers), info: r.Info}} + return outputStatusData(data, cliContext.String("output"), cliContext.String("template")) } diff --git a/cmd/crictl/logs.go b/cmd/crictl/logs.go index 77f4db55f5..54a5260458 100644 --- a/cmd/crictl/logs.go +++ b/cmd/crictl/logs.go @@ -197,14 +197,18 @@ func parseTimestamp(value string) (*metav1.Time, error) { if value == "" { return nil, nil } + str, err := timetypes.GetTimestamp(value, time.Now()) if err != nil { return nil, err } + s, ns, err := timetypes.ParseTimestamps(str, 0) if err != nil { return nil, err } + t := metav1.NewTime(time.Unix(s, ns)) + return &t, nil } diff --git a/cmd/crictl/main.go b/cmd/crictl/main.go index 7002f8893b..1374125767 100644 --- a/cmd/crictl/main.go +++ b/cmd/crictl/main.go @@ -74,6 +74,7 @@ func getRuntimeService(_ *cli.Context, timeout time.Duration) (res internalapi.R if RuntimeEndpointIsSet && RuntimeEndpoint == "" { return nil, errors.New("--runtime-endpoint is not set") } + logrus.Debug("Get runtime connection") // Check if a custom timeout is provided. @@ -81,6 +82,7 @@ func getRuntimeService(_ *cli.Context, timeout time.Duration) (res internalapi.R if timeout != 0 { t = timeout } + logrus.Debugf("Using runtime connection timeout: %v", t) // Use the noop tracer provider and not tracerProvider directly, otherwise @@ -107,14 +109,18 @@ func getRuntimeService(_ *cli.Context, timeout time.Duration) (res internalapi.R res, err = remote.NewRemoteRuntimeService(endPoint, t, tp, &logger) if err != nil { logrus.Error(err) + continue } logrus.Debugf("Connected successfully using endpoint: %s", endPoint) + break } + return res, err } + return remote.NewRemoteRuntimeService(RuntimeEndpoint, t, tp, &logger) } @@ -123,6 +129,7 @@ func getImageService(*cli.Context) (res internalapi.ImageManagerService, err err if RuntimeEndpointIsSet && RuntimeEndpoint == "" { return nil, errors.New("--image-endpoint is not set") } + ImageEndpoint = RuntimeEndpoint ImageEndpointIsSet = RuntimeEndpointIsSet } @@ -153,14 +160,18 @@ func getImageService(*cli.Context) (res internalapi.ImageManagerService, err err res, err = remote.NewRemoteImageService(endPoint, Timeout, tp, &logger) if err != nil { logrus.Error(err) + continue } logrus.Debugf("Connected successfully using endpoint: %s", endPoint) + break } + return res, err } + return remote.NewRemoteImageService(ImageEndpoint, Timeout, tp, &logger) } @@ -292,6 +303,7 @@ func main() { app.Before = func(context *cli.Context) (err error) { var config *common.ServerConfiguration + var exePath string cpuProfilePath := context.String("profile-cpu") @@ -316,6 +328,7 @@ func main() { if exePath, err = os.Executable(); err != nil { logrus.Fatal(err) } + if config, err = common.GetServerConfigFromFile(context.String("config"), exePath); err != nil { if context.IsSet("config") { logrus.Fatal(err) @@ -324,18 +337,23 @@ func main() { if config == nil { RuntimeEndpoint = context.String("runtime-endpoint") + if context.IsSet("runtime-endpoint") { RuntimeEndpointIsSet = true } + ImageEndpoint = context.String("image-endpoint") + if context.IsSet("image-endpoint") { ImageEndpointIsSet = true } + if context.IsSet("timeout") { Timeout = getTimeout(context.Duration("timeout")) } else { Timeout = context.Duration("timeout") } + Debug = context.Bool("debug") DisablePullOnRun = false } else { @@ -349,6 +367,7 @@ func main() { } else { RuntimeEndpoint = context.String("runtime-endpoint") } + if context.IsSet("image-endpoint") { //nolint:gocritic ImageEndpoint = context.String("image-endpoint") ImageEndpointIsSet = true @@ -358,6 +377,7 @@ func main() { } else { ImageEndpoint = context.String("image-endpoint") } + if context.IsSet("timeout") { //nolint:gocritic Timeout = getTimeout(context.Duration("timeout")) } else if config.Timeout > 0 { // 0/neg value set to default timeout @@ -365,11 +385,13 @@ func main() { } else { Timeout = context.Duration("timeout") } + if context.IsSet("debug") { Debug = context.Bool("debug") } else { Debug = config.Debug } + PullImageOnCreate = config.PullImageOnCreate DisablePullOnRun = config.DisablePullOnRun } @@ -424,6 +446,7 @@ func main() { for _, cmd := range app.Commands { sort.Sort(cli.FlagsByName(cmd.Flags)) } + sort.Sort(cli.FlagsByName(app.Flags)) if err := app.Run(os.Args); err != nil { @@ -434,6 +457,7 @@ func main() { if tracerProvider != nil { ctx, cancel := context.WithTimeout(context.Background(), Timeout) defer cancel() + if err := tracerProvider.Shutdown(ctx); err != nil { logrus.Errorf("Unable to shutdown tracer provider: %v", err) } diff --git a/cmd/crictl/pod_metrics.go b/cmd/crictl/pod_metrics.go index ddaf6618e8..c82512f54a 100644 --- a/cmd/crictl/pod_metrics.go +++ b/cmd/crictl/pod_metrics.go @@ -85,6 +85,7 @@ func podMetrics( opts podMetricsOptions, ) error { d := podMetricsDisplayer{opts} + return handleDisplay(c, client, opts.watch, d.displayPodMetrics) } @@ -102,12 +103,14 @@ func (p *podMetricsDisplayer) displayPodMetrics( } response := &pb.ListPodSandboxMetricsResponse{PodMetrics: metrics} + switch p.opts.output { case outputTypeJSON, "": return outputProtobufObjAsJSON(response) case outputTypeYAML: return outputProtobufObjAsYAML(response) } + return nil } @@ -118,6 +121,7 @@ func podSandboxMetrics(ctx context.Context, client cri.RuntimeService) ([]*pb.Po if err != nil { return nil, fmt.Errorf("list pod sandbox metrics: %w", err) } + logrus.Debugf("PodMetrics: %v", metrics) return metrics, nil diff --git a/cmd/crictl/pod_stats.go b/cmd/crictl/pod_stats.go index 08a122450a..5c52ba0d4d 100644 --- a/cmd/crictl/pod_stats.go +++ b/cmd/crictl/pod_stats.go @@ -134,6 +134,7 @@ func podStats( if opts.id != "" { filter.Id = opts.id } + if opts.labels != nil { filter.LabelSelector = opts.labels } @@ -143,6 +144,7 @@ func podStats( opts: opts, display: newDefaultTableDisplay(), } + return handleDisplay(c, client, opts.watch, d.displayPodStats) } @@ -156,6 +158,7 @@ func (d *podStatsDisplayer) displayPodStats( } response := &pb.ListPodSandboxStatsResponse{Stats: stats} + switch d.opts.output { case outputTypeJSON: return outputProtobufObjAsJSON(response) @@ -164,10 +167,12 @@ func (d *podStatsDisplayer) displayPodStats( } oldStats := make(map[string]*pb.PodSandboxStats) + for _, s := range stats { if c.Err() != nil { return c.Err() } + oldStats[s.Attributes.Id] = s } @@ -179,13 +184,16 @@ func (d *podStatsDisplayer) displayPodStats( } d.display.AddRow([]string{columnPodName, columnPodID, columnCPU, columnMemory}) + for _, s := range stats { if c.Err() != nil { return c.Err() } + id := getTruncatedID(s.Attributes.Id, "") var cpu, mem uint64 + var ts int64 linux := s.GetLinux() @@ -214,6 +222,7 @@ func (d *podStatsDisplayer) displayPodStats( oldCPU uint64 oldCPUTs int64 ) + old, ok := oldStats[s.Attributes.Id] if !ok { // Skip new pod @@ -222,6 +231,7 @@ func (d *podStatsDisplayer) displayPodStats( oldLinux := old.GetLinux() oldWindows := old.GetWindows() + if linux != nil { oldCPUTs = oldLinux.GetCpu().GetTimestamp() oldCPU = oldLinux.GetCpu().GetUsageCoreNanoSeconds().GetValue() @@ -231,14 +241,17 @@ func (d *podStatsDisplayer) displayPodStats( } var cpuPerc float64 + if cpu != 0 { // Only generate cpuPerc for running sandbox duration := ts - oldCPUTs if duration == 0 { return errors.New("cpu stat is not updated during sample") } + cpuPerc = float64(cpu-oldCPU) / float64(duration) * 100 } + d.display.AddRow([]string{ s.Attributes.GetMetadata().GetName(), id, @@ -246,6 +259,7 @@ func (d *podStatsDisplayer) displayPodStats( units.HumanSize(float64(mem)), }) } + d.display.ClearScreen() d.display.Flush() @@ -265,6 +279,7 @@ func getPodSandboxStats( if err != nil { return nil, fmt.Errorf("list pod sandbox stats: %w", err) } + logrus.Debugf("Stats: %v", stats) sort.Sort(podStatsByID(stats)) diff --git a/cmd/crictl/portforward.go b/cmd/crictl/portforward.go index 53e465116d..b09e8462ad 100644 --- a/cmd/crictl/portforward.go +++ b/cmd/crictl/portforward.go @@ -88,6 +88,7 @@ var runtimePortForwardCommand = &cli.Command{ if err = PortForward(runtimeClient, opts); err != nil { return fmt.Errorf("port forward: %w", err) } + return nil }, } @@ -97,14 +98,17 @@ func PortForward(client internalapi.RuntimeService, opts portforwardOptions) err if opts.id == "" { return errors.New("ID cannot be empty") } + request := &pb.PortForwardRequest{ PodSandboxId: opts.id, } logrus.Debugf("PortForwardRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.PortForwardResponse, error) { return client.PortForward(ctx, request) }) logrus.Debugf("PortForwardResponse; %v", r) + if err != nil { return err } @@ -123,6 +127,7 @@ func PortForward(client internalapi.RuntimeService, opts portforwardOptions) err } logrus.Debugf("PortForward URL: %v", parsedURL) + dialer, err := getDialer(opts.transport, parsedURL, opts.tlsConfig) if err != nil { return fmt.Errorf("get dialer: %w", err) @@ -131,10 +136,12 @@ func PortForward(client internalapi.RuntimeService, opts portforwardOptions) err readyChan := make(chan struct{}) logrus.Debugf("Ports to forward: %v", opts.ports) + pf, err := portforward.New(dialer, opts.ports, SetupInterruptSignalHandler(), readyChan, os.Stdout, os.Stderr) if err != nil { return err } + return pf.ForwardPorts() } @@ -147,6 +154,7 @@ func getDialer(transport string, parsedURL *url.URL, tlsConfig *rest.TLSClientCo if err != nil { return nil, fmt.Errorf("get SPDY round tripper: %w", err) } + return spdy.NewDialer(upgrader, &http.Client{Transport: tr}, "POST", parsedURL), nil case transportWebsocket: diff --git a/cmd/crictl/sandbox.go b/cmd/crictl/sandbox.go index cdc87dbfb2..f25da452e5 100644 --- a/cmd/crictl/sandbox.go +++ b/cmd/crictl/sandbox.go @@ -93,6 +93,7 @@ var runPodCommand = &cli.Command{ return fmt.Errorf("run pod sandbox: %w", err) } fmt.Println(podID) + return nil }, } @@ -116,6 +117,7 @@ var stopPodCommand = &cli.Command{ return fmt.Errorf("stopping the pod sandbox %q: %w", id, err) } } + return nil }, } @@ -160,8 +162,10 @@ var removePodCommand = &cli.Command{ if len(ids) == 0 { if ctx.Bool("all") { logrus.Info("No pods to remove") + return nil } + return cli.ShowSubcommandHelp(ctx) } @@ -276,6 +280,7 @@ var podStatusCommand = &cli.Command{ if len(ids) == 0 { logrus.Error("No IDs provided or nothing found per filter") + return cli.ShowSubcommandHelp(c) } @@ -376,6 +381,7 @@ var listPodCommand = &cli.Command{ if err = OutputPodSandboxes(runtimeClient, opts); err != nil { return fmt.Errorf("listing pod sandboxes: %w", err) } + return nil }, } @@ -388,13 +394,16 @@ func RunPodSandbox(client internalapi.RuntimeService, config *pb.PodSandboxConfi RuntimeHandler: runtime, } logrus.Debugf("RunPodSandboxRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (string, error) { return client.RunPodSandbox(ctx, config, runtime) }) logrus.Debugf("RunPodSandboxResponse: %v", r) + if err != nil { return "", err } + return r, nil } @@ -404,7 +413,9 @@ func StopPodSandbox(client internalapi.RuntimeService, id string) error { if id == "" { return errors.New("ID cannot be empty") } + logrus.Debugf("Stopping pod sandbox: %s", id) + if _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.StopPodSandbox(ctx, id) }); err != nil { @@ -412,6 +423,7 @@ func StopPodSandbox(client internalapi.RuntimeService, id string) error { } fmt.Printf("Stopped sandbox %s\n", id) + return nil } @@ -421,13 +433,17 @@ func RemovePodSandbox(client internalapi.RuntimeService, id string) error { if id == "" { return errors.New("ID cannot be empty") } + logrus.Debugf("Removing pod sandbox: %s", id) + if _, err := InterruptableRPC(nil, func(ctx context.Context) (any, error) { return nil, client.RemovePodSandbox(ctx, id) }); err != nil { return err } + fmt.Printf("Removed sandbox %s\n", id) + return nil } @@ -438,12 +454,16 @@ func marshalPodSandboxStatus(ps *pb.PodSandboxStatus) (string, error) { if err != nil { return "", err } + jsonMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(statusStr), &jsonMap) if err != nil { return "", err } + jsonMap["createdAt"] = time.Unix(0, ps.CreatedAt).Format(time.RFC3339Nano) + return marshalMapInOrder(jsonMap, *ps) } @@ -453,25 +473,30 @@ func marshalPodSandboxStatus(ps *pb.PodSandboxStatus) (string, error) { //nolint:dupl // pods and containers are similar, but still different func podSandboxStatus(client internalapi.RuntimeService, ids []string, output string, quiet bool, tmplStr string) error { verbose := !(quiet) + if output == "" { // default to json output output = outputTypeJSON } + if len(ids) == 0 { return errors.New("ID cannot be empty") } statuses := []statusData{} + for _, id := range ids { request := &pb.PodSandboxStatusRequest{ PodSandboxId: id, Verbose: verbose, } logrus.Debugf("PodSandboxStatusRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.PodSandboxStatusResponse, error) { return client.PodSandboxStatus(ctx, id, verbose) }) logrus.Debugf("PodSandboxStatusResponse: %v", r) + if err != nil { return fmt.Errorf("get pod sandbox status: %w", err) } @@ -494,40 +519,51 @@ func podSandboxStatus(client internalapi.RuntimeService, ids []string, output st func outputPodSandboxStatusTable(r *pb.PodSandboxStatusResponse, verbose bool) { // output in table format by default. fmt.Printf("ID: %s\n", r.Status.Id) + if r.Status.Metadata != nil { if r.Status.Metadata.Name != "" { fmt.Printf("Name: %s\n", r.Status.Metadata.Name) } + if r.Status.Metadata.Uid != "" { fmt.Printf("UID: %s\n", r.Status.Metadata.Uid) } + if r.Status.Metadata.Namespace != "" { fmt.Printf("Namespace: %s\n", r.Status.Metadata.Namespace) } + fmt.Printf("Attempt: %v\n", r.Status.Metadata.Attempt) } + fmt.Printf("Status: %s\n", r.Status.State) ctm := time.Unix(0, r.Status.CreatedAt) fmt.Printf("Created: %v\n", ctm) if r.Status.Network != nil { fmt.Printf("IP Addresses: %v\n", r.Status.Network.Ip) + for _, ip := range r.Status.Network.AdditionalIps { fmt.Printf("Additional IP: %v\n", ip.Ip) } } + if r.Status.Labels != nil { fmt.Println("Labels:") + for _, k := range getSortedKeys(r.Status.Labels) { fmt.Printf("\t%s -> %s\n", k, r.Status.Labels[k]) } } + if r.Status.Annotations != nil { fmt.Println("Annotations:") + for _, k := range getSortedKeys(r.Status.Annotations) { fmt.Printf("\t%s -> %s\n", k, r.Status.Annotations[k]) } } + if verbose { fmt.Printf("Info: %v\n", r.GetInfo()) } @@ -540,9 +576,11 @@ func ListPodSandboxes(client internalapi.RuntimeService, opts *listOptions) ([]* if opts.id != "" { filter.Id = opts.id } + if opts.state != "" { st := &pb.PodSandboxStateValue{} st.State = pb.PodSandboxState_SANDBOX_NOTREADY + switch strings.ToLower(opts.state) { case "ready": st.State = pb.PodSandboxState_SANDBOX_READY @@ -554,20 +592,25 @@ func ListPodSandboxes(client internalapi.RuntimeService, opts *listOptions) ([]* log.Fatalf("--state should be ready or notready") } } + if opts.labels != nil { filter.LabelSelector = opts.labels } + request := &pb.ListPodSandboxRequest{ Filter: filter, } logrus.Debugf("ListPodSandboxRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) ([]*pb.PodSandbox, error) { return client.ListPodSandbox(ctx, filter) }) logrus.Debugf("ListPodSandboxResponse: %v", r) + if err != nil { return nil, fmt.Errorf("call list sandboxes RPC: %w", err) } + return getSandboxesList(r, opts), nil } @@ -602,19 +645,25 @@ func OutputPodSandboxes(client internalapi.RuntimeService, opts *listOptions) er columnPodRuntime, }) } + c := cases.Title(language.Und) + for _, pod := range r { if opts.quiet { fmt.Printf("%s\n", pod.Id) + continue } + if !opts.verbose { createdAt := time.Unix(0, pod.CreatedAt) ctm := units.HumanDuration(time.Now().UTC().Sub(createdAt)) + " ago" + id := pod.Id if !opts.noTrunc { id = getTruncatedID(id, "") } + display.AddRow([]string{ id, ctm, @@ -624,39 +673,50 @@ func OutputPodSandboxes(client internalapi.RuntimeService, opts *listOptions) er strconv.FormatUint(uint64(pod.Metadata.Attempt), 10), getSandboxesRuntimeHandler(pod), }) + continue } fmt.Printf("ID: %s\n", pod.Id) + if pod.Metadata != nil { if pod.Metadata.Name != "" { fmt.Printf("Name: %s\n", pod.Metadata.Name) } + if pod.Metadata.Uid != "" { fmt.Printf("UID: %s\n", pod.Metadata.Uid) } + if pod.Metadata.Namespace != "" { fmt.Printf("Namespace: %s\n", pod.Metadata.Namespace) } + if pod.Metadata.Attempt != 0 { fmt.Printf("Attempt: %v\n", pod.Metadata.Attempt) } } + fmt.Printf("Status: %s\n", convertPodState(pod.State)) ctm := time.Unix(0, pod.CreatedAt) fmt.Printf("Created: %v\n", ctm) + if pod.Labels != nil { fmt.Println("Labels:") + for _, k := range getSortedKeys(pod.Labels) { fmt.Printf("\t%s -> %s\n", k, pod.Labels[k]) } } + if pod.Annotations != nil { fmt.Println("Annotations:") + for _, k := range getSortedKeys(pod.Annotations) { fmt.Printf("\t%s -> %s\n", k, pod.Annotations[k]) } } + fmt.Printf("%s: %s\n", c.String(columnPodRuntime), getSandboxesRuntimeHandler(pod)) @@ -665,6 +725,7 @@ func OutputPodSandboxes(client internalapi.RuntimeService, opts *listOptions) er } display.Flush() + return nil } @@ -676,6 +737,7 @@ func convertPodState(state pb.PodSandboxState) string { return "NotReady" default: log.Fatalf("Unknown pod state %q", state) + return "" } } @@ -684,11 +746,13 @@ func getSandboxesRuntimeHandler(sandbox *pb.PodSandbox) string { if sandbox.RuntimeHandler == "" { return "(default)" } + return sandbox.RuntimeHandler } func getSandboxesList(sandboxesList []*pb.PodSandbox, opts *listOptions) []*pb.PodSandbox { filtered := []*pb.PodSandbox{} + for _, p := range sandboxesList { // Filter by pod name/namespace regular expressions. if p.Metadata != nil && matchesRegex(opts.nameRegexp, p.Metadata.Name) && @@ -698,17 +762,21 @@ func getSandboxesList(sandboxesList []*pb.PodSandbox, opts *listOptions) []*pb.P } sort.Sort(sandboxByCreated(filtered)) + n := len(filtered) if opts.latest { n = 1 } + if opts.last > 0 { n = opts.last } + n = func(a, b int) int { if a < b { return a } + return b }(n, len(filtered)) diff --git a/cmd/crictl/templates.go b/cmd/crictl/templates.go index 0631c5236f..4f4985341f 100644 --- a/cmd/crictl/templates.go +++ b/cmd/crictl/templates.go @@ -31,6 +31,7 @@ func builtinTmplFuncs() template.FuncMap { t := cases.Title(language.Und, cases.NoLower) l := cases.Lower(language.Und) u := cases.Upper(language.Und) + return template.FuncMap{ outputTypeJSON: jsonBuiltinTmplFunc, "title": t.String, @@ -42,10 +43,12 @@ func builtinTmplFuncs() template.FuncMap { // jsonBuiltinTmplFunc allows to jsonify result of template execution. func jsonBuiltinTmplFunc(v interface{}) string { o := new(bytes.Buffer) + enc := json.NewEncoder(o) if err := enc.Encode(v); err != nil { logrus.Fatalf("Unable to encode JSON: %v", err) } + return o.String() } @@ -63,6 +66,7 @@ func tmplExecuteRawJSON(tmplStr, rawJSON string) (string, error) { } o := new(bytes.Buffer) + tmpl, err := template.New("tmplExecuteRawJSON").Funcs(builtinTmplFuncs()).Parse(tmplStr) if err != nil { return "", fmt.Errorf("failed to generate go-template: %w", err) @@ -73,10 +77,12 @@ func tmplExecuteRawJSON(tmplStr, rawJSON string) (string, error) { if err := tmpl.Execute(o, raw); err != nil { return "", fmt.Errorf("failed to template data: %w", err) } + return o.String(), nil } func validateTemplate(tmplStr string) error { _, err := template.New("").Funcs(builtinTmplFuncs()).Parse(tmplStr) + return err } diff --git a/cmd/crictl/templates_test.go b/cmd/crictl/templates_test.go index 92756be4f7..6763ce88df 100644 --- a/cmd/crictl/templates_test.go +++ b/cmd/crictl/templates_test.go @@ -22,6 +22,7 @@ import ( func TestTmplExecuteRawJSON(t *testing.T) { t.Parallel() + testcases := []struct { rawJSON string tmplStr string diff --git a/cmd/crictl/update_runtime_config.go b/cmd/crictl/update_runtime_config.go index f0e9f07288..4f4dc3455f 100644 --- a/cmd/crictl/update_runtime_config.go +++ b/cmd/crictl/update_runtime_config.go @@ -62,6 +62,7 @@ var updateRuntimeConfigCommand = &cli.Command{ } logrus.Info("Runtime config successfully updated") + return nil }, } diff --git a/cmd/crictl/util.go b/cmd/crictl/util.go index 5802c899a1..69b2ace267 100644 --- a/cmd/crictl/util.go +++ b/cmd/crictl/util.go @@ -68,6 +68,7 @@ func SetupInterruptSignalHandler() <-chan struct{} { signalIntStopCh = make(chan struct{}) c := make(chan os.Signal, 2) signal.Notify(c, shutdownSignals...) + go func() { <-c close(signalIntStopCh) @@ -75,6 +76,7 @@ func SetupInterruptSignalHandler() <-chan struct{} { os.Exit(1) // Exit immediately on second signal }() }) + return signalIntStopCh } @@ -86,6 +88,7 @@ func InterruptableRPC[T any]( //nolint:contextcheck // creating a new context is intentional ctx = context.Background() } + ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -96,6 +99,7 @@ func InterruptableRPC[T any]( res, err := rpcFunc(ctx) if err != nil { errCh <- err + return } resCh <- res @@ -104,6 +108,7 @@ func InterruptableRPC[T any]( select { case <-SetupInterruptSignalHandler(): cancel() + return res, fmt.Errorf("interrupted: %w", ctx.Err()) case err := <-errCh: return res, err @@ -191,6 +196,7 @@ func getSortedKeys(m map[string]string) []string { for k := range m { keys = append(keys, k) } + sort.Strings(keys) return keys @@ -221,11 +227,14 @@ func loadContainerConfig(path string) (*pb.ContainerConfig, error) { func printJSONSchema(value any) error { schema := jsonschema.Reflect(value) + data, err := json.MarshalIndent(schema, "", " ") if err != nil { return fmt.Errorf("marshal JSON schema: %w", err) } + fmt.Println(string(data)) + return nil } @@ -258,17 +267,21 @@ func openFile(path string) (*os.File, error) { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("config at %s not found", path) } + return nil, err } + return f, nil } func protobufObjectToJSON(obj protoiface.MessageV1) (string, error) { msg := protoadapt.MessageV2Of(obj) + marshaledJSON, err := protojson.MarshalOptions{EmitDefaultValues: true, Indent: " "}.Marshal(msg) if err != nil { return "", err } + return string(marshaledJSON), nil } @@ -279,6 +292,7 @@ func outputProtobufObjAsJSON(obj protoiface.MessageV1) error { } fmt.Println(marshaledJSON) + return nil } @@ -287,12 +301,14 @@ func outputProtobufObjAsYAML(obj protoiface.MessageV1) error { if err != nil { return err } + marshaledYAML, err := yaml.JSONToYAML([]byte(marshaledJSON)) if err != nil { return err } fmt.Println(string(marshaledYAML)) + return nil } @@ -308,30 +324,37 @@ func outputStatusData(statuses []statusData, format, tmplStr string) (err error) } result := []map[string]any{} + for _, status := range statuses { // Sort all keys keys := []string{} for k := range status.info { keys = append(keys, k) } + sort.Strings(keys) + infoMap := map[string]any{} if status.json != "" { var statusVal map[string]any + err := json.Unmarshal([]byte(status.json), &statusVal) if err != nil { return fmt.Errorf("unmarshal status JSON: %w", err) } + infoMap["status"] = statusVal } if status.runtimeHandlers != "" { var handlersVal []*any + err := json.Unmarshal([]byte(status.runtimeHandlers), &handlersVal) if err != nil { return fmt.Errorf("unmarshal runtime handlers: %w", err) } + if handlersVal != nil { infoMap["runtimeHandlers"] = handlersVal } @@ -346,6 +369,7 @@ func outputStatusData(statuses []statusData, format, tmplStr string) (err error) if err := json.Unmarshal([]byte(val), &genericVal); err != nil { return fmt.Errorf("unmarshal status info JSON: %w", err) } + infoMap[k] = genericVal } else { // Assume a string and remove any double quotes @@ -374,18 +398,21 @@ func outputStatusData(statuses []statusData, format, tmplStr string) (err error) if err != nil { return fmt.Errorf("JSON result to YAML: %w", err) } + fmt.Println(string(yamlInfo)) case outputTypeJSON: var output bytes.Buffer if err := json.Indent(&output, jsonResult, "", " "); err != nil { return fmt.Errorf("indent JSON result: %w", err) } + fmt.Println(output.String()) case outputTypeGoTemplate: output, err := tmplExecuteRawJSON(tmplStr, string(jsonResult)) if err != nil { return fmt.Errorf("execute template: %w", err) } + fmt.Println(output) default: return fmt.Errorf("unsupported format: %q", format) @@ -411,26 +438,32 @@ func outputEvent(event protoiface.MessageV1, format, tmplStr string) error { if err != nil { return err } + output, err := tmplExecuteRawJSON(tmplStr, jsonEvent) if err != nil { return err } + fmt.Println(output) default: fmt.Printf("Don't support %q format\n", format) } + return nil } func parseLabelStringSlice(ss []string) (map[string]string, error) { labels := make(map[string]string) + for _, s := range ss { pair := strings.Split(s, "=") if len(pair) != 2 { return nil, fmt.Errorf("incorrectly specified label: %v", s) } + labels[pair[0]] = pair[1] } + return labels, nil } @@ -438,36 +471,46 @@ func parseLabelStringSlice(ss []string) (map[string]string, error) { // data structure. func marshalMapInOrder(m map[string]interface{}, t interface{}) (string, error) { s := "{" + v := reflect.ValueOf(t) for i := range v.Type().NumField() { field := jsonFieldFromTag(v.Type().Field(i).Tag) if field == "" || field == "-" { continue } + value, err := json.Marshal(m[field]) if err != nil { return "", err } + s += fmt.Sprintf("%q:%s,", field, value) } + s = s[:len(s)-1] s += "}" + var buf bytes.Buffer + if err := json.Indent(&buf, []byte(s), "", " "); err != nil { return "", err } + return buf.String(), nil } // jsonFieldFromTag gets json field name from field tag. func jsonFieldFromTag(tag reflect.StructTag) string { field := strings.Split(tag.Get(outputTypeJSON), ",")[0] + for _, f := range strings.Split(tag.Get("protobuf"), ",") { if !strings.HasPrefix(f, "json=") { continue } + field = strings.TrimPrefix(f, "json=") } + return field } @@ -476,6 +519,7 @@ func getTruncatedID(id, prefix string) string { if len(id) > truncatedIDLen { id = id[:truncatedIDLen] } + return id } @@ -483,11 +527,13 @@ func matchesRegex(pattern, target string) bool { if pattern == "" { return true } + matched, err := regexp.MatchString(pattern, target) if err != nil { // Assume it's not a match if an error occurs. return false } + return matched } @@ -495,18 +541,22 @@ func matchesImage(imageClient internalapi.ImageManagerService, image, containerI if image == "" || imageClient == nil { return true, nil } + r1, err := ImageStatus(imageClient, image, false) if err != nil { return false, err } + r2, err := ImageStatus(imageClient, containerImage, false) if err != nil { return false, err } + if r1.Image == nil || r2.Image == nil { // Always return not match if the image doesn't exist. return false, nil } + return r1.Image.Id == r2.Image.Id, nil } @@ -515,9 +565,11 @@ func getRepoImage(imageClient internalapi.ImageManagerService, image string) (st if err != nil { return "", err } + if len(r.Image.RepoTags) > 0 { return r.Image.RepoTags[0], nil } + return image, nil } @@ -532,6 +584,7 @@ func handleDisplay( } displayErrCh := make(chan error, 1) + ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() @@ -545,6 +598,7 @@ func handleDisplay( for range ticker.C { if err := displayFunc(watchCtx, client); err != nil { displayErrCh <- err + break } } @@ -554,6 +608,7 @@ func handleDisplay( select { case <-SetupInterruptSignalHandler(): cancelFn() + return nil case err := <-displayErrCh: return err diff --git a/cmd/crictl/util_test.go b/cmd/crictl/util_test.go index 556fb95c3b..465100b2af 100644 --- a/cmd/crictl/util_test.go +++ b/cmd/crictl/util_test.go @@ -27,6 +27,7 @@ import ( func TestNameFilterByRegex(t *testing.T) { t.Parallel() + testCases := []struct { desc string pattern string @@ -67,6 +68,7 @@ func TestNameFilterByRegex(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { t.Parallel() + r := matchesRegex(tc.pattern, tc.name) if r != tc.isMatch { t.Errorf("expected matched to be %v; actual result is %v", tc.isMatch, r) @@ -103,6 +105,7 @@ func TestOutputStatusData(t *testing.T) { ]` emptyResponse = "" ) + testCases := []struct { name string status string @@ -159,10 +162,12 @@ func TestOutputStatusData(t *testing.T) { t.Run(tc.name, func(t *testing.T) { captureOutput := func(f func() error) (string, error) { var err error + old := os.Stdout r, w, _ := os.Pipe() os.Stdout = w + defer func() { os.Stdout = old }() @@ -178,15 +183,18 @@ func TestOutputStatusData(t *testing.T) { } out, err := io.ReadAll(r) + return strings.TrimRight(string(out), "\n"), err } outStr, err := captureOutput(func() error { data := []statusData{{json: tc.status, runtimeHandlers: tc.handlers, info: tc.info}} + err := outputStatusData(data, tc.format, tc.tmplStr) if err != nil { t.Errorf("Unexpected error: %v", err) } + return nil }) if err != nil { diff --git a/cmd/crictl/version.go b/cmd/crictl/version.go index 89e4a2957c..9a3d409c2b 100644 --- a/cmd/crictl/version.go +++ b/cmd/crictl/version.go @@ -42,6 +42,7 @@ var runtimeVersionCommand = &cli.Command{ if err := Version(runtimeClient, string(remote.CRIVersionV1)); err != nil { return fmt.Errorf("getting the runtime version: %w", err) } + return nil }, } @@ -50,16 +51,20 @@ var runtimeVersionCommand = &cli.Command{ func Version(client internalapi.RuntimeService, version string) error { request := &pb.VersionRequest{Version: version} logrus.Debugf("VersionRequest: %v", request) + r, err := InterruptableRPC(nil, func(ctx context.Context) (*pb.VersionResponse, error) { return client.Version(ctx, version) }) logrus.Debugf("VersionResponse: %v", r) + if err != nil { return err } + fmt.Println("Version: ", r.Version) fmt.Println("RuntimeName: ", r.RuntimeName) fmt.Println("RuntimeVersion: ", r.RuntimeVersion) fmt.Println("RuntimeApiVersion: ", r.RuntimeApiVersion) + return nil } diff --git a/cmd/critest/cri_test.go b/cmd/critest/cri_test.go index 68925cecd5..bd726dd305 100644 --- a/cmd/critest/cri_test.go +++ b/cmd/critest/cri_test.go @@ -69,6 +69,7 @@ func getConfigFromFile() { if !isFlagSet("runtime-endpoint") && configFromFile.RuntimeEndpoint != "" { framework.TestContext.RuntimeServiceAddr = configFromFile.RuntimeEndpoint } + if !isFlagSet("image-endpoint") && configFromFile.ImageEndpoint != "" { framework.TestContext.ImageServiceAddr = configFromFile.ImageEndpoint } @@ -77,11 +78,13 @@ func getConfigFromFile() { func isFlagSet(name string) bool { found := false + flag.Visit(func(f *flag.Flag) { if f.Name == name { found = true } }) + return found } @@ -102,34 +105,41 @@ func generateTempTestName() (string, error) { if err != nil { return "", err } + return filepath.Join(dir, "critest-"+string(suffix)+".test"), nil } func runParallelTestSuite(t *testing.T) { t.Helper() + criPath, err := os.Executable() if err != nil { t.Fatalf("Failed to lookup path of critest: %v", err) } + t.Logf("critest path: %s", criPath) tempFileName, err := generateTempTestName() if err != nil { t.Fatalf("Failed to generate temp test name: %v", err) } + err = os.Symlink(criPath, tempFileName) if err != nil { t.Fatalf("Failed to lookup path of critest: %v", err) } + defer os.Remove(tempFileName) ginkgoArgs, err := generateGinkgoRunFlags() if err != nil { t.Fatalf("Failed to generate ginkgo args: %v", err) } + ginkgoArgs = append(ginkgoArgs, fmt.Sprintf("--nodes=%d", *parallel)) var testArgs []string + flag.Visit(func(f *flag.Flag) { // NOTE(fuweid): // @@ -150,8 +160,10 @@ func runParallelTestSuite(t *testing.T) { if f.Name == parallelFlag || f.Name == benchmarkFlag { return } + testArgs = append(testArgs, fmt.Sprintf("-%s=%s", f.Name, f.Value.String())) }) + var args []string args = append(args, ginkgoArgs...) args = append(args, tempFileName, "--") @@ -160,6 +172,7 @@ func runParallelTestSuite(t *testing.T) { cmd := exec.Command("ginkgo", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + err = cmd.Run() if err != nil { t.Fatalf("Failed to run tests in parallel: %v", err) @@ -179,6 +192,7 @@ func TestCRISuite(t *testing.T) { if err := flag.Set("ginkgo.focus", "benchmark"); err != nil { t.Fatalf("set ginkgo benchmark focus: %v", err) } + if err := flag.Set("ginkgo.succinct", "true"); err != nil { t.Fatalf("set ginkgo succinct: %v", err) } @@ -211,5 +225,6 @@ func generateGinkgoRunFlags() ([]string, error) { "S": &suiteConfig, "R": &reporterConfig, } + return ginkgotypes.GenerateFlagArgs(flags, bindings) } diff --git a/pkg/benchmark/pod_container.go b/pkg/benchmark/pod_container.go index 5c962f1098..a7714657e9 100644 --- a/pkg/benchmark/pod_container.go +++ b/pkg/benchmark/pod_container.go @@ -38,6 +38,7 @@ func getPodContainerBenchmarkTimeoutSeconds() int { if framework.TestContext.BenchmarkingParams.PodContainerStartBenchmarkTimeoutSeconds > 0 { timeout = framework.TestContext.BenchmarkingParams.PodContainerStartBenchmarkTimeoutSeconds } + return timeout } diff --git a/pkg/benchmark/util.go b/pkg/benchmark/util.go index 3121104288..4731990b7e 100644 --- a/pkg/benchmark/util.go +++ b/pkg/benchmark/util.go @@ -94,14 +94,17 @@ func NewLifecycleBenchmarksResultsManager(initialResultsSet LifecycleBenchmarksR if lbrm.resultsSet.Datapoints == nil { lbrm.resultsSet.Datapoints = make([]LifecycleBenchmarkDatapoint, 0) } + return &lbrm } // Function which continuously consumes results from the resultsChannel until receiving a nil. func (lbrm *LifecycleBenchmarksResultsManager) awaitResult() { numOperations := len(lbrm.resultsSet.OperationsNames) + for { var res *LifecycleBenchmarkDatapoint + timeout := time.After(time.Duration(lbrm.resultsChannelTimeoutSeconds) * time.Second) select { @@ -109,8 +112,10 @@ func (lbrm *LifecycleBenchmarksResultsManager) awaitResult() { // Receiving nil indicates results are over: if res == nil { logrus.Info("Results ended") + lbrm.resultsConsumerRunning = false lbrm.resultsOverChannel <- true + return } @@ -138,6 +143,7 @@ func (lbrm *LifecycleBenchmarksResultsManager) StartResultsConsumer() chan *Life lbrm.resultsConsumerRunning = true go lbrm.awaitResult() } + return lbrm.resultsChannel } @@ -151,9 +157,11 @@ func (lbrm *LifecycleBenchmarksResultsManager) AwaitAllResults(timeoutSeconds in select { case <-lbrm.resultsOverChannel: lbrm.resultsConsumerRunning = false + return nil case <-timeout: logrus.Warnf("Failed to await all results. Results registered so far were: %+v", lbrm.resultsSet) + return fmt.Errorf("benchmark results waiting timed out after %d seconds", timeoutSeconds) } } diff --git a/pkg/common/config.go b/pkg/common/config.go index 5cd165df8e..9aa17eb04a 100644 --- a/pkg/common/config.go +++ b/pkg/common/config.go @@ -45,6 +45,7 @@ type ServerConfiguration struct { // GetServerConfigFromFile returns the CRI server configuration from file. func GetServerConfigFromFile(configFileName, currentDir string) (*ServerConfiguration, error) { serverConfig := ServerConfiguration{} + if _, err := os.Stat(configFileName); err != nil { if !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("load config file: %w", err) @@ -54,6 +55,7 @@ func GetServerConfigFromFile(configFileName, currentDir string) (*ServerConfigur // is placed with the cri tools binary. nextConfigFileName := filepath.Join(filepath.Dir(currentDir), "crictl.yaml") logrus.Warnf("Config %q does not exist, trying next: %q", configFileName, nextConfigFileName) + if _, err := os.Stat(nextConfigFileName); err != nil { return nil, fmt.Errorf("load config file: %w", err) } @@ -72,5 +74,6 @@ func GetServerConfigFromFile(configFileName, currentDir string) (*ServerConfigur serverConfig.Debug = config.Debug serverConfig.PullImageOnCreate = config.PullImageOnCreate serverConfig.DisablePullOnRun = config.DisablePullOnRun + return &serverConfig, nil } diff --git a/pkg/common/file.go b/pkg/common/file.go index 8637fc4850..ac8df37a79 100644 --- a/pkg/common/file.go +++ b/pkg/common/file.go @@ -64,15 +64,19 @@ func ReadConfig(filepath string) (*Config, error) { if err != nil { return nil, err } + yamlConfig := &yaml.Node{} + err = yaml.Unmarshal(data, yamlConfig) if err != nil { return nil, err } + config, err := getConfigOptions(yamlConfig) if err != nil { return nil, err } + return config, err } @@ -82,6 +86,7 @@ func WriteConfig(c *Config, filepath string) error { if c == nil { c = &Config{} } + if c.yamlData == nil { c.yamlData = &yaml.Node{} } @@ -96,6 +101,7 @@ func WriteConfig(c *Config, filepath string) error { if err := os.MkdirAll(gofilepath.Dir(filepath), 0o755); err != nil { return err } + return os.WriteFile(filepath, data, 0o644) } @@ -107,6 +113,7 @@ func getConfigOptions(yamlData *yaml.Node) (*Config, error) { yamlData.Content[0].Content == nil { return config, nil } + contentLen := len(yamlData.Content[0].Content) // YAML representation contains 2 yaml ScalarNodes per config option. @@ -117,7 +124,9 @@ func getConfigOptions(yamlData *yaml.Node) (*Config, error) { configOption := yamlData.Content[0].Content[indx] name := configOption.Value value := yamlData.Content[0].Content[indx+1].Value + var err error + switch name { case RuntimeEndpoint: config.RuntimeEndpoint = value @@ -146,6 +155,7 @@ func getConfigOptions(yamlData *yaml.Node) (*Config, error) { default: return nil, fmt.Errorf("Config option '%s' is not valid", name) } + indx += 2 } @@ -172,8 +182,10 @@ func setConfigOption(configName, configValue string, yamlData *yaml.Node) { Tag: "!!map", } } + contentLen := 0 foundOption := false + if yamlData.Content[0].Content != nil { contentLen = len(yamlData.Content[0].Content) } @@ -186,6 +198,7 @@ func setConfigOption(configName, configValue string, yamlData *yaml.Node) { yamlData.Content[0].Content[indx+1].Value = configValue foundOption = true } + indx += 2 } @@ -201,12 +214,15 @@ func setConfigOption(configName, configValue string, yamlData *yaml.Node) { tagBool = tagPrefix + "bool" tagInt = tagPrefix + "int" ) + name := &yaml.Node{ Kind: yaml.ScalarNode, Value: configName, Tag: tagStr, } + var tagType string + switch configName { case Timeout: tagType = tagInt diff --git a/pkg/common/file_test.go b/pkg/common/file_test.go index 1691d50678..75da24227d 100644 --- a/pkg/common/file_test.go +++ b/pkg/common/file_test.go @@ -37,6 +37,7 @@ var _ = DescribeTable("ReadConfig", readConfig, err := common.ReadConfig(f.Name()) if shouldFail { Expect(err).To(HaveOccurred()) + return } else { Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/common/pod_config.go b/pkg/common/pod_config.go index 0c6129c5ae..a3947a2303 100644 --- a/pkg/common/pod_config.go +++ b/pkg/common/pod_config.go @@ -32,12 +32,15 @@ func GetCgroupParent(ctx context.Context, c internalapi.RuntimeService) string { if err != nil { return DefaultSystemdCgroupSlice } + if runtimeConfig == nil || runtimeConfig.Linux == nil { return DefaultSystemdCgroupSlice } + cgroupDriver := runtimeConfig.Linux.GetCgroupDriver() if cgroupDriver == runtimev1.CgroupDriver_CGROUPFS { return "" } + return DefaultSystemdCgroupSlice } diff --git a/pkg/framework/framework.go b/pkg/framework/framework.go index b1131bdc65..84c8790265 100644 --- a/pkg/framework/framework.go +++ b/pkg/framework/framework.go @@ -57,6 +57,7 @@ func (f *Framework) BeforeEach() { if f.CRIClient == nil { c, err := LoadCRIClient() Expect(err).NotTo(HaveOccurred()) + f.CRIClient = c } } diff --git a/pkg/framework/test_context.go b/pkg/framework/test_context.go index 339c0a4253..48b7afb124 100644 --- a/pkg/framework/test_context.go +++ b/pkg/framework/test_context.go @@ -151,10 +151,12 @@ func RegisterFlags() { svcaddr := ContainerdSockPathUnix defaultConfigPath := "/etc/crictl.yaml" + if runtime.GOOS == OSWindows { svcaddr = ContainerdSockPathWindows defaultConfigPath = filepath.Join(os.Getenv("USERPROFILE"), ".crictl", "crictl.yaml") } + flag.StringVar(&TestContext.ConfigPath, "config", defaultConfigPath, "Location of the client config file. If not specified and the default does not exist, the program's directory is searched as well") flag.StringVar(&TestContext.RuntimeServiceAddr, "runtime-endpoint", svcaddr, "Runtime service socket for client to connect.") flag.DurationVar(&TestContext.RuntimeServiceTimeout, "runtime-service-timeout", 300*time.Second, "Timeout when trying to connect to a runtime service.") @@ -168,6 +170,7 @@ func RegisterFlags() { } else { TestContext.IsLcow = false } + flag.StringVar(&TestContext.RegistryPrefix, "registry-prefix", DefaultRegistryPrefix, "A possible registry prefix added to all images, like 'localhost:5000'") } @@ -180,6 +183,7 @@ func (tc *TestContextType) LoadYamlConfigFiles() error { return fmt.Errorf("error loading custom test images file: %w", err) } } + Logf("Testing context container image list: %+v", TestContext.TestImageList) // Attempt to load benchmark settings file: @@ -189,6 +193,7 @@ func (tc *TestContextType) LoadYamlConfigFiles() error { return err } } + Logf("Testing context benchmarking params: %+v", TestContext.BenchmarkingParams) return nil diff --git a/pkg/framework/util.go b/pkg/framework/util.go index 15fd542329..315720c3a2 100644 --- a/pkg/framework/util.go +++ b/pkg/framework/util.go @@ -126,6 +126,7 @@ var _ = BeforeSuite(func() { // AddBeforeSuiteCallback adds a callback to run during BeforeSuite. func AddBeforeSuiteCallback(callback func()) bool { beforeSuiteCallbacks = append(beforeSuiteCallbacks, callback) + return true } @@ -146,6 +147,7 @@ func LoadCRIClient() (*InternalAPIClient, error) { // Fallback to runtime service endpoint imageServiceAddr = TestContext.RuntimeServiceAddr } + iService, err := remote.NewRemoteImageService(imageServiceAddr, TestContext.ImageServiceTimeout, nil, nil) if err != nil { return nil, err @@ -182,6 +184,7 @@ func ExpectNoError(err error, explain ...interface{}) { if err != nil { Logf("Unexpected error occurred: %v", err) } + ExpectWithOffset(1, err).NotTo(HaveOccurred(), explain...) } @@ -202,6 +205,7 @@ func RunDefaultPodSandbox(c internalapi.RuntimeService, prefix string) string { }, Labels: DefaultPodLabels, } + return RunPodSandbox(c, config) } @@ -219,6 +223,7 @@ func BuildPodSandboxMetadata(podSandboxName, uid, namespace string, attempt uint func RunPodSandbox(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string { podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler) ExpectNoError(err, "failed to create PodSandbox: %v", err) + return podID } @@ -226,6 +231,7 @@ func RunPodSandbox(c internalapi.RuntimeService, config *runtimeapi.PodSandboxCo func RunPodSandboxError(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string { podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler) Expect(err).To(HaveOccurred()) + return podID } @@ -243,6 +249,7 @@ func CreatePodSandboxForContainer(c internalapi.RuntimeService) (string, *runtim } podID := RunPodSandbox(c, config) + return podID, config } @@ -292,6 +299,7 @@ func CreateContainerWithError(rc internalapi.RuntimeService, ic internalapi.Imag imageName := config.Image.Image if !strings.Contains(imageName, ":") { imageName += ":latest" + Logf("Use latest as default image tag.") } @@ -305,7 +313,9 @@ func CreateContainerWithError(rc internalapi.RuntimeService, ic internalapi.Imag } By("Create container.") + containerID, err := rc.CreateContainer(context.TODO(), podID, config, podConfig) + return containerID, err } @@ -314,6 +324,7 @@ func CreateContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerS containerID, err := CreateContainerWithError(rc, ic, config, podID, podConfig) ExpectNoError(err, "failed to create container: %v", err) Logf("Created container %q\n", containerID) + return containerID } @@ -325,6 +336,7 @@ func ImageStatus(c internalapi.ImageManagerService, imageName string) *runtimeap } status, err := c.ImageStatus(context.TODO(), imageSpec, false) ExpectNoError(err, "failed to get image status: %v", err) + return status.GetImage() } @@ -332,6 +344,7 @@ func ImageStatus(c internalapi.ImageManagerService, imageName string) *runtimeap func ListImage(c internalapi.ImageManagerService, filter *runtimeapi.ImageFilter) []*runtimeapi.Image { images, err := c.ListImages(context.TODO(), filter) ExpectNoError(err, "Failed to get image list: %v", err) + return images } @@ -353,10 +366,12 @@ func PrepareImageName(imageName string) string { ref, err = reference.ParseNamed(r) ExpectNoError(err, "failed to parse new image name: %v", err) } + imageName = ref.String() if !strings.Contains(imageName, ":") { imageName += ":latest" + Logf("Use latest as default image tag.") } @@ -373,12 +388,14 @@ func PullPublicImage(c internalapi.ImageManagerService, imageName string, podCon } id, err := c.PullImage(context.TODO(), imageSpec, nil, podConfig) ExpectNoError(err, "failed to pull image: %v", err) + return id } // LoadYamlFile attempts to load the given YAML file into the given struct. func LoadYamlFile(filepath string, obj interface{}) error { Logf("Attempting to load YAML file %q into %+v", filepath, obj) + fileContent, err := os.ReadFile(filepath) if err != nil { return fmt.Errorf("error reading %q file contents: %w", filepath, err) @@ -390,5 +407,6 @@ func LoadYamlFile(filepath string, obj interface{}) error { } Logf("Successfully loaded YAML file %q into %+v", filepath, obj) + return nil } diff --git a/pkg/tracing/tracing.go b/pkg/tracing/tracing.go index 47f61da6db..e1c7799b0e 100644 --- a/pkg/tracing/tracing.go +++ b/pkg/tracing/tracing.go @@ -72,6 +72,7 @@ func Init(ctx context.Context, collectorAddress string, samplingRate int) (*trac ) tmp := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) + otel.SetTracerProvider(tp) otel.SetTextMapPropagator(tmp) diff --git a/pkg/validate/apparmor_linux.go b/pkg/validate/apparmor_linux.go index ad7436ae05..3c1c415a3d 100644 --- a/pkg/validate/apparmor_linux.go +++ b/pkg/validate/apparmor_linux.go @@ -71,6 +71,7 @@ var _ = framework.KubeDescribe("AppArmor", func() { buf, err := os.ReadFile("/sys/module/apparmor/parameters/enabled") appArmorEnabled = err == nil && len(buf) > 1 && buf[0] == 'Y' }) + return appArmorEnabled } @@ -227,6 +228,7 @@ var _ = framework.KubeDescribe("AppArmor", func() { func createContainerWithAppArmor(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, sandboxID string, sandboxConfig *runtimeapi.PodSandboxConfig, profile *runtimeapi.LinuxContainerSecurityContext, shouldSucceed bool) string { By("create a container with apparmor") + containerName := "apparmor-test-" + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -241,6 +243,7 @@ func createContainerWithAppArmor(rc internalapi.RuntimeService, ic internalapi.I if shouldSucceed { Expect(err).ToNot(HaveOccurred()) By("start container with apparmor") + err := rc.StartContainer(context.TODO(), containerID) Expect(err).NotTo(HaveOccurred()) @@ -257,6 +260,7 @@ func createContainerWithAppArmor(rc internalapi.RuntimeService, ic internalapi.I func checkContainerApparmor(rc internalapi.RuntimeService, containerID string, shouldRun bool) { By("get container status") + resp, err := rc.ContainerStatus(context.TODO(), containerID, false) Expect(err).NotTo(HaveOccurred()) @@ -272,6 +276,7 @@ func loadTestProfiles() error { if err != nil { return fmt.Errorf("open temp file: %w", err) } + defer os.Remove(f.Name()) defer f.Close() @@ -290,6 +295,7 @@ func loadTestProfiles() error { if stderr.Len() > 0 { logrus.Warn(stderr.String()) } + if len(out) > 0 { logrus.Infof("apparmor_parser: %s", out) } diff --git a/pkg/validate/container.go b/pkg/validate/container.go index d4232be3df..11b70f0cac 100644 --- a/pkg/validate/container.go +++ b/pkg/validate/container.go @@ -384,6 +384,7 @@ func containerFound(containers []*runtimeapi.Container, containerID string) bool return true } } + return false } @@ -394,6 +395,7 @@ func statFound(stats []*runtimeapi.ContainerStats, containerID string) bool { return true } } + return false } @@ -402,6 +404,7 @@ func getContainerStatus(c internalapi.RuntimeService, containerID string) *runti By("Get container status for containerID: " + containerID) status, err := c.ContainerStatus(context.TODO(), containerID, false) framework.ExpectNoError(err, "failed to get container %q status: %v", containerID, err) + return status.GetStatus() } @@ -440,10 +443,12 @@ func testStartContainer(rc internalapi.RuntimeService, containerID string) { // stopContainer stops the container for containerID. func stopContainer(c internalapi.RuntimeService, containerID string, timeout int64) { By("Stop container for containerID: " + containerID) + stopped := make(chan bool, 1) go func() { defer GinkgoRecover() + err := c.StopContainer(context.TODO(), containerID, timeout) framework.ExpectNoError(err, "failed to stop container: %v", err) stopped <- true @@ -481,6 +486,7 @@ func listContainerForID(c internalapi.RuntimeService, containerID string) []*run } containers, err := c.ListContainers(context.TODO(), filter) framework.ExpectNoError(err, "failed to list containers %q status: %v", containerID, err) + return containers } @@ -498,6 +504,7 @@ func execSyncContainer(c internalapi.RuntimeService, containerID string, command // execSyncContainer test execSync for containerID and make sure the response is right. func verifyExecSyncOutput(c internalapi.RuntimeService, containerID string, command []string, expectedLogMessage string) { By("verify execSync output") + stdout := execSyncContainer(c, containerID, command) Expect(stdout).To(Equal(expectedLogMessage), "The stdout output of execSync should be %s", expectedLogMessage) framework.Logf("verify Execsync output succeed") @@ -519,12 +526,14 @@ func createHostPath(podID string) (hostPath string) { func createSymlink(path string) string { symlinkPath := path + "-symlink" framework.ExpectNoError(os.Symlink(path, symlinkPath), "failed to create symlink %q", symlinkPath) + return symlinkPath } // createVolumeContainer creates a container with volume and the prefix of containerName and fails if it gets error. func createVolumeContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, prefix, podID string, podConfig *runtimeapi.PodSandboxConfig, hostPath string) string { By("create a container with volume and name") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -546,6 +555,7 @@ func createVolumeContainer(rc internalapi.RuntimeService, ic internalapi.ImageMa // createLogContainer creates a container with log and the prefix of containerName. func createLogContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, prefix, podID string, podConfig *runtimeapi.PodSandboxConfig) (logPath, containerID string) { By("create a container with log and name") + containerName := prefix + framework.NewUUID() path := containerName + ".log" containerConfig := &runtimeapi.ContainerConfig{ @@ -554,12 +564,14 @@ func createLogContainer(rc internalapi.RuntimeService, ic internalapi.ImageManag Command: logDefaultCmd, LogPath: path, } + return containerConfig.LogPath, framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) } // createKeepLoggingContainer creates a container keeps logging defaultLog to output. func createKeepLoggingContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, prefix, podID string, podConfig *runtimeapi.PodSandboxConfig) (logPath, containerID string) { By("create a container with log and name") + containerName := prefix + framework.NewUUID() path := containerName + ".log" containerConfig := &runtimeapi.ContainerConfig{ @@ -568,6 +580,7 @@ func createKeepLoggingContainer(rc internalapi.RuntimeService, ic internalapi.Im Command: loopLogDefaultCmd, LogPath: path, } + return containerConfig.LogPath, framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) } @@ -577,10 +590,13 @@ func pathExists(path string) bool { if err == nil { return true } + if errors.Is(err, os.ErrNotExist) { return false } + framework.ExpectNoError(err, "failed to check whether %q Exists: %v", path, err) + return false } @@ -591,12 +607,15 @@ func pathExists(path string) bool { // 2016-10-06T00:17:10.113242941Z stderr F The content of the log entry 2 func parseCRILog(log string, msg *logMessage) { logMessage := strings.SplitN(log, " ", 4) + if len(log) < 4 { err := errors.New("invalid CRI log") framework.ExpectNoError(err, "failed to parse CRI log: %v", err) } + timeStamp, err := time.Parse(time.RFC3339Nano, logMessage[0]) framework.ExpectNoError(err, "failed to parse timeStamp: %v", err) + stream := logMessage[1] msg.timestamp = timeStamp @@ -611,9 +630,11 @@ func parseLogLine(podConfig *runtimeapi.PodSandboxConfig, logPath string) []logM f, err := os.Open(path) framework.ExpectNoError(err, "failed to open log file: %v", err) framework.Logf("Open log file %s", path) + defer f.Close() var msg logMessage + var msgLog []logMessage scanner := bufio.NewScanner(f) @@ -625,6 +646,7 @@ func parseLogLine(podConfig *runtimeapi.PodSandboxConfig, logPath string) []logM if err := scanner.Err(); err != nil { framework.ExpectNoError(err, "failed to read log by row: %v", err) } + framework.Logf("Parse container log succeed") return msgLog @@ -633,30 +655,38 @@ func parseLogLine(podConfig *runtimeapi.PodSandboxConfig, logPath string) []logM // verifyLogContents verifies the contents of container log. func verifyLogContents(podConfig *runtimeapi.PodSandboxConfig, logPath, log string, stream streamType) { By("verify log contents") + msgs := parseLogLine(podConfig, logPath) found := false + for _, msg := range msgs { if msg.log == log && msg.stream == stream { found = true + break } } + Expect(found).To(BeTrue(), "expected log %q (stream=%q) not found in logs %+v", log, stream, msgs) } // verifyLogContentsRe verifies the contents of container log using the provided regular expression pattern. func verifyLogContentsRe(podConfig *runtimeapi.PodSandboxConfig, logPath, pattern string, stream streamType) { By("verify log contents using regex pattern") + msgs := parseLogLine(podConfig, logPath) found := false + for _, msg := range msgs { if matched, _ := regexp.MatchString(pattern, msg.log); matched && msg.stream == stream { found = true + break } } + Expect(found).To(BeTrue(), "expected log pattern %q (stream=%q) to match logs %+v", pattern, stream, msgs) } @@ -665,13 +695,16 @@ func listContainerStatsForID(c internalapi.RuntimeService, containerID string) * By("List container stats for containerID: " + containerID) stats, err := c.ContainerStats(context.TODO(), containerID) framework.ExpectNoError(err, "failed to list container stats for %q status: %v", containerID, err) + return stats } // listContainerStats lists stats for containers based on filter. func listContainerStats(c internalapi.RuntimeService, filter *runtimeapi.ContainerStatsFilter) []*runtimeapi.ContainerStats { By("List container stats for all containers:") + stats, err := c.ListContainerStats(context.TODO(), filter) framework.ExpectNoError(err, "failed to list container stats for containers status: %v", err) + return stats } diff --git a/pkg/validate/container_linux.go b/pkg/validate/container_linux.go index 02d25d9647..34b7224b36 100644 --- a/pkg/validate/container_linux.go +++ b/pkg/validate/container_linux.go @@ -205,6 +205,7 @@ func createHostPathForMountPropagation(podID string, propagationOpt runtimeapi.M clearHostPath = func() { By("clean up the TempDir") + err := unix.Unmount(propagationMntPoint, unix.MNT_DETACH) framework.ExpectNoError(err, "failed to unmount \"propagationMntPoint\": %v", err) err = unix.Unmount(mntSource, unix.MNT_DETACH) @@ -230,6 +231,7 @@ func createMountPropagationContainer( propagation runtimeapi.MountPropagation, ) string { By("create a container with volume and name") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -253,6 +255,7 @@ func createMountPropagationContainer( containerID := framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) By("verifying container status") + resp, err := rc.ContainerStatus(context.TODO(), containerID, true) framework.ExpectNoError(err, "unable to get container status") Expect(resp.Status.Mounts).To(HaveLen(1)) @@ -282,6 +285,7 @@ func createOOMKilledContainer( podConfig *runtimeapi.PodSandboxConfig, ) string { By("create a container that will be killed by OOMKiller") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -302,6 +306,7 @@ func createOOMKilledContainer( containerID := framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) By("verifying container status") + _, err := rc.ContainerStatus(context.TODO(), containerID, true) framework.ExpectNoError(err, "unable to get container status") @@ -337,6 +342,7 @@ var _ = framework.KubeDescribe("Container Mount Readonly", func() { testRRO := func(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, rro bool) { if rro && !runtimeSupportsRRO(rc, framework.TestContext.RuntimeHandler) { Skip("runtime does not implement recursive readonly mounts") + return } @@ -367,6 +373,7 @@ var _ = framework.KubeDescribe("Container Mount Readonly", func() { testRROInvalidPropagation := func(prop runtimeapi.MountPropagation) { if !runtimeSupportsRRO(rc, framework.TestContext.RuntimeHandler) { Skip("runtime does not implement recursive readonly mounts") + return } hostPath, clearHostPath := createHostPathForRROMount(podID) @@ -393,6 +400,7 @@ var _ = framework.KubeDescribe("Container Mount Readonly", func() { It("should reject a recursive readonly mount with ReadOnly: false", func() { if !runtimeSupportsRRO(rc, framework.TestContext.RuntimeHandler) { Skip("runtime does not implement recursive readonly mounts") + return } hostPath, clearHostPath := createHostPathForRROMount(podID) @@ -416,6 +424,7 @@ func runtimeSupportsRRO(rc internalapi.RuntimeService, runtimeHandlerName string ctx := context.Background() status, err := rc.Status(ctx, false) framework.ExpectNoError(err, "failed to check runtime status") + for _, h := range status.RuntimeHandlers { if h.Name == runtimeHandlerName { if f := h.Features; f != nil { @@ -423,6 +432,7 @@ func runtimeSupportsRRO(rc internalapi.RuntimeService, runtimeHandlerName string } } } + return false } @@ -442,6 +452,7 @@ func createHostPathForRROMount(podID string) (hostPath string, clearHostPath fun clearHostPath = func() { By("clean up the TempDir") + err := unix.Unmount(tmpfsMntPoint, unix.MNT_DETACH) framework.ExpectNoError(err, "failed to unmount \"tmpfsMntPoint\": %v", err) err = os.RemoveAll(hostPath) @@ -468,6 +479,7 @@ func createRROMountContainer( SelinuxRelabel: true, }, } + return createMountContainer(rc, ic, podID, podConfig, mounts, false) } @@ -480,6 +492,7 @@ func createMountContainer( expectErr bool, ) string { By("create a container with volume and name") + containerName := "test-mount-" + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -491,12 +504,14 @@ func createMountContainer( if expectErr { _, err := framework.CreateContainerWithError(rc, ic, containerConfig, podID, podConfig) Expect(err).To(HaveOccurred()) + return "" } containerID := framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) By("verifying container status") + resp, err := rc.ContainerStatus(context.TODO(), containerID, true) framework.ExpectNoError(err, "unable to get container status") Expect(resp.Status.Mounts).To(HaveLen(len(mounts))) diff --git a/pkg/validate/image.go b/pkg/validate/image.go index beb1a1ceef..6cf8880e8c 100644 --- a/pkg/validate/image.go +++ b/pkg/validate/image.go @@ -158,6 +158,7 @@ var _ = framework.KubeDescribe("Image Manager", func() { if img.Id == id { Expect(img.RepoTags).To(HaveLen(1), "Should only have 1 repo tag") Expect(img.RepoTags[0]).To(Equal(testDifferentTagDifferentImageList[i]), "Repo tag should be correct") + break } } @@ -181,6 +182,7 @@ var _ = framework.KubeDescribe("Image Manager", func() { if img.Id == ids[0] { sort.Strings(img.RepoTags) Expect(img.RepoTags).To(Equal(testDifferentTagSameImageList), "Should have 3 repoTags in single image") + break } } @@ -193,6 +195,7 @@ func testRemoveImage(c internalapi.ImageManagerService, imageName string) { removeImage(c, imageName) By("Check image list empty") + imageStatus := framework.ImageStatus(c, imageName) Expect(imageStatus).To(BeNil(), "Should have none image in list") } @@ -209,6 +212,7 @@ func testPullPublicImage(c internalapi.ImageManagerService, imageName string, po Expect(imageStatus).NotTo(BeNil(), "Should have one image in list") Expect(imageStatus.Id).NotTo(BeNil(), "Image Id should not be nil") Expect(imageStatus.Size_).NotTo(BeNil(), "Image Size should not be nil") + if statusCheck != nil { statusCheck(imageStatus) } @@ -221,6 +225,7 @@ func pullImageList(c internalapi.ImageManagerService, imageList []string, podCon for _, imageName := range imageList { ids = append(ids, framework.PullPublicImage(c, imageName, podConfig)) } + return ids } diff --git a/pkg/validate/multi_container_linux.go b/pkg/validate/multi_container_linux.go index 98af038628..9baa4e940e 100644 --- a/pkg/validate/multi_container_linux.go +++ b/pkg/validate/multi_container_linux.go @@ -86,6 +86,7 @@ var _ = framework.KubeDescribe("Multiple Containers [Conformance]", func() { if err != nil { return false, err } + return strings.Contains(string(content), expected), nil } } @@ -126,6 +127,7 @@ func createMultiContainerTestPodSandbox(c internalapi.RuntimeService) (sandboxID CgroupParent: common.GetCgroupParent(context.TODO(), c), }, } + return framework.RunPodSandbox(c, podConfig), podConfig, logDir } @@ -140,6 +142,7 @@ func createMultiContainerTestHttpdContainer(rc internalapi.RuntimeService, ic in Linux: &runtimeapi.LinuxContainerConfig{}, LogPath: containerName + ".log", } + return framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) } @@ -154,5 +157,6 @@ func createMultiContainerTestBusyboxContainer(rc internalapi.RuntimeService, ic Command: []string{"sh", "-c", "echo " + defaultLog + "; sleep 1000"}, LogPath: containerName + ".log", } + return framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) } diff --git a/pkg/validate/networking.go b/pkg/validate/networking.go index d10f90f3ec..c6936770d6 100644 --- a/pkg/validate/networking.go +++ b/pkg/validate/networking.go @@ -143,6 +143,7 @@ func createPodSandWithHostname(c internalapi.RuntimeService, hostname string) (s } podID := framework.RunPodSandbox(c, config) + return podID, config } @@ -165,6 +166,7 @@ func createPodSandWithDNSConfig(c internalapi.RuntimeService) (string, *runtimea } podID := framework.RunPodSandbox(c, config) + return podID, config } @@ -173,6 +175,7 @@ func createPodSandboxWithPortMapping(c internalapi.RuntimeService, portMappings podSandboxName := "create-PodSandbox-with-port-mapping" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + config := &runtimeapi.PodSandboxConfig{ Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), PortMappings: portMappings, @@ -190,12 +193,14 @@ func createPodSandboxWithPortMapping(c internalapi.RuntimeService, portMappings } podID := framework.RunPodSandbox(c, config) + return podID, config } // checkHostname checks the container hostname. func checkHostname(c internalapi.RuntimeService, containerID, hostname string) { By("get the current hostname via execSync") + stdout, stderr, err := c.ExecSync(context.TODO(), containerID, getHostnameCmd, time.Duration(defaultExecSyncTimeout)*time.Second) framework.ExpectNoError(err, "failed to execSync in container %q", containerID) Expect(strings.EqualFold(strings.TrimSpace(string(stdout)), hostname)).To(BeTrue()) @@ -206,11 +211,14 @@ func checkHostname(c internalapi.RuntimeService, containerID, hostname string) { // checkDNSConfig checks the content of /etc/resolv.conf. func checkDNSConfig(c internalapi.RuntimeService, containerID string, expectedContent []string) { By("get the current dns config via execSync") + stdout, stderr, err := c.ExecSync(context.TODO(), containerID, getDNSConfigCmd, time.Duration(defaultExecSyncTimeout)*time.Second) framework.ExpectNoError(err, "failed to execSync in container %q", containerID) + for _, content := range expectedContent { Expect(string(stdout)).To(ContainSubstring(content), "The stdout output of execSync should contain %q", content) } + Expect(string(stderr)).To(BeEmpty(), "The stderr should be empty.") framework.Logf("check DNS config succeed") } @@ -223,6 +231,7 @@ func createWebServerContainer(rc internalapi.RuntimeService, ic internalapi.Imag Image: &runtimeapi.ImageSpec{Image: webServerImage}, Linux: &runtimeapi.LinuxContainerConfig{}, } + return framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) } @@ -234,6 +243,7 @@ func createHostNetWebServerContainer(rc internalapi.RuntimeService, ic internala Image: &runtimeapi.ImageSpec{Image: hostNetWebServerImage}, Linux: &runtimeapi.LinuxContainerConfig{}, } + return framework.CreateContainer(rc, ic, containerConfig, podID, podConfig) } @@ -250,6 +260,7 @@ func checkMainPage(c internalapi.RuntimeService, podID string, hostPort, contain Expect(status.GetNetwork().Ip).NotTo(BeNil(), "The IP should not be nil.") url += status.GetNetwork().Ip + ":" + strconv.Itoa(int(containerPort)) } + framework.Logf("the IP:port is " + url) By("check the content of " + url) @@ -272,6 +283,7 @@ func checkMainPage(c internalapi.RuntimeService, podID string, hostPort, contain } defer resp.Body.Close() respChan <- resp + return nil }, time.Minute, time.Second).Should(Succeed()) diff --git a/pkg/validate/pod.go b/pkg/validate/pod.go index c5d5133fcb..2d34d0b663 100644 --- a/pkg/validate/pod.go +++ b/pkg/validate/pod.go @@ -89,6 +89,7 @@ func podSandboxFound(podSandboxs []*runtimeapi.PodSandbox, podID string) bool { return true } } + return false } @@ -102,6 +103,7 @@ func verifyPodSandboxStatus(c internalapi.RuntimeService, podID string, expected func testRunDefaultPodSandbox(c internalapi.RuntimeService) string { podID := framework.RunDefaultPodSandbox(c, "PodSandbox-for-create-test-") verifyPodSandboxStatus(c, podID, runtimeapi.PodSandboxState_SANDBOX_READY, "ready") + return podID } @@ -110,6 +112,7 @@ func getPodSandboxStatus(c internalapi.RuntimeService, podID string) *runtimeapi By("Get PodSandbox status for podID: " + podID) status, err := c.PodSandboxStatus(context.TODO(), podID, false) framework.ExpectNoError(err, "failed to get PodSandbox %q status: %v", podID, err) + return status.GetStatus() } @@ -148,15 +151,18 @@ func listPodSandboxForID(c internalapi.RuntimeService, podID string) []*runtimea filter := &runtimeapi.PodSandboxFilter{ Id: podID, } + return listPodSandbox(c, filter) } // listPodSandbox lists PodSandbox. func listPodSandbox(c internalapi.RuntimeService, filter *runtimeapi.PodSandboxFilter) []*runtimeapi.PodSandbox { By("List PodSandbox.") + pods, err := c.ListPodSandbox(context.TODO(), filter) framework.ExpectNoError(err, "failed to list PodSandbox status: %v", err) framework.Logf("List PodSandbox succeed") + return pods } @@ -174,6 +180,7 @@ func createLogTempDir(podSandboxName string) (hostPath, podLogPath string) { // createPodSandboxWithLogDirectory creates a PodSandbox with log directory. func createPodSandboxWithLogDirectory(c internalapi.RuntimeService) (sandboxID string, podConfig *runtimeapi.PodSandboxConfig, hostPath string) { By("create a PodSandbox with log directory") + podSandboxName := "PodSandbox-with-log-directory-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() @@ -186,5 +193,6 @@ func createPodSandboxWithLogDirectory(c internalapi.RuntimeService) (sandboxID s CgroupParent: common.GetCgroupParent(context.TODO(), c), }, } + return framework.RunPodSandbox(c, podConfig), podConfig, hostPath } diff --git a/pkg/validate/pod_linux.go b/pkg/validate/pod_linux.go index d4da07f04c..020220b02d 100644 --- a/pkg/validate/pod_linux.go +++ b/pkg/validate/pod_linux.go @@ -87,6 +87,7 @@ var _ = framework.KubeDescribe("PodSandbox", func() { // createSandboxWithSysctls creates a PodSandbox with specified sysctls. func createSandboxWithSysctls(rc internalapi.RuntimeService, sysctls map[string]string) (string, *runtimeapi.PodSandboxConfig) { By("create a PodSandbox with sysctls") + podSandboxName := "pod-sandbox-with-sysctls-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() @@ -97,6 +98,7 @@ func createSandboxWithSysctls(rc internalapi.RuntimeService, sysctls map[string] Sysctls: sysctls, }, } + return framework.RunPodSandbox(rc, podConfig), podConfig } diff --git a/pkg/validate/runtime_info.go b/pkg/validate/runtime_info.go index 07230e1a59..84ee0817e4 100644 --- a/pkg/validate/runtime_info.go +++ b/pkg/validate/runtime_info.go @@ -64,6 +64,7 @@ func TestGetVersion(c internalapi.RuntimeService) { // TestGetRuntimeStatus test if we can get runtime status. func TestGetRuntimeStatus(c internalapi.RuntimeService) { var count int + status, err := c.Status(context.TODO(), false) framework.ExpectNoError(err, "failed to get runtime conditions: %v", err) @@ -71,10 +72,12 @@ func TestGetRuntimeStatus(c internalapi.RuntimeService) { if condition.Type == "RuntimeReady" && condition.Status { count++ } + if condition.Type == "NetworkReady" && condition.Status { count++ } } + Expect(count).To(BeNumerically(">=", 2), "should return all the required runtime conditions") } @@ -82,5 +85,6 @@ func TestGetRuntimeStatus(c internalapi.RuntimeService) { func getVersion(c internalapi.RuntimeService) *runtimeapi.VersionResponse { version, err := c.Version(context.TODO(), defaultAPIVersion) framework.ExpectNoError(err, "failed to get version: %v", err) + return version } diff --git a/pkg/validate/security_context_linux.go b/pkg/validate/security_context_linux.go index e8a13ba89d..10f783edc4 100644 --- a/pkg/validate/security_context_linux.go +++ b/pkg/validate/security_context_linux.go @@ -785,17 +785,20 @@ var _ = framework.KubeDescribe("Security Context", func() { profileDir, err = createSeccompProfileDir() if err != nil { Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed creating seccomp profile directory: %v", err)) + return } dirToCleanup = append(dirToCleanup, profileDir) blockHostNameProfilePath, err = createSeccompProfile(seccompBlockHostNameProfile, "block-host-name.json", profileDir) if err != nil { Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed creating seccomp block hostname profile: %v", err)) + return } blockchmodProfilePath, err = createSeccompProfile(seccompBlockChmodProfile, "block-chmod.json", profileDir) if err != nil { Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed creating seccomp block chmod profile: %v", err)) + return } }) @@ -987,6 +990,7 @@ var _ = framework.KubeDescribe("Security Context", func() { if rh.GetName() == framework.TestContext.RuntimeHandler { if rh.GetFeatures().GetUserNamespaces() { supportsUserNamespaces = true + break } } @@ -1128,11 +1132,13 @@ func matchContainerOutputRe(podConfig *runtimeapi.PodSandboxConfig, name, patter // createRunAsUserContainer creates the container with specified RunAsUser in ContainerConfig. func createRunAsUserContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) (containerID, expectedLogMessage string) { By("create RunAsUser container") + var uidV runtimeapi.Int64Value uidV.Value = 1001 expectedLogMessage = "1001\n" By("create a container with RunAsUser") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -1151,10 +1157,12 @@ func createRunAsUserContainer(rc internalapi.RuntimeService, ic internalapi.Imag // createRunAsUserNameContainer creates the container with specified RunAsUserName in ContainerConfig. func createRunAsUserNameContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) (containerID, expectedLogMessage string) { By("create RunAsUserName container") + userName := "nobody" expectedLogMessage = userName + "\n" By("create a container with RunAsUserName") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -1166,18 +1174,21 @@ func createRunAsUserNameContainer(rc internalapi.RuntimeService, ic internalapi. }, }, } + return framework.CreateContainer(rc, ic, containerConfig, podID, podConfig), expectedLogMessage } // createRunAsGroupContainer creates the container with specified RunAsGroup in ContainerConfig. func createRunAsGroupContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, containerName string) (containerID, expectedLogMessage string) { By("create RunAsGroup container") + var uidV, gidV runtimeapi.Int64Value uidV.Value = 1001 gidV.Value = 1002 expectedLogMessage = "1001:1002\n" By("create a container with RunAsUser and RunAsGroup") + containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), Image: &runtimeapi.ImageSpec{Image: framework.TestContext.TestImageList.DefaultTestContainerImage}, @@ -1190,6 +1201,7 @@ func createRunAsGroupContainer(rc internalapi.RuntimeService, ic internalapi.Ima }, LogPath: containerName + ".log", } + return framework.CreateContainer(rc, ic, containerConfig, podID, podConfig), expectedLogMessage } @@ -1197,10 +1209,12 @@ func createRunAsGroupContainer(rc internalapi.RuntimeService, ic internalapi.Ima // RunAsUser specified in ContainerConfig. func createInvalidRunAsGroupContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, containerName string) { By("create invalid RunAsGroup container") + var gidV runtimeapi.Int64Value gidV.Value = 1002 By("create a container with RunAsGroup without RunAsUser") + containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), Image: &runtimeapi.ImageSpec{Image: framework.TestContext.TestImageList.DefaultTestContainerImage}, @@ -1218,6 +1232,7 @@ func createInvalidRunAsGroupContainer(rc internalapi.RuntimeService, ic internal // createNamespacePodSandbox creates a PodSandbox with different NamespaceOption config for creating containers. func createNamespacePodSandbox(rc internalapi.RuntimeService, podSandboxNamespace *runtimeapi.NamespaceOption, podSandboxName, podLogPath string) (string, *runtimeapi.PodSandboxConfig) { By("create NamespaceOption podSandbox") + uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() config := &runtimeapi.PodSandboxConfig{ @@ -1239,6 +1254,7 @@ func createNamespacePodSandbox(rc internalapi.RuntimeService, podSandboxNamespac // createNamespaceContainer creates container with different NamespaceOption config. func createNamespaceContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, containerName, image string, containerNamespace *runtimeapi.NamespaceOption, command []string, path string) (containerID, logPath string) { By("create NamespaceOption container") + containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), Image: &runtimeapi.ImageSpec{Image: image}, @@ -1257,6 +1273,7 @@ func createNamespaceContainer(rc internalapi.RuntimeService, ic internalapi.Imag // createReadOnlyRootfsContainer creates the container with specified ReadOnlyRootfs in ContainerConfig. func createReadOnlyRootfsContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string, readonly bool) (containerID, logPath string) { By("create ReadOnlyRootfs container") + containerName := prefix + framework.NewUUID() path := containerName + ".log" containerConfig := &runtimeapi.ContainerConfig{ @@ -1290,6 +1307,7 @@ func checkRootfs(podConfig *runtimeapi.PodSandboxConfig, logpath string, readOnl // createPrivilegedPodSandbox creates a PodSandbox with Privileged of SecurityContext config. func createPrivilegedPodSandbox(rc internalapi.RuntimeService, privileged bool) (string, *runtimeapi.PodSandboxConfig) { By("create Privileged podSandbox") + podSandboxName := "create-Privileged-PodSandbox-for-container-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() @@ -1310,6 +1328,7 @@ func createPrivilegedPodSandbox(rc internalapi.RuntimeService, privileged bool) // createPrivilegedContainer creates container with specified Privileged in ContainerConfig. func createPrivilegedContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string, privileged bool) string { By("create Privileged container") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -1342,6 +1361,7 @@ func checkNetworkManagement(rc internalapi.RuntimeService, containerID string, m // createCapabilityContainer creates container with specified Capability in ContainerConfig. func createCapabilityContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string, add, drop []string) string { By("create Capability container") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -1362,10 +1382,12 @@ func createCapabilityContainer(rc internalapi.RuntimeService, ic internalapi.Ima func createAndCheckHostNetwork(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podSandboxName, hostNetworkPort string, hostNetwork bool) (podID, podLogDir string) { By(fmt.Sprintf("creating a podSandbox with hostNetwork %v", hostNetwork)) + netNSMode := runtimeapi.NamespaceMode_POD if hostNetwork { netNSMode = runtimeapi.NamespaceMode_NODE } + namespaceOptions := &runtimeapi.NamespaceOption{ Pid: runtimeapi.NamespaceMode_POD, Ipc: runtimeapi.NamespaceMode_POD, @@ -1375,6 +1397,7 @@ func createAndCheckHostNetwork(rc internalapi.RuntimeService, ic internalapi.Ima podID, podConfig := createNamespacePodSandbox(rc, namespaceOptions, podSandboxName, podLogPath) By("create a container in the sandbox") + command := []string{"sh", "-c", "netstat -ln"} containerName := "container-with-HostNetwork-test-" + framework.NewUUID() path := containerName + ".log" @@ -1399,6 +1422,7 @@ func createAndCheckHostNetwork(rc internalapi.RuntimeService, ic internalapi.Ima if hostNetwork { return fmt.Errorf("host port %s should be in container's port list", hostNetworkPort) } + return nil }, time.Minute, time.Second).Should(Succeed()) @@ -1411,22 +1435,24 @@ func createSeccompProfileDir() (string, error) { if err != nil { return "", fmt.Errorf("create tempdir %q: %w", hostPath, err) } + return hostPath, nil } // createSeccompProfile creates a seccomp test profile with profileContents. func createSeccompProfile(profileContents, profileName, hostPath string) (string, error) { profilePath := filepath.Join(hostPath, profileName) - err := os.WriteFile(profilePath, []byte(profileContents), 0o644) - if err != nil { + if err := os.WriteFile(profilePath, []byte(profileContents), 0o644); err != nil { return "", fmt.Errorf("create %s: %w", profilePath, err) } + return profilePath, nil } // seccompTestContainer creates and starts a seccomp sandbox and a container. func seccompTestContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, profile *runtimeapi.SecurityProfile) (podID, containerID string) { By("create seccomp sandbox") + podSandboxName := "seccomp-sandbox-" + framework.NewUUID() uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() @@ -1443,6 +1469,7 @@ func seccompTestContainer(rc internalapi.RuntimeService, ic internalapi.ImageMan podID = framework.RunPodSandbox(rc, podConfig) By("create container") + containerNamePrefix := "seccomp-container-" + framework.NewUUID() containerName := containerNamePrefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ @@ -1491,6 +1518,7 @@ func createSeccompContainer(rc internalapi.RuntimeService, expectContainerCreateToPass bool, ) string { By("create " + profile.GetProfileType().String() + " Seccomp container") + containerName := prefix + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -1527,11 +1555,14 @@ func createContainerWithExpectation(rc internalapi.RuntimeService, if !strings.Contains(imageName, ":") { imageName += ":latest" } + status := framework.ImageStatus(ic, imageName) if status == nil { framework.PullPublicImage(ic, imageName, nil) } + By("Create container.") + containerID, err := rc.CreateContainer(context.TODO(), podID, config, podConfig) if !expectContainerCreateToPass { @@ -1541,6 +1572,7 @@ func createContainerWithExpectation(rc internalapi.RuntimeService, framework.ExpectNoError(err, "failed to create container: %v", err) framework.Logf("Created container %q\n", containerID) } + return containerID } @@ -1566,6 +1598,7 @@ func runUserNamespaceContainer( podConfig *runtimeapi.PodSandboxConfig, ) string { By("create user namespaces container") + containerName := "user-namespaces-container-" + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -1631,19 +1664,24 @@ func supportsIDMap(path string) error { UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: usernsHostID, Size: usernsSize}}, GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: usernsHostID, Size: usernsSize}}, } + if err := cmd.Start(); err != nil { return err } + defer func() { _ = cmd.Process.Kill() _ = cmd.Wait() }() usernsPath := fmt.Sprintf("/proc/%d/ns/user", cmd.Process.Pid) + var usernsFile *os.File + if usernsFile, err = os.Open(usernsPath); err != nil { return err } + defer usernsFile.Close() attr := unix.MountAttr{ @@ -1664,6 +1702,7 @@ func supportsIDMap(path string) error { func rootfsPath(info map[string]string) string { defaultPath := "/var/lib" jsonCfg, ok := info["config"] + if !ok { return defaultPath } @@ -1672,10 +1711,12 @@ func rootfsPath(info map[string]string) string { type containerdConfig struct { StateDir string `json:"stateDir"` } + cfg := containerdConfig{} if err := json.Unmarshal([]byte(jsonCfg), &cfg); err != nil { return defaultPath } + if cfg.StateDir == "" { return defaultPath } @@ -1687,10 +1728,12 @@ func rootfsPath(info map[string]string) string { func hostUsernsContent() string { uidMapPath := "/proc/self/uid_map" + uidMapContent, err := os.ReadFile(uidMapPath) if err != nil { return "" } + return string(uidMapContent) } @@ -1703,5 +1746,6 @@ func parseUsernsMappingLine(line string) []string { m = slices.DeleteFunc(m, func(s string) bool { return s == "" }) + return m } diff --git a/pkg/validate/selinux_linux.go b/pkg/validate/selinux_linux.go index ad32bd528d..751923f4b5 100644 --- a/pkg/validate/selinux_linux.go +++ b/pkg/validate/selinux_linux.go @@ -159,6 +159,7 @@ var _ = framework.KubeDescribe("SELinux", func() { func createContainerWithSelinux(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, sandboxID string, sandboxConfig *runtimeapi.PodSandboxConfig, options *runtimeapi.SELinuxOption, privileged, shouldStart, shouldCreate bool) string { By("create a container with selinux") + containerName := "selinux-test-" + framework.NewUUID() containerConfig := &runtimeapi.ContainerConfig{ Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), @@ -171,15 +172,18 @@ func createContainerWithSelinux(rc internalapi.RuntimeService, ic internalapi.Im }, }, } + containerID, err := framework.CreateContainerWithError(rc, ic, containerConfig, sandboxID, sandboxConfig) if !shouldCreate { Expect(err).To(HaveOccurred()) + return "" } Expect(err).NotTo(HaveOccurred()) By("start container with selinux") + err = rc.StartContainer(context.TODO(), containerID) if shouldStart { Expect(err).NotTo(HaveOccurred()) @@ -197,6 +201,7 @@ func createContainerWithSelinux(rc internalapi.RuntimeService, ic internalapi.Im func checkContainerSelinux(rc internalapi.RuntimeService, containerID string, shouldRun bool) { By("get container status") + status, err := rc.ContainerStatus(context.TODO(), containerID, false) Expect(err).NotTo(HaveOccurred()) @@ -204,6 +209,7 @@ func checkContainerSelinux(rc internalapi.RuntimeService, containerID string, sh Expect(status.GetStatus().GetExitCode()).To(Equal(int32(0))) } else { Expect(status.GetStatus().GetExitCode()).NotTo(Equal(int32(0))) + return } @@ -236,6 +242,7 @@ func checkProcessLabelRoleType(rc internalapi.RuntimeService, containerID string label := strings.Trim(string(stdout), "\x00") msg := fmt.Sprintf("cmd %v, stdout %q, stderr %q", cmd, stdout, stderr) Expect(err).NotTo(HaveOccurred(), msg) + if privileged { Expect(label).To(ContainSubstring(":system_r:spc_t:")) } else { @@ -260,6 +267,7 @@ func checkProcessLabelMCS(rc internalapi.RuntimeService, containerID string, pri label := strings.Trim(string(stdout), "\x00") msg := fmt.Sprintf("cmd %v, stdout %q, stderr %q", cmd, stdout, stderr) Expect(err).NotTo(HaveOccurred(), msg) + if privileged { // check that a process label exists with optional MCS, where level is always s0 and we permit all categories Expect(label).To(MatchRegexp(`:s0(-s0)?(:c0\.c1023)?$`)) @@ -267,5 +275,6 @@ func checkProcessLabelMCS(rc internalapi.RuntimeService, containerID string, pri // check that a process label exists with MCS, where level is always s0 and there are two or more categories Expect(label).To(MatchRegexp(`:s0(-s0)?:c[0-9]+(,c[0-9]+)+$`)) } + return label } diff --git a/pkg/validate/streaming.go b/pkg/validate/streaming.go index 4d97f703ca..b926e48c88 100644 --- a/pkg/validate/streaming.go +++ b/pkg/validate/streaming.go @@ -153,6 +153,7 @@ func createExec(c internalapi.RuntimeService, execReq *runtimeapi.ExecRequest) s resp, err := c.Exec(context.TODO(), execReq) framework.ExpectNoError(err, "failed to exec in container %q", execReq.ContainerId) framework.Logf("Get exec URL: " + resp.Url) + return resp.Url } @@ -170,12 +171,14 @@ func checkExec(c internalapi.RuntimeService, execServerURL, stdout string, stdou go func() { defer wg.Done() defer localInWrite.Close() + ticker := time.NewTicker(defaultExecStdinCloseTimeout) select { case <-testDone: case <-ticker.C: } }() + defer func() { close(testDone) wg.Wait() @@ -208,6 +211,7 @@ func checkExec(c internalapi.RuntimeService, execServerURL, stdout string, stdou } else { Expect(localOut.String()).To(ContainSubstring(stdout), "The stdout of exec should contain "+stdout) } + Expect(localErr.String()).To(BeEmpty(), "The stderr of exec should be empty") framework.Logf("Check exec URL %q succeed", execServerURL) } @@ -221,6 +225,7 @@ func parseURL(c internalapi.RuntimeService, serverURL string) *url.URL { if parsedURL.Host == "" { parsedURL.Host = defaultStreamServerAddress } + if parsedURL.Scheme == "" { parsedURL.Scheme = defaultStreamServerScheme } @@ -228,6 +233,7 @@ func parseURL(c internalapi.RuntimeService, serverURL string) *url.URL { Expect(parsedURL.Host).NotTo(BeEmpty(), "The host of URL should not be empty") framework.Logf("Parse URL %q succeed", serverURL) + return parsedURL } @@ -244,6 +250,7 @@ func createDefaultAttach(c internalapi.RuntimeService, containerID string) strin resp, err := c.Attach(context.TODO(), req) framework.ExpectNoError(err, "failed to attach in container %q", containerID) framework.Logf("Get attach URL: " + resp.Url) + return resp.Url } @@ -258,6 +265,7 @@ type safeBuffer struct { func (s *safeBuffer) Write(p []byte) (n int, err error) { s.mu.Lock() defer s.mu.Unlock() + return s.buffer.Write(p) } @@ -266,6 +274,7 @@ func (s *safeBuffer) Write(p []byte) (n int, err error) { func (s *safeBuffer) String() string { s.mu.Lock() defer s.mu.Unlock() + return s.buffer.String() } @@ -273,16 +282,21 @@ func checkAttach(c internalapi.RuntimeService, attachServerURL string) { localOut := &safeBuffer{buffer: bytes.Buffer{}} localErr := &safeBuffer{buffer: bytes.Buffer{}} reader, writer := io.Pipe() + go func() { defer GinkgoRecover() defer writer.Close() + header := localOut.String() + time.Sleep(1 * time.Second) Eventually(func() bool { oldHeader := header header = localOut.String() + return len(header) == len(oldHeader) }, 10*time.Second, time.Second).Should(BeTrue(), "The container should stop output when there is no input") + _, err := writer.Write([]byte(strings.Join(echoHelloCmd, " ") + "\n")) Expect(err).NotTo(HaveOccurred()) Eventually(func() string { @@ -323,6 +337,7 @@ func createDefaultPortForward(c internalapi.RuntimeService, podID string) string resp, err := c.PortForward(context.TODO(), req) framework.ExpectNoError(err, "failed to port forward PodSandbox %q", podID) framework.Logf("Get port forward URL: " + resp.Url) + return resp.Url } @@ -333,6 +348,7 @@ func checkPortForward(c internalapi.RuntimeService, portForwardSeverURL string, transport, upgrader, err := spdy.RoundTripperFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) framework.ExpectNoError(err, "failed to create spdy round tripper") + parsedURL := parseURL(c, portForwardSeverURL) dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", parsedURL) @@ -345,6 +361,7 @@ func checkPortForward(c internalapi.RuntimeService, portForwardSeverURL string, go func() { defer GinkgoRecover() By("start port forward") + err := pf.ForwardPorts() framework.ExpectNoError(err, "failed to start port forward for %q, stdout: %s, stderr: %s", portForwardSeverURL, stdout.String(), stderr.String()) }() diff --git a/test/e2e/config_test.go b/test/e2e/config_test.go index 8c9043fa8e..8a9e9b8c2c 100644 --- a/test/e2e/config_test.go +++ b/test/e2e/config_test.go @@ -40,6 +40,7 @@ var _ = t.Describe("config", func() { listConfig := func() string { res := t.Crictl("--config " + configFile.Name() + " config --list") Expect(res).To(Exit(0)) + return string(res.Out.Contents()) } diff --git a/test/framework/framework.go b/test/framework/framework.go index 74f2a122c7..8a72c57cca 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -69,6 +69,7 @@ func (t *TestFramework) Describe(text string, body func()) bool { // Convenience method for command creation. func cmd(workDir, format string, args ...interface{}) *Session { c := strings.Split(fmt.Sprintf(format, args...), " ") + command := exec.Command(c[0], c[1:]...) if workDir != "" { command.Dir = workDir @@ -84,6 +85,7 @@ func crictlBinaryPathFlag() (path string) { if crictlBinaryPath != "" { return crictlBinaryPath } + return "crictl" } @@ -91,6 +93,7 @@ func crictlRuntimeEndpointFlag() string { if crictlRuntimeEndpoint != "" { return " --runtime-endpoint=" + crictlRuntimeEndpoint } + return "" } @@ -118,11 +121,13 @@ func (t *TestFramework) CrictlExpect( // Then Expect(res).To(Exit(exit)) + if expectedOut == "" { Expect(string(res.Out.Contents())).To(BeEmpty()) } else { Expect(res.Out).To(Say(expectedOut)) } + if expectedErr == "" { Expect(string(res.Err.Contents())).To(BeEmpty()) } else {