update progress on podvolumebackup CR and restic log

Author: jmontleon

pkg/apis/ark/v1/pod_volume_backup.go

@@ -72,6 +72,9 @@ type PodVolumeBackupStatus struct {
 	// Phase is the current state of the PodVolumeBackup.
 	Phase PodVolumeBackupPhase `json:"phase"`
 
+	// Progress shows the current progress copying data for the backup
+	Progress string `json:"progress"`
+
 	// Path is the full path within the controller pod being backed up.
 	Path string `json:"path"`
 

pkg/controller/pod_volume_backup_controller.go

@@ -37,7 +37,6 @@ import (
 	informers "github.com/heptio/velero/pkg/generated/informers/externalversions/velero/v1"
 	listers "github.com/heptio/velero/pkg/generated/listers/velero/v1"
 	"github.com/heptio/velero/pkg/restic"
-	veleroexec "github.com/heptio/velero/pkg/util/exec"
 	"github.com/heptio/velero/pkg/util/filesystem"
 	"github.com/heptio/velero/pkg/util/kube"
 )
@@ -228,7 +227,7 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 
 	var stdout, stderr string
 
-	if stdout, stderr, err = veleroexec.RunCommand(resticCmd.Cmd()); err != nil {
+	if stdout, stderr, err = c.RunBackupCommand(resticCmd.Cmd(), req); err != nil {
 		log.WithError(errors.WithStack(err)).Errorf("Error running command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)
 		return c.fail(req, fmt.Sprintf("error running restic backup, stderr=%s: %s", stderr, err.Error()), log)
 	}
@@ -243,6 +242,7 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack
 	// update status to Completed with path & snapshot id
 	req, err = c.patchPodVolumeBackup(req, func(r *velerov1api.PodVolumeBackup) {
 		r.Status.Path = path
+		r.Status.Progress = "100%"
 		r.Status.SnapshotID = snapshotID
 		r.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted
 	})

pkg/controller/pod_volume_backup_exec.go

@@ -0,0 +1,76 @@
+package controller
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"sync"
+	"time"
+
+	velerov1api "github.com/heptio/velero/pkg/apis/velero/v1"
+)
+
+type Progress struct {
+	PercentDone float64 `json:"percent_done"`
+}
+
+func (c *podVolumeBackupController) RunBackupCommand(cmd *exec.Cmd, req *velerov1api.PodVolumeBackup) (string, string, error) {
+
+	var stdoutBuf, stderrBuf, stdoutStatBuf bytes.Buffer
+	stdoutIn, _ := cmd.StdoutPipe()
+	stderrIn, _ := cmd.StderrPipe()
+
+	var errStdout error
+	stdout := io.MultiWriter(os.Stdout, &stdoutBuf, &stdoutStatBuf)
+	stderr := io.MultiWriter(os.Stderr, &stderrBuf)
+
+	err := cmd.Start()
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		_, errStdout = io.Copy(stdout, stdoutIn)
+		wg.Done()
+	}()
+
+	wg.Add(1)
+	go func() {
+		var line string
+		var progress Progress
+		for {
+			time.Sleep(10 * time.Second)
+			scanner := bufio.NewScanner(&stdoutStatBuf)
+			line = ""
+			for scanner.Scan() {
+				line = (scanner.Text())
+			}
+			if err := scanner.Err(); err != nil {
+				fmt.Fprintln(os.Stderr, "reading standard input:", err)
+				break
+			}
+			stdoutStatBuf.Reset()
+			err = json.Unmarshal([]byte(line), &progress)
+			if err != nil {
+				break
+			}
+			_, err = c.patchPodVolumeBackup(req, func(r *velerov1api.PodVolumeBackup) {
+				r.Status.Progress = fmt.Sprintf("%.2f%%", progress.PercentDone*100)
+			})
+		}
+
+		wg.Done()
+	}()
+
+	_, _ = io.Copy(stderr, stderrIn)
+	wg.Wait()
+
+	err = cmd.Wait()
+
+	outStr, errStr := string(stdoutBuf.Bytes()), string(stderrBuf.Bytes())
+
+	return outStr, errStr, err
+}

pkg/restic/backupper.go

@@ -257,6 +257,9 @@ func newPodVolumeBackup(backup *velerov1api.Backup, pod *corev1api.Pod, volumeNa
 			BackupStorageLocation: backup.Spec.StorageLocation,
 			RepoIdentifier:        repoIdentifier,
 		},
+		Status: velerov1api.PodVolumeBackupStatus{
+			Progress: "0%",
+		},
 	}
 }
 

pkg/restic/command_factory.go

@@ -34,7 +34,7 @@ func BackupCommand(repoIdentifier, passwordFile, path string, tags map[string]st
 		PasswordFile:   passwordFile,
 		Dir:            path,
 		Args:           []string{"."},
-		ExtraFlags:     append(backupTagFlags(tags), "--host=velero"),
+		ExtraFlags:     append(backupTagFlags(tags), "--host=velero", "--json"),
 	}
 }