Ignore some errors during manifest loading

When loading manifests, we attempt to walk the base file tree twice. Once
while looking for directories with Helm charts in them, and then to load
all the .yaml/.yml files. The former keeps track of those directories
to exclude them while doing the latter since we do not want to load
yamels from directories with Helm charts in them.

While walking those file trees any error aborted the whole loading
process and as a consequence, the API calls `ListServices` and
`ListImages` would return an error. Suddenly disappearing files such as
Git's gc lock file would trigger an error. Since the Git repo is cloned
before manifests are loaded, there was a race sometimes between having
that lock file being enumerated but then disappear while trying to
retrieve information about it.

This PR ignores all errors while enumerating the Helm chart directories
since permissions are unlikely to change, and disappearing files do not
modify the list of excluded directories.

When walking the yamels afterwards, only errors for files or directories
that we actually care about will be reported. That means any error for
files that do not have a yamel extension will be just ignored.

go.sum

@@ -672,6 +672,7 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
 github.com/weaveworks/common v0.0.0-20190410110702-87611edc252e h1:Y5z9Bu95l0laB3xOF3BT6LWkcR7VcRxN0n+nf2S72xA=
 github.com/weaveworks/common v0.0.0-20190410110702-87611edc252e/go.mod h1:pSm+0KR57BG3pvGoJWFXJSAC7+sEPewcvdt5StevL3A=
+github.com/weaveworks/flux v1.19.0 h1:/5K3yCyiu9XI3XkugR+V91ZWeWGe8BQD63yROi9C6Qo=
 github.com/weaveworks/go-checkpoint v0.0.0-20170503165305-ebbb8b0518ab h1:mW+hgchD9qUUBqnuaDBj7BkcpFPk/FxeFcUFI5lvvUw=
 github.com/weaveworks/go-checkpoint v0.0.0-20170503165305-ebbb8b0518ab/go.mod h1:qkbvw5GPibQ/Nf7IZJL0UoLwmJ6858b4S/hUWRd+cH4=
 github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=

pkg/cluster/kubernetes/resource/load.go

@@ -8,9 +8,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/pkg/errors"
 	sops "go.mozilla.org/sops/v3"
 	"go.mozilla.org/sops/v3/decrypt"
-	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -29,37 +29,46 @@ func Load(base string, paths []string, sopsEnabled bool) (map[string]KubeManifes
 	}
 	for _, root := range paths {
 		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return errors.Wrapf(err, "walking %q for yaml files", path)
+			if info.IsDir() {
+				if charts.isDirChart(path) {
+					return filepath.SkipDir
+				}
+				if err != nil {
+					return errors.Wrapf(err, "walking dir %q for yaml files", path)
+				}
+				return nil
 			}
 
-			if charts.isDirChart(path) {
-				return filepath.SkipDir
+			// No need to check for errors for files we are not interested in anyway.
+			if filepath.Ext(path) != ".yaml" && filepath.Ext(path) != ".yml" {
+				return nil
 			}
-
 			if charts.isPathInChart(path) {
 				return nil
 			}
 
-			if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" {
-				bytes, err := loadFile(path, sopsEnabled)
-				if err != nil {
-					return errors.Wrapf(err, "unable to read file at %q", path)
-				}
-				source, err := filepath.Rel(base, path)
-				if err != nil {
-					return errors.Wrapf(err, "path to scan %q is not under base %q", path, base)
-				}
-				docsInFile, err := ParseMultidoc(bytes, source)
-				if err != nil {
-					return err
-				}
-				for id, obj := range docsInFile {
-					if alreadyDefined, ok := objs[id]; ok {
-						return fmt.Errorf(`duplicate definition of '%s' (in %s and %s)`, id, alreadyDefined.Source(), source)
-					}
-					objs[id] = obj
+			if err != nil {
+				return errors.Wrapf(err, "walking file %q for yaml docs", path)
+			}
+
+			// Load file
+			bytes, err := ioutil.ReadFile(path)
+			if err != nil {
+				return errors.Wrapf(err, "unable to read file at %q", path)
+			}
+			source, err := filepath.Rel(base, path)
+			if err != nil {
+				return errors.Wrapf(err, "path to scan %q is not under base %q", path, base)
+			}
+			docsInFile, err := ParseMultidoc(bytes, source)
+			if err != nil {
+				return err
+			}
+			for id, obj := range docsInFile {
+				if alreadyDefined, ok := objs[id]; ok {
+					return fmt.Errorf(`duplicate definition of '%s' (in %s and %s)`, id, alreadyDefined.Source(), source)
 				}
+				objs[id] = obj
 			}
 			return nil
 		})
@@ -71,13 +80,19 @@ func Load(base string, paths []string, sopsEnabled bool) (map[string]KubeManifes
 	return objs, nil
 }
 
+// chartTracker keeps track of paths that contain Helm charts in them.
 type chartTracker map[string]bool
 
 func newChartTracker(root string) (chartTracker, error) {
-	var chartdirs = make(map[string]bool)
+	chartdirs := make(chartTracker)
+	// Enumerate directories that contain charts. This will never
+	// return an error since our callback function swallows it.
 	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
-			return errors.Wrapf(err, "walking %q for charts", path)
+			// If a file or directory cannot be walked now we presume it will
+			// also not be available for walking when looking for yamels. If
+			// we do need access to it we can raise the error there.
+			return nil
 		}
 
 		if info.IsDir() && looksLikeChart(path) {
@@ -90,8 +105,7 @@ func newChartTracker(root string) (chartTracker, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	return chartTracker(chartdirs), nil
+	return chartdirs, nil
 }
 
 func (c chartTracker) isDirChart(path string) bool {
@@ -182,7 +196,7 @@ func ParseMultidoc(multidoc []byte, source string) (map[string]KubeManifest, err
 	return objs, nil
 }
 
-// loadFile attempts to load a file from the path supplied. If sopsEnabled is set, 
+// loadFile attempts to load a file from the path supplied. If sopsEnabled is set,
 // it will try to decrypt it before returning the data
 func loadFile(path string, sopsEnabled bool) ([]byte, error) {
 	bytes, err := ioutil.ReadFile(path)