diff --git a/api/internal/git/repospec.go b/api/internal/git/repospec.go index a1c29e6c71..99b8e76df1 100644 --- a/api/internal/git/repospec.go +++ b/api/internal/git/repospec.go @@ -21,6 +21,12 @@ import ( // with a unique name that isn't created until clone time. const notCloned = filesys.ConfirmedDir("/notCloned") +func Schemes() []string { + return []string{ + "ssh://", "https://", "http://", + } +} + // RepoSpec specifies a git repository and a branch and path therein. type RepoSpec struct { // Raw, original spec, used to look for cycles. @@ -240,6 +246,7 @@ func parseHostSpec(n string) (string, string) { } // If host is a http(s) or ssh URL, grab the domain part. + // TODO: replace with Schemes for _, p := range []string{ "ssh://", "https://", "http://"} { if strings.HasSuffix(host, p) { diff --git a/api/internal/localizer/errors.go b/api/internal/localizer/errors.go new file mode 100644 index 0000000000..a12513bf10 --- /dev/null +++ b/api/internal/localizer/errors.go @@ -0,0 +1,34 @@ +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package localizer + +import ( + "fmt" +) + +type InvalidRootError struct{} + +func (ir InvalidRootError) Error() string { + return "invalid root reference" +} + +type LocalizeDirExistsError struct{} + +func (lde LocalizeDirExistsError) Error() string { + return LocalizeDir + " localize directory already exists" +} + +type NoRefError struct { + Root string +} + +func (nr NoRefError) Error() string { + return fmt.Sprintf("localize remote root %q missing ref query string parameter", nr.Root) +} + +// prefixRelErrWhenContains returns a prefix for the error in the event that filepath.Rel(basePath, targPath) returns one, +// where basePath contains targPath +func prefixRelErrWhenContains(basePath string, targPath string) string { + return fmt.Sprintf("cannot find path from directory %q to %q inside directory", basePath, targPath) +} diff --git a/api/internal/localizer/localizer.go b/api/internal/localizer/localizer.go new file mode 100644 index 0000000000..f1c25b0692 --- /dev/null +++ b/api/internal/localizer/localizer.go @@ -0,0 +1,279 @@ +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package localizer + +import ( + "log" + "path/filepath" + "strings" + + "sigs.k8s.io/kustomize/api/ifc" + plgnsLdr "sigs.k8s.io/kustomize/api/internal/plugins/loader" + "sigs.k8s.io/kustomize/api/internal/target" + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/loader" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/yaml" +) + +// Localizer encapsulates all state needed to localize the root at ldr. +type Localizer struct { + fSys filesys.FileSystem + + // kusttarget fields + validator ifc.Validator + rFactory *resmap.Factory + pLdr *plgnsLdr.Loader + + // all localize directories created + localizeDirs map[filesys.ConfirmedDir]struct{} + + // should be LocLoader + ldr ifc.Loader + + // destination directory in newDir that mirrors ldr's current root. + dst filesys.ConfirmedDir +} + +// NewLocalizer is the factory method for Localizer +func NewLocalizer(ldr *Loader, validator ifc.Validator, rFactory *resmap.Factory, pLdr *plgnsLdr.Loader) (*Localizer, error) { + toDst, err := filepath.Rel(ldr.args.Scope.String(), ldr.Root()) + if err != nil { + log.Fatalf("%s: %s", prefixRelErrWhenContains(ldr.args.Scope.String(), ldr.Root()), err.Error()) + } + dst := ldr.args.NewDir.Join(toDst) + if err = ldr.fSys.MkdirAll(dst); err != nil { + return nil, errors.WrapPrefixf(err, "unable to create directory in localize destination") + } + return &Localizer{ + fSys: ldr.fSys, + validator: validator, + rFactory: rFactory, + pLdr: pLdr, + localizeDirs: make(map[filesys.ConfirmedDir]struct{}), + ldr: ldr, + dst: filesys.ConfirmedDir(dst), + }, nil +} + +// Localize localizes the root that lt is at +func (lc *Localizer) Localize() error { + kt := target.NewKustTarget(lc.ldr, lc.validator, lc.rFactory, lc.pLdr) + err := kt.Load() + if err != nil { + return errors.Wrap(err) + } + + kust, err := lc.processKust(kt) + if err != nil { + return err + } + + content, err := yaml.Marshal(kust) + if err != nil { + return errors.WrapPrefixf(err, "unable to serialize localized kustomization file") + } + if err = lc.fSys.WriteFile(lc.dst.Join(konfig.DefaultKustomizationFileName()), content); err != nil { + return errors.WrapPrefixf(err, "unable to write localized kustomization file") + } + return nil +} + +// processKust returns a copy of the kustomization at kt with all paths localized. +func (lc *Localizer) processKust(kt *target.KustTarget) (*types.Kustomization, error) { + kust := kt.Kustomization() + + for name, field := range map[string]*struct { + Mapper func(string) (string, error) + Paths []string + }{ + "resources": { + lc.localizePath, + kust.Resources, + }, + "components": { + lc.localizeDir, + kust.Components, + }, + "crds": { + lc.localizeFile, + kust.Crds, + }, + "configurations": { + lc.localizeFile, + kust.Configurations, + }, + } { + for i, path := range field.Paths { + newPath, err := field.Mapper(path) + if err != nil { + return &kust, errors.WrapPrefixf(err, "unable to localize %s path", name) + } + field.Paths[i] = newPath + } + } + + if path, exists := kust.OpenAPI["path"]; exists { + newPath, err := lc.localizeFile(path) + if err != nil { + return &kust, errors.WrapPrefixf(err, "unable to localize openapi path") + } + kust.OpenAPI["path"] = newPath + } + + for i := range kust.ConfigMapGenerator { + if err := localizeGenerator(lc, &kust.ConfigMapGenerator[i].GeneratorArgs); err != nil { + return nil, errors.WrapPrefixf(err, "unable to localize configMapGenerator") + } + } + for i := range kust.SecretGenerator { + if err := localizeGenerator(lc, &kust.SecretGenerator[i].GeneratorArgs); err != nil { + return nil, errors.WrapPrefixf(err, "unable to localize secretGenerator") + } + } + for name, patches := range map[string][]types.Patch{ + "patches": kust.Patches, + "patchesJson6902": kust.PatchesJson6902, + } { + for i := range patches { + if patches[i].Path != "" { + newPath, err := lc.localizeFile(patches[i].Path) + if err != nil { + return nil, errors.WrapPrefixf(err, "unable to localize %s path", name) + } + patches[i].Path = newPath + } + } + } + if err := localizePatchesStrategicMerge(lc, kust.PatchesStrategicMerge); err != nil { + return nil, err + } + if err := localizeReplacements(lc, kust.Replacements); err != nil { + return nil, err + } + + return &kust, nil +} + +// localizePath localizes path, a root or file, and returns the localized path +func (lc *Localizer) localizePath(path string) (string, error) { + locPath, err := lc.localizeDir(path) + if errors.Is(err, InvalidRootError{}) { + locPath, err = lc.localizeFile(path) + } + if err != nil { + return "", err + } + return locPath, nil +} + +// localizeFile localizes file path and returns the localized path +func (lc *Localizer) localizeFile(path string) (string, error) { + content, err := lc.ldr.Load(path) + if err != nil { + return "", errors.Wrap(err) + } + + var locPath string + if loader.HasRemoteFileScheme(path) { + if !lc.addLocalizeDir() { + return "", errors.Errorf("cannot localize remote %q: %w at %q", path, LocalizeDirExistsError{}, lc.ldr.Root()) + } + lc.localizeDirs[lc.dst] = struct{}{} + locPath = locFilePath(path) + } else { // path must be relative; subject to change in beta + // avoid symlinks; only write file corresponding to actual location in root + // avoid path that Load() shows to be in root, but may traverse outside + // temporarily; for example, ../root/config; problematic for rename and + // relocation + locPath = cleanFilePath(lc.fSys, filesys.ConfirmedDir(lc.ldr.Root()), path) + if !lc.guardLocalizeDir(locPath) { + abs := filepath.Join(lc.ldr.Root(), locPath) + return "", errors.Errorf("invalid local file path %q at %q: %w", path, abs, LocalizeDirExistsError{}) + } + } + cleanPath := lc.dst.Join(locPath) + if err = lc.fSys.MkdirAll(filepath.Dir(cleanPath)); err != nil { + return "", errors.WrapPrefixf(err, "unable to create directories to localize file %q", path) + } + if err = lc.fSys.WriteFile(cleanPath, content); err != nil { + return "", errors.WrapPrefixf(err, "unable to localize file %q", path) + } + return locPath, nil +} + +// localizeDir localizes root path and returns the localized path +func (lc *Localizer) localizeDir(path string) (string, error) { + ldr, err := lc.ldr.New(path) + if err != nil { + return "", errors.Wrap(err) + } + defer func() { _ = ldr.Cleanup() }() + + var locPath string + if repo, isRemote := ldr.Repo(); isRemote { + if !lc.addLocalizeDir() { + return "", errors.Errorf("cannot localize remote %q: %w at %q", path, LocalizeDirExistsError{}, lc.ldr.Root()) + } + locPath = locRootPath(path, filesys.ConfirmedDir(repo), filesys.ConfirmedDir(ldr.Root())) + } else { + locPath, err = filepath.Rel(lc.ldr.Root(), ldr.Root()) + if err != nil { + //nolint:gocritic // should never occur, but if hit, something fundamentally wrong; should immediately exit + log.Fatalf("rel path error for 2 navigable roots %q, %q: %s", lc.ldr.Root(), ldr.Root(), err.Error()) + } + if !lc.guardLocalizeDir(locPath) { + return "", errors.Errorf("invalid local root path %q at %q: %w", path, ldr.Root(), LocalizeDirExistsError{}) + } + } + newDst := lc.dst.Join(locPath) + if err = lc.fSys.MkdirAll(newDst); err != nil { + return "", errors.WrapPrefixf(err, "unable to create root %q in localize destination", path) + } + err = (&Localizer{ + fSys: lc.fSys, + validator: lc.validator, + rFactory: lc.rFactory, + pLdr: lc.pLdr, + localizeDirs: lc.localizeDirs, + ldr: ldr, + dst: filesys.ConfirmedDir(newDst), + }).Localize() + if err != nil { + return "", errors.WrapPrefixf(err, "unable to localize root %q", path) + } + return locPath, nil +} + +// addLocalizeDir returns whether it is able to add a localize directory at dst +func (lc *Localizer) addLocalizeDir() bool { + if _, exists := lc.localizeDirs[lc.dst]; !exists && lc.fSys.Exists(lc.dst.Join(LocalizeDir)) { + return false + } + lc.localizeDirs[lc.dst] = struct{}{} + return true +} + +// guardLocalizeDir returns false if local path enters a localize directory, and true otherwise +func (lc *Localizer) guardLocalizeDir(path string) bool { + var prefix string + for _, dir := range strings.Split(path, string(filepath.Separator)) { + parent := lc.dst.Join(prefix) + // if never processed parent, inner directories cannot be localize directories + if !lc.fSys.Exists(parent) { + return true + } + prefix = filepath.Join(prefix, dir) + if dir != LocalizeDir { + continue + } + if _, exists := lc.localizeDirs[filesys.ConfirmedDir(parent)]; exists { + return false + } + } + return true +} diff --git a/api/internal/localizer/localizer_test.go b/api/internal/localizer/localizer_test.go new file mode 100644 index 0000000000..aff2a248bd --- /dev/null +++ b/api/internal/localizer/localizer_test.go @@ -0,0 +1,399 @@ +// Copyright 2022 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package localizer_test + +import ( + "bytes" + "log" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/kustomize/api/hasher" + "sigs.k8s.io/kustomize/api/internal/localizer" + "sigs.k8s.io/kustomize/api/internal/plugins/loader" + "sigs.k8s.io/kustomize/api/internal/validate" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +func makeMemoryFs(t *testing.T) filesys.FileSystem { + t.Helper() + req := require.New(t) + + fSys := filesys.MakeFsInMemory() + req.NoError(fSys.MkdirAll("/a/b")) + req.NoError(fSys.WriteFile("/a/pod.yaml", []byte("pod configuration"))) + + dirChain := "/alpha/beta/gamma/delta" + req.NoError(fSys.MkdirAll(dirChain)) + req.NoError(fSys.WriteFile(filepath.Join(dirChain, "deployment.yaml"), []byte("deployment configuration"))) + req.NoError(fSys.Mkdir("/alpha/beta/say")) + return fSys +} + +func addFiles(t *testing.T, fSys filesys.FileSystem, parentDir string, files map[string]string) { + t.Helper() + + // in-memory file system makes all necessary dirs when writing files + for file, content := range files { + require.NoError(t, fSys.WriteFile(filepath.Join(parentDir, file), []byte(content))) /**/ + } +} + +func RunLocalize(t *testing.T, fSys filesys.FileSystem, target string, scope string, newDir string) error { + t.Helper() + + // no need to re-test LocLoader + ldr, _, err := localizer.NewLoader(target, scope, newDir, fSys) + require.NoError(t, err) + rmFactory := resmap.NewFactory(resource.NewFactory(&hasher.Hasher{})) + lc, err := localizer.NewLocalizer( + ldr, + validate.NewFieldValidator(), + rmFactory, + // file system can be in memory, as plugin configuration will prevent the use of file system anyway + loader.NewLoader(types.DisabledPluginConfig(), rmFactory, fSys)) + require.NoError(t, err) + return errors.Wrap(lc.Localize()) +} + +func TestPatchStrategicMergeOnFile(t *testing.T) { + req := require.New(t) + + var buf bytes.Buffer + log.SetOutput(&buf) + fSys := makeMemoryFs(t) + + // tests both inline and file patches + // tests localize handles nested directory and winding path + files := map[string]string{ + "kustomization.yaml": `patchesStrategicMerge: +- ../beta/say/patch.yaml +- |- + apiVersion: v1 + metadata: + name: myPod + kind: Pod + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +resources: +- localized-files`, + // in the absence of remote references, localize directory name can be used by other files + "localized-files": "deployment configuration", + "say/patch.yaml": `apiVersion: v1 +metadata: + name: myPod +kind: Pod +spec: + containers: + - name: app + image: images.my-company.example/app:v4`, + } + addFiles(t, fSys, "/alpha/beta", files) + err := RunLocalize(t, fSys, "/alpha/beta", "", "/alpha/newDir") + req.NoError(err) + req.Empty(buf.String()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/alpha/beta", files) + files["kustomization.yaml"] = `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patchesStrategicMerge: +- say/patch.yaml +- |- + apiVersion: v1 + metadata: + name: myPod + kind: Pod + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +resources: +- localized-files +` + // directories in scope, but not referenced should not be copied to destination + addFiles(t, fSysExpected, "/alpha/newDir", files) + req.Equal(fSysExpected, fSys) +} + +func TestSecretGenerator(t *testing.T) { + req := require.New(t) + + var buf bytes.Buffer + log.SetOutput(&buf) + fSys := makeMemoryFs(t) + + files := map[string]string{ + // test configurations + // test generatorOptions does not affect secretGenerator + // show that localize currently replaces deprecated kustomization fields + "kustomization.yaml": ` +configurations: +- name-suffix-config +nameSuffix: -my +generatorOptions: + disableNameSuffixHash: true +secretGenerator: +- name: my-secret + env: data + options: + labels: + type: fruit +`, + // test no file extensions + "name-suffix-config": "nameSuffix field specs", + "data": "APPLE=orange", + } + addFiles(t, fSys, "/alpha/beta", files) + req.NoError(RunLocalize(t, fSys, "/alpha/beta", "", "/alpha/newDir")) + req.Empty(buf.String()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/alpha/beta", files) + files["kustomization.yaml"] = `apiVersion: kustomize.config.k8s.io/v1beta1 +configurations: +- name-suffix-config +generatorOptions: + disableNameSuffixHash: true +kind: Kustomization +nameSuffix: -my +secretGenerator: +- envs: + - data + name: my-secret + options: + labels: + type: fruit +` + addFiles(t, fSysExpected, "/alpha/newDir", files) + req.Equal(fSysExpected, fSys) +} + +func TestComponents(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + fSys := makeMemoryFs(t) + + // components test directory references + files := map[string]string{ + // winding directory path + "a/kustomization.yaml": ` +components: +- b/../../alpha/beta/.. +- localized-files +resources: +- pod.yaml +- job.yaml +`, + + "a/job.yaml": "job configuration", + + // should recognize different kustomization names + // inline and file replacements + "alpha/kustomization.yml": `apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +replacements: +- source: + fieldPath: metadata.name + kind: Job + targets: + - fieldPaths: + - metadata.name + select: + kind: Pod +- path: my-replacement.yaml +`, + + "alpha/my-replacement.yaml": "replacement configuration", + + // test inline and file patchesJson6902 + // in the absence of remote references, directories can share localize directory name + "a/localized-files/Kustomization": `apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patchesJson6902: +- patch: |- + - op: replace + path: /spec/containers/0/name + value: my-nginx + target: + kind: Pod +- path: patch.yaml + target: + kind: Pod +`, + + "a/localized-files/patch.yaml": "patch configuration", + } + addFiles(t, fSys, "/", files) + + err := RunLocalize(t, fSys, "/a", "/", "") + require.NoError(t, err) + require.Empty(t, buf.String()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/", files) + + filesExpected := map[string]string{ + "a/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +components: +- ../alpha +- localized-files +kind: Kustomization +resources: +- pod.yaml +- job.yaml +`, + "a/pod.yaml": "pod configuration", + "a/job.yaml": files["a/job.yaml"], + "alpha/kustomization.yaml": files["alpha/kustomization.yml"], + "alpha/my-replacement.yaml": files["alpha/my-replacement.yaml"], + "a/localized-files/kustomization.yaml": files["a/localized-files/Kustomization"], + "a/localized-files/patch.yaml": files["a/localized-files/patch.yaml"], + } + addFiles(t, fSysExpected, "/localized-a", filesExpected) + require.Equal(t, fSysExpected, fSys) +} + +func TestOpenAPI(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + fSys := makeMemoryFs(t) + + files := map[string]string{ + // test patches + "a/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +openapi: + path: custom-api.json +patches: +- patch: |- + - op: replace + path: /spec/count + value: 2 + target: + kind: CustomObject +- path: patch.yaml +resources: +- custom-object.yaml +`, + + "a/custom-api.json": "schema", + "a/patch.yaml": "strategic merge patch", + "a/custom-object.yaml": "custom object configuration", + } + addFiles(t, fSys, "/", files) + + require.NoError(t, RunLocalize(t, fSys, "/a", "/", "")) + require.Empty(t, buf.String()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/", files) + addFiles(t, fSysExpected, "/localized-a", files) + require.Equal(t, fSysExpected, fSys) +} + +func TestNestedRoots(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + + fSys := makeMemoryFs(t) + files := map[string]string{ + // both file and directory resources + // kustomization fields without paths should also be copied + "beta/gamma/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namePrefix: nested-roots- +resources: +- delta/deployment.yaml +- ../say +`, + + // configMapGenerator with envs and files, with both the default filename and keys + "beta/say/kustomization.yaml": ` +resources: +- ../gamma/./delta/ +- ../../beta/gamma/delta/epsilon +configMapGenerator: +- name: my-config-map + behavior: create + files: + - application.properties + - environment.properties=../gamma/../say/weird-name + literals: + - THIS_KEY=/really/does/not/matter + envs: + - ./more.properties`, + + "beta/say/application.properties": "application properties", + + "beta/say/weird-name": "weird-name properties", + + "beta/say/more.properties": "more properties", + + // test crds + "beta/gamma/delta/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +commonLabels: + label: value +crds: +- epsilon/type-new-kind.yaml +kind: Kustomization +resources: +- new-kind.yaml +`, + + "beta/gamma/delta/new-kind.yaml": "new-kind configuration", + + "beta/gamma/delta/epsilon/kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +commonLabels: + label: anotherValue +crds: +- type-new-kind.yaml +kind: Kustomization +resources: +- new-kind.yaml +`, + + "beta/gamma/delta/epsilon/new-kind.yaml": "another new-kind configuration", + + // referenced more than once + "beta/gamma/delta/epsilon/type-new-kind.yaml": "new-kind definition", + } + addFiles(t, fSys, "/alpha", files) + err := RunLocalize(t, fSys, "/alpha/beta/gamma", "/alpha", "/alpha/beta/gamma/delta/newDir") + require.NoError(t, err) + require.Empty(t, buf.String()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/alpha", files) + files["beta/say/kustomization.yaml"] = `apiVersion: kustomize.config.k8s.io/v1beta1 +configMapGenerator: +- behavior: create + envs: + - more.properties + files: + - application.properties + - environment.properties=weird-name + literals: + - THIS_KEY=/really/does/not/matter + name: my-config-map +kind: Kustomization +resources: +- ../gamma/delta +- ../gamma/delta/epsilon +` + files["beta/gamma/delta/deployment.yaml"] = "deployment configuration" + addFiles(t, fSysExpected, "/alpha/beta/gamma/delta/newDir", files) + require.Equal(t, fSysExpected, fSys) +} diff --git a/api/internal/localizer/locloader.go b/api/internal/localizer/locloader.go index 76188847f1..1a5c6aad10 100644 --- a/api/internal/localizer/locloader.go +++ b/api/internal/localizer/locloader.go @@ -4,11 +4,11 @@ package localizer import ( - "log" "path/filepath" "sigs.k8s.io/kustomize/api/ifc" "sigs.k8s.io/kustomize/api/internal/git" + "sigs.k8s.io/kustomize/api/internal/utils" "sigs.k8s.io/kustomize/api/loader" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/filesys" @@ -16,98 +16,93 @@ import ( const dstPrefix = "localized" -// LocArgs holds localize arguments -type LocArgs struct { +// Args holds localize arguments +type Args struct { // target; local copy if remote Target filesys.ConfirmedDir - // directory that bounds target's local references, empty string if target is remote + // directory that bounds target's local references + // repo directory of local copy if target is remote Scope filesys.ConfirmedDir // localize destination NewDir filesys.ConfirmedDir } -// locLoader is the Loader for kustomize localize. It is an ifc.Loader that enforces localize constraints. -type locLoader struct { +// Loader is the Loader for kustomize localize. It is an ifc.Loader that enforces localize constraints. +type Loader struct { fSys filesys.FileSystem - args *LocArgs + args *Args - // loader at locLoader's current directory + // loader at Loader's current directory ifc.Loader - // whether locLoader and all its ancestors are the result of local references + // whether Loader and all its ancestors are the result of local references local bool } -var _ ifc.Loader = &locLoader{} +var _ ifc.Loader = &Loader{} -// NewLocLoader is the factory method for Loader, under localize constraints, at targetArg. For invalid localize arguments, -// NewLocLoader returns an error. -func NewLocLoader(targetArg string, scopeArg string, newDirArg string, fSys filesys.FileSystem) (ifc.Loader, LocArgs, error) { +// NewLoader is the factory method for Loader, under localize constraints, at rawTarget. +func NewLoader(rawTarget string, rawScope string, rawDst string, fSys filesys.FileSystem) (*Loader, Args, error) { // check earlier to avoid cleanup - repoSpec, err := git.NewRepoSpecFromURL(targetArg) + repoSpec, err := git.NewRepoSpecFromURL(rawTarget) if err == nil && repoSpec.Ref == "" { - return nil, LocArgs{}, - errors.Errorf("localize remote root '%s' missing ref query string parameter", targetArg) + return nil, Args{}, errors.Wrap(NoRefError{rawTarget}) } // for security, should enforce load restrictions - ldr, err := loader.NewLoader(loader.RestrictionRootOnly, targetArg, fSys) + ldr, err := loader.NewLoader(loader.RestrictionRootOnly, rawTarget, fSys) if err != nil { - return nil, LocArgs{}, errors.WrapPrefixf(err, "unable to establish localize target '%s'", targetArg) + return nil, Args{}, errors.WrapPrefixf(err, "unable to establish localize target %q", rawTarget) } - scope, err := establishScope(scopeArg, targetArg, ldr, fSys) + scope, err := establishScope(rawScope, rawTarget, ldr, fSys) if err != nil { _ = ldr.Cleanup() - return nil, LocArgs{}, errors.WrapPrefixf(err, "invalid localize scope '%s'", scopeArg) + return nil, Args{}, errors.WrapPrefixf(err, "invalid localize scope %q", rawScope) } - newDir, err := createNewDir(newDirArg, ldr, repoSpec, fSys) + newDir, err := createNewDir(rawDst, ldr, repoSpec, fSys) if err != nil { _ = ldr.Cleanup() - return nil, LocArgs{}, errors.WrapPrefixf(err, "invalid localize destination '%s'", newDirArg) + return nil, Args{}, errors.WrapPrefixf(err, "invalid localize destination %q", rawDst) } - args := LocArgs{ + args := Args{ Target: filesys.ConfirmedDir(ldr.Root()), Scope: scope, NewDir: newDir, } - return &locLoader{ + _, isRemote := ldr.Repo() + return &Loader{ fSys: fSys, args: &args, Loader: ldr, - local: scope != "", + local: !isRemote, }, args, nil } // Load returns the contents of path if path is a valid localize file. -// Otherwise, Load returns an error. -func (ll *locLoader) Load(path string) ([]byte, error) { +func (ll *Loader) Load(path string) ([]byte, error) { // checks in root, and thus in scope content, err := ll.Loader.Load(path) if err != nil { return nil, errors.WrapPrefixf(err, "invalid file reference") } if filepath.IsAbs(path) { - return nil, errors.Errorf("absolute paths not yet supported in alpha: file path '%s' is absolute", path) + return nil, errors.Errorf("absolute paths not yet supported in alpha: file path %q is absolute", path) } - if ll.local { - abs := filepath.Join(ll.Root(), path) - dir, f, err := ll.fSys.CleanedAbs(abs) - if err != nil { - // should never happen - log.Fatalf(errors.WrapPrefixf(err, "cannot clean validated file path '%s'", abs).Error()) - } + if !loader.HasRemoteFileScheme(path) && ll.local { + cleanPath := cleanFilePath(ll.fSys, filesys.ConfirmedDir(ll.Root()), path) + cleanAbs := filepath.Join(ll.Root(), cleanPath) + dir := filesys.ConfirmedDir(filepath.Dir(cleanAbs)) // target cannot reference newDir, as this load would've failed prior to localize; // not a problem if remote because then reference could only be in newDir if repo copy, // which will be cleaned, is inside newDir if dir.HasPrefix(ll.args.NewDir) { - return nil, errors.Errorf( - "file path '%s' references into localize destination '%s'", dir.Join(f), ll.args.NewDir) + return nil, errors.Errorf("file %q at %q enters localize destination %q", path, cleanAbs, ll.args.NewDir) } } return content, nil @@ -115,29 +110,34 @@ func (ll *locLoader) Load(path string) ([]byte, error) { // New returns a Loader at path if path is a valid localize root. // Otherwise, New returns an error. -func (ll *locLoader) New(path string) (ifc.Loader, error) { - repoSpec, err := git.NewRepoSpecFromURL(path) - if err == nil && repoSpec.Ref == "" { - return nil, errors.Errorf("localize remote root '%s' missing ref query string parameter", path) - } - +func (ll *Loader) New(path string) (ifc.Loader, error) { ldr, err := ll.Loader.New(path) + // timeout indicates path is a root, not a file + if utils.IsErrTimeout(err) { + if !hasRef(path) { + return nil, errors.Wrap(NoRefError{path}) + } + return nil, errors.Wrap(err) + } + // otherwise, invalid root; upper layer should try file if err != nil { - return nil, errors.WrapPrefixf(err, "invalid root reference") + return nil, errors.Errorf("%w: %s", InvalidRootError{}, err.Error()) } - - var isRemote bool - if _, isRemote = ldr.Repo(); !isRemote { - if ll.local && !filesys.ConfirmedDir(ldr.Root()).HasPrefix(ll.args.Scope) { - return nil, errors.Errorf("root '%s' outside localize scope '%s'", ldr.Root(), ll.args.Scope) + _, isRemote := ldr.Repo() + if isRemote { + if !hasRef(path) { + return nil, errors.Wrap(NoRefError{path}) + } + } else if ll.local { + if !filesys.ConfirmedDir(ldr.Root()).HasPrefix(ll.args.Scope) { + return nil, errors.Errorf("root %q at %q outside localize scope %q", path, ldr.Root(), ll.args.Scope) } - if ll.local && filesys.ConfirmedDir(ldr.Root()).HasPrefix(ll.args.NewDir) { - return nil, errors.Errorf( - "root '%s' references into localize destination '%s'", ldr.Root(), ll.args.NewDir) + if filesys.ConfirmedDir(ldr.Root()).HasPrefix(ll.args.NewDir) { + return nil, errors.Errorf("root %q at %q enters localize destination %q", path, ldr.Root(), ll.args.NewDir) } } - return &locLoader{ + return &Loader{ fSys: ll.fSys, args: ll.args, Loader: ldr, diff --git a/api/internal/localizer/locloader_test.go b/api/internal/localizer/locloader_test.go index 7d0f58858e..1bf19ae691 100644 --- a/api/internal/localizer/locloader_test.go +++ b/api/internal/localizer/locloader_test.go @@ -10,28 +10,14 @@ import ( "github.com/stretchr/testify/require" "sigs.k8s.io/kustomize/api/ifc" - lclzr "sigs.k8s.io/kustomize/api/internal/localizer" + "sigs.k8s.io/kustomize/api/internal/localizer" "sigs.k8s.io/kustomize/kyaml/filesys" ) const dstPrefix = "localized" -func makeMemoryFs(t *testing.T) filesys.FileSystem { - t.Helper() - req := require.New(t) - - fSys := filesys.MakeFsInMemory() - req.NoError(fSys.MkdirAll("/a/b")) - req.NoError(fSys.WriteFile("/a/kustomization.yaml", []byte("/a"))) - - dirChain := "/alpha/beta/gamma/delta" - req.NoError(fSys.MkdirAll(dirChain)) - req.NoError(fSys.WriteFile(dirChain+"/kustomization.yaml", []byte(dirChain))) - req.NoError(fSys.Mkdir("/alpha/beta/c")) - return fSys -} - -func checkNewLocLoader(req *require.Assertions, ldr ifc.Loader, args *lclzr.LocArgs, target string, scope string, newDir string, fSys filesys.FileSystem) { +func checkNewLoader(req *require.Assertions, ldr *localizer.Loader, args *localizer.Args, target string, + scope string, newDir string, fSys filesys.FileSystem) { checkLoader(req, ldr, target) checkLocArgs(req, args, target, scope, newDir, fSys) } @@ -39,11 +25,12 @@ func checkNewLocLoader(req *require.Assertions, ldr ifc.Loader, args *lclzr.LocA func checkLoader(req *require.Assertions, ldr ifc.Loader, root string) { req.Equal(root, ldr.Root()) repo, isRemote := ldr.Repo() - req.Equal(false, isRemote) - req.Equal("", repo) + req.False(isRemote) + req.Empty(repo) } -func checkLocArgs(req *require.Assertions, args *lclzr.LocArgs, target string, scope string, newDir string, fSys filesys.FileSystem) { +func checkLocArgs(req *require.Assertions, args *localizer.Args, target string, scope string, newDir string, + fSys filesys.FileSystem) { req.Equal(target, args.Target.String()) req.Equal(scope, args.Scope.String()) req.Equal(newDir, args.NewDir.String()) @@ -57,18 +44,18 @@ func TestLocalLoadNewAndCleanup(t *testing.T) { var buf bytes.Buffer log.SetOutput(&buf) // typical setup - ldr, args, err := lclzr.NewLocLoader("a", "/", "/newDir", fSys) + ldr, args, err := localizer.NewLoader("a", "/", "/newDir", fSys) req.NoError(err) - checkNewLocLoader(req, ldr, &args, "/a", "/", "/newDir", fSys) + checkNewLoader(req, ldr, &args, "/a", "/", "/newDir", fSys) fSysCopy := makeMemoryFs(t) req.NoError(fSysCopy.Mkdir("/newDir")) req.Equal(fSysCopy, fSys) // easy load directly in root - content, err := ldr.Load("kustomization.yaml") + content, err := ldr.Load("pod.yaml") req.NoError(err) - req.Equal([]byte("/a"), content) + req.Equal([]byte("pod configuration"), content) // typical sibling root reference sibLdr, err := ldr.New("../alpha") @@ -104,23 +91,23 @@ func TestNewLocLoaderDefaultForRootTarget(t *testing.T) { req := require.New(t) fSys := makeMemoryFs(t) - ldr, args, err := lclzr.NewLocLoader(params.target, params.scope, "", fSys) + ldr, args, err := localizer.NewLoader(params.target, params.scope, "", fSys) req.NoError(err) - checkNewLocLoader(req, ldr, &args, "/", "/", "/"+dstPrefix, fSys) + checkNewLoader(req, ldr, &args, "/", "/", "/"+dstPrefix, fSys) // file in root, but nested - content, err := ldr.Load("a/kustomization.yaml") + content, err := ldr.Load("a/pod.yaml") req.NoError(err) - req.Equal([]byte("/a"), content) + req.Equal([]byte("pod configuration"), content) childLdr, err := ldr.New("a") req.NoError(err) checkLoader(req, childLdr, "/a") // messy, uncleaned path - content, err = childLdr.Load("./../a/kustomization.yaml") + content, err = childLdr.Load("./../a/pod.yaml") req.NoError(err) - req.Equal([]byte("/a"), content) + req.Equal([]byte("pod configuration"), content) }) } } @@ -131,9 +118,9 @@ func TestNewMultiple(t *testing.T) { // default destination for non-file system root target // destination outside of scope - ldr, args, err := lclzr.NewLocLoader("/alpha/beta", "/alpha", "", fSys) + ldr, args, err := localizer.NewLoader("/alpha/beta", "/alpha", "", fSys) req.NoError(err) - checkNewLocLoader(req, ldr, &args, "/alpha/beta", "/alpha", "/"+dstPrefix+"-beta", fSys) + checkNewLoader(req, ldr, &args, "/alpha/beta", "/alpha", "/"+dstPrefix+"-beta", fSys) // nested child root that isn't cleaned descLdr, err := ldr.New("../beta/gamma/delta") @@ -141,9 +128,9 @@ func TestNewMultiple(t *testing.T) { checkLoader(req, descLdr, "/alpha/beta/gamma/delta") // upwards traversal - higherLdr, err := descLdr.New("../../c") + higherLdr, err := descLdr.New("../../say") req.NoError(err) - checkLoader(req, higherLdr, "/alpha/beta/c") + checkLoader(req, higherLdr, "/alpha/beta/say") } func makeWdFs(t *testing.T) map[string]filesys.FileSystem { @@ -192,7 +179,7 @@ func TestNewLocLoaderCwdNotRoot(t *testing.T) { req := require.New(t) fSys := makeWdFs(t)[test.wd] - ldr, args, err := lclzr.NewLocLoader(test.target, test.scope, test.newDir, fSys) + ldr, args, err := localizer.NewLoader(test.target, test.scope, test.newDir, fSys) req.NoError(err) checkLoader(req, ldr, "a/b/c/d/e") @@ -217,7 +204,7 @@ func TestNewLocLoaderFails(t *testing.T) { "/newDir", }, "file target": { - "/a/kustomization.yaml", + "/a/pod.yaml", "/", "/newDir", }, @@ -242,7 +229,7 @@ func TestNewLocLoaderFails(t *testing.T) { t.Run(name, func(t *testing.T) { var buf bytes.Buffer log.SetOutput(&buf) - _, _, err := lclzr.NewLocLoader(params.target, params.scope, params.dest, makeMemoryFs(t)) + _, _, err := localizer.NewLoader(params.target, params.scope, params.dest, makeMemoryFs(t)) require.Error(t, err) require.Empty(t, buf.String()) }) @@ -253,27 +240,47 @@ func TestNewFails(t *testing.T) { req := require.New(t) fSys := makeMemoryFs(t) - ldr, args, err := lclzr.NewLocLoader("/alpha/beta/gamma", "alpha", "alpha/beta/gamma/newDir", fSys) + ldr, args, err := localizer.NewLoader("/alpha/beta/gamma", "alpha", "alpha/beta/gamma/newDir", fSys) req.NoError(err) - checkNewLocLoader(req, ldr, &args, "/alpha/beta/gamma", "/alpha", "/alpha/beta/gamma/newDir", fSys) + checkNewLoader(req, ldr, &args, "/alpha/beta/gamma", "/alpha", "/alpha/beta/gamma/newDir", fSys) - cases := map[string]string{ - "outside scope": "../../../a", - "at dst": "newDir", - "ancestor": "../../beta", - "non-existent root": "delt", - "file": "delta/kustomization.yaml", + cases := map[string]*struct { + arg string + + // whether error is specific to localize + localizeError bool + }{ + "outside scope": { + "../../../a", + true, + }, + "at dst": { + "newDir", + true, + }, + "ancestor": { + arg: "../../beta", + }, + "non-existent root": { + arg: "delt", + }, + "file": { + arg: "delta/deployment.yaml", + }, } for name, root := range cases { - root := root t.Run(name, func(t *testing.T) { fSys := makeMemoryFs(t) - ldr, _, err := lclzr.NewLocLoader("/alpha/beta/gamma", "alpha", "alpha/beta/gamma/newDir", fSys) + ldr, _, err := localizer.NewLoader("/alpha/beta/gamma", "alpha", + "alpha/beta/gamma/newDir", fSys) require.NoError(t, err) - _, err = ldr.New(root) + _, err = ldr.New(root.arg) require.Error(t, err) + if !root.localizeError { + require.ErrorIs(t, err, localizer.InvalidRootError{}) + } }) } } @@ -282,16 +289,16 @@ func TestLoadFails(t *testing.T) { req := require.New(t) fSys := makeMemoryFs(t) - ldr, args, err := lclzr.NewLocLoader("./a/../a", "/a/../a", "/a/newDir", fSys) + ldr, args, err := localizer.NewLoader("./a/../a", "/a/../a", "/a/newDir", fSys) req.NoError(err) - checkNewLocLoader(req, ldr, &args, "/a", "/a", "/a/newDir", fSys) + checkNewLoader(req, ldr, &args, "/a", "/a", "/a/newDir", fSys) cases := map[string]string{ - "absolute path": "/a/kustomization.yaml", + "absolute path": "/a/pod.yaml", "directory": "b", "non-existent file": "kubectl.yaml", - "file outside root": "../alpha/beta/gamma/delta/kustomization.yaml", - "inside dst": "newDir/kustomization.yaml", + "file outside root": "../alpha/beta/gamma/delta/deployment.yaml", + "inside dst": "newDir/pod.yaml", } for name, file := range cases { file := file @@ -299,10 +306,10 @@ func TestLoadFails(t *testing.T) { req := require.New(t) fSys := makeMemoryFs(t) - ldr, _, err := lclzr.NewLocLoader("./a/../a", "/a/../a", "/a/newDir", fSys) + ldr, _, err := localizer.NewLoader("./a/../a", "/a/../a", "/a/newDir", fSys) req.NoError(err) - req.NoError(fSys.WriteFile("/a/newDir/kustomization.yaml", []byte("/a/newDir"))) + req.NoError(fSys.WriteFile("/a/newDir/pod.yaml", []byte("pod configuration"))) _, err = ldr.Load(file) req.Error(err) diff --git a/api/internal/localizer/util.go b/api/internal/localizer/util.go index 9c88b0912a..19e9d95e1d 100644 --- a/api/internal/localizer/util.go +++ b/api/internal/localizer/util.go @@ -4,53 +4,58 @@ package localizer import ( + "fmt" "log" + "net/url" "path/filepath" "strings" "sigs.k8s.io/kustomize/api/ifc" "sigs.k8s.io/kustomize/api/internal/git" + "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/filesys" ) -// establishScope returns the scope given localize arguments and targetLdr at targetArg -func establishScope(scopeArg string, targetArg string, targetLdr ifc.Loader, fSys filesys.FileSystem) (filesys.ConfirmedDir, error) { - if _, isRemote := targetLdr.Repo(); isRemote { - if scopeArg != "" { - return "", errors.Errorf("scope '%s' specified for remote localize target '%s'", scopeArg, targetArg) +const LocalizeDir = "localized-files" + +// establishScope returns the scope given localize arguments and targetLdr at rawTarget +func establishScope(rawScope string, rawTarget string, targetLdr ifc.Loader, fSys filesys.FileSystem) (filesys.ConfirmedDir, error) { + if repo, isRemote := targetLdr.Repo(); isRemote { + if rawScope != "" { + return "", errors.Errorf("scope %q specified for remote localize target %q", rawScope, rawTarget) } - return "", nil + return filesys.ConfirmedDir(repo), nil } // default scope - if scopeArg == "" { + if rawScope == "" { return filesys.ConfirmedDir(targetLdr.Root()), nil } - scope, err := filesys.ConfirmDir(fSys, scopeArg) + scope, err := filesys.ConfirmDir(fSys, rawScope) if err != nil { return "", errors.WrapPrefixf(err, "unable to establish localize scope") } if !filesys.ConfirmedDir(targetLdr.Root()).HasPrefix(scope) { - return scope, errors.Errorf("localize scope '%s' does not contain target '%s' at '%s'", - scopeArg, targetArg, targetLdr.Root()) + return scope, errors.Errorf("localize scope %q does not contain target %q at %q", + rawScope, rawTarget, targetLdr.Root()) } return scope, nil } // createNewDir returns the localize destination directory or error. Note that spec is nil if targetLdr is at local // target. -func createNewDir(newDirArg string, targetLdr ifc.Loader, spec *git.RepoSpec, fSys filesys.FileSystem) (filesys.ConfirmedDir, error) { - if newDirArg == "" { - newDirArg = defaultNewDir(targetLdr, spec) +func createNewDir(rawDst string, targetLdr ifc.Loader, spec *git.RepoSpec, fSys filesys.FileSystem) (filesys.ConfirmedDir, error) { + if rawDst == "" { + rawDst = defaultNewDir(targetLdr, spec) } - if fSys.Exists(newDirArg) { - return "", errors.Errorf("localize destination '%s' already exists", newDirArg) + if fSys.Exists(rawDst) { + return "", errors.Errorf("localize destination %q already exists", rawDst) } // destination directory must sit in an existing directory - if err := fSys.Mkdir(newDirArg); err != nil { + if err := fSys.Mkdir(rawDst); err != nil { return "", errors.WrapPrefixf(err, "unable to create localize destination directory") } - newDir, err := filesys.ConfirmDir(fSys, newDirArg) + newDir, err := filesys.ConfirmDir(fSys, rawDst) if err != nil { if errCleanup := fSys.RemoveAll(newDir.String()); errCleanup != nil { log.Printf("%s", errors.WrapPrefixf(errCleanup, "unable to clean localize destination").Error()) @@ -80,6 +85,15 @@ func defaultNewDir(targetLdr ifc.Loader, spec *git.RepoSpec) string { return strings.Join([]string{dstPrefix, targetDir}, "-") } +// hasRef checks if path url has ref query string parameter +func hasRef(path string) bool { + repoSpec, err := git.NewRepoSpecFromURL(path) + if err != nil { + log.Fatalf("%s: %s", "unable to parse validated root url", err.Error()) + } + return repoSpec.Ref != "" +} + // urlBase is the url equivalent of filepath.Base func urlBase(url string) string { cleaned := strings.TrimRight(url, "/") @@ -89,3 +103,163 @@ func urlBase(url string) string { } return cleaned[i+1:] } + +// cleanFilePath returns the cleaned relPath, a validated relative file path inside root +func cleanFilePath(fSys filesys.FileSystem, root filesys.ConfirmedDir, file string) string { + abs := root.Join(file) + dir, f, err := fSys.CleanedAbs(abs) + if err != nil { + log.Fatalf("cannot clean validated file path %q: %s", abs, err.Error()) + } + locPath, err := filepath.Rel(root.String(), dir.Join(f)) + if err != nil { + log.Fatalf("%s: %s", prefixRelErrWhenContains(root.String(), dir.Join(f)), err.Error()) + } + return locPath +} + +// locFilePath returns the relative localized path of validated file url fileURL +func locFilePath(fileURL string) string { + // file urls must have http or https scheme + u, err := url.Parse(fileURL) + if err != nil { + log.Fatalf("cannot parse validated file url %q: %s", fileURL, err.Error()) + } + + // preserve percent-encodings in case of sub-delims special meaning + // remove extraneous '..' parent directory dot-segments + path := filepath.Join(string(filepath.Separator), filepath.FromSlash(u.EscapedPath())) + + // raw github urls are the only type of file urls kustomize officially accepts, in which case + // path consists of org, repo, version, path in repo + + // host should not include userinfo or port + return filepath.Join(LocalizeDir, u.Hostname(), path) +} + +// locRootPath returns the relative localized path of the validated root url rootURL, where the local copy of its repo +// is at repoDir and the copy of its root is at rootDir +func locRootPath(rootURL string, repoDir filesys.ConfirmedDir, rootDir filesys.ConfirmedDir) string { + repoSpec, err := git.NewRepoSpecFromURL(rootURL) + if err != nil { + log.Fatalf("cannot parse validated repo url %q: %s", rootURL, err.Error()) + } + // calculate from copy instead of url to straighten symlinks + inRepo, err := filepath.Rel(repoDir.String(), rootDir.String()) + if err != nil { + log.Fatalf("%s: %s", prefixRelErrWhenContains(repoDir.String(), rootDir.String()), err.Error()) + } + // org, repo unlikely to contain dot-segments since repo is folder name when cloned + // git does not allow ref value to contain dot-segments + return filepath.Join(LocalizeDir, + parseDomain(repoSpec.Host), + filepath.FromSlash(repoSpec.OrgRepo), + filepath.FromSlash(repoSpec.Ref), + inRepo) +} + +// parseDomain returns the domain from git.RepoSpec Host +func parseDomain(host string) string { + if host == "gh:" { + return "github.com" + } + target := host + // remove scheme from target + for _, p := range git.Schemes() { + if strings.HasPrefix(target, p) { + target = target[len(p):] + break + } + } + // remove userinfo + if i := strings.Index(target, "@"); i > -1 { + target = target[i+1:] + } + // remove ssh path delimiter or port delimiter + if i := strings.Index(target, ":"); i > -1 { + target = target[:i] + } + // remove http path delimiter + return strings.TrimSuffix(target, "/") +} + +// ParseFileSource parses the source given. +// +// Acceptable formats include: +// 1. source-path: the basename will become the key name +// 2. source-name=source-path: the source-name will become the key name and +// source-path is the path to the key file. +// +// Key names cannot include '='. +func parseFileSource(source string) (keyName, filePath string, err error) { + numSeparators := strings.Count(source, "=") + switch { + case numSeparators == 0: + return filepath.Base(source), source, nil + case numSeparators == 1 && strings.HasPrefix(source, "="): + return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "=")) + case numSeparators == 1 && strings.HasSuffix(source, "="): + return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "=")) + case numSeparators > 1: + return "", "", errors.Errorf("key names or file paths cannot contain '='") + default: + components := strings.Split(source, "=") + return components[0], components[1], nil + } +} + +// localizeGenerator localizes the file paths on generator, which must not contain deprecated fields +func localizeGenerator(lc *Localizer, generator *types.GeneratorArgs) error { + for i, env := range generator.EnvSources { + newPath, err := lc.localizeFile(env) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize generator envs file") + } + generator.EnvSources[i] = newPath + } + for i, file := range generator.FileSources { + k, f, err := parseFileSource(file) + if err != nil { + return errors.Wrap(err) + } + newFile, err := lc.localizeFile(f) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize generator files path") + } + if f != file { + newFile = k + "=" + newFile + } + generator.FileSources[i] = newFile + } + return nil +} + +// localizePatchesStrategicMerge localizes the file paths in patches +func localizePatchesStrategicMerge(lc *Localizer, patches []types.PatchStrategicMerge) error { + for i := range patches { + // try to parse as inline + if _, err := lc.rFactory.RF().SliceFromBytes([]byte(patches[i])); err != nil { + // must be file path + newPath, err := lc.localizeFile(string(patches[i])) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize patchesStrategicMerge path") + } + patches[i] = types.PatchStrategicMerge(newPath) + } + } + return nil +} + +// localizeReplacements localizes the file paths in replacements +func localizeReplacements(lc *Localizer, replacements []types.ReplacementField) error { + for i := range replacements { + if replacements[i].Path != "" { + newPath, err := lc.localizeFile(replacements[i].Path) + if err != nil { + return errors.WrapPrefixf(err, "unable to localize replacements path") + } + replacements[i].Path = newPath + } + } + return nil +} diff --git a/api/internal/localizer/util_test.go b/api/internal/localizer/util_test.go index ca8096e739..dd58abb9f3 100644 --- a/api/internal/localizer/util_test.go +++ b/api/internal/localizer/util_test.go @@ -4,15 +4,311 @@ package localizer //nolint:testpackage import ( + "fmt" + "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" + "sigs.k8s.io/kustomize/api/ifc" + "sigs.k8s.io/kustomize/api/internal/git" + "sigs.k8s.io/kustomize/kyaml/filesys" ) func TestUrlBase(t *testing.T) { - require.Equal(t, "repo", urlBase("https://github.com/org/repo")) + tests := map[string]struct { + url, base string + }{ + "simple": { + url: "https://github.com/org/repo", + base: "repo", + }, + "trailing_slash": { + url: "github.com/org/repo//", + base: "repo", + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + require.Equal(t, test.base, urlBase(test.url)) + }) + } } -func TestUrlBaseTrailingSlash(t *testing.T) { - require.Equal(t, "repo", urlBase("github.com/org/repo//")) +// simpleJoin is filepath.Join() without side effects +func simpleJoin(t *testing.T, elems ...string) string { + t.Helper() + + return strings.Join(elems, string(filepath.Separator)) +} + +func TestLocFilePath(t *testing.T) { + for name, tUnit := range map[string]struct { + url, path string + }{ + "official": { + url: "https://raw.githubusercontent.com/org/repo/ref/path/to/file.yaml", + path: simpleJoin(t, "raw.githubusercontent.com", "org", "repo", "ref", "path", "to", "file.yaml"), + }, + "empty_path": { + url: "https://host", + path: "host", + }, + "empty_path_segment": { + url: "https://host//", + path: "host", + }, + "extraneous_components": { + url: "http://userinfo@host:1234/path/file?query", + path: simpleJoin(t, "host", "path", "file"), + }, + "percent-encoding": { + url: "https://host/file%2Eyaml", + path: simpleJoin(t, "host", "file%2Eyaml"), + }, + "dot-segments": { + url: "https://host/path/blah/../to/foo/bar/../../file/./", + path: simpleJoin(t, "host", "path", "to", "file"), + }, + "extraneous_dot-segments": { + url: "https://host/foo/bar/baz/../../../../file", + path: simpleJoin(t, "host", "file"), + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, simpleJoin(t, LocalizeDir, tUnit.path), locFilePath(tUnit.url)) + }) + } +} + +func TestLocFilePathColon(t *testing.T) { + req := require.New(t) + + // colon once used as unix file separator; also check IPv6 processing + const url = "https://[2001:4860:4860::8888]/file.yaml" + const host = "2001:4860:4860::8888" + const file = "file.yaml" + req.Equal(simpleJoin(t, LocalizeDir, host, file), locFilePath(url)) + + fSys := filesys.MakeFsOnDisk() + targetDir := simpleJoin(t, t.TempDir(), host) + + // make sure to create single directory to check that ':' not used as file separators + req.NoError(fSys.Mkdir(targetDir)) + _, err := fSys.Create(simpleJoin(t, targetDir, file)) + req.NoError(err) + + // readable + files, err := fSys.ReadDir(targetDir) + req.NoError(err) + req.Equal([]string{file}, files) +} + +func TestLocFilePathSpecialChar(t *testing.T) { + req := require.New(t) + + const wildcard = "*" + req.Equal(simpleJoin(t, LocalizeDir, "host", wildcard), locFilePath("https://host/*")) + + fSys := filesys.MakeFsOnDisk() + testDir := t.TempDir() + req.NoError(fSys.Mkdir(simpleJoin(t, testDir, "a"))) + req.NoError(fSys.WriteFile(simpleJoin(t, testDir, "b"), []byte{})) + // check wildcard name not being matched to existing files + // and can be successfully created + req.NoError(fSys.WriteFile(simpleJoin(t, testDir, wildcard), []byte("test"))) + + content, err := fSys.ReadFile(simpleJoin(t, testDir, wildcard)) + req.NoError(err) + req.Equal("test", string(content)) +} + +func TestLocFilePathSpecialFiles(t *testing.T) { + for name, tFSys := range map[string]struct { + urlPath string + pathDir, pathFile string + }{ + "windows_reserved_name": { + urlPath: "/aux/file", + pathDir: "aux", + pathFile: "file", + }, + "hidden_files": { + urlPath: "/.../.file", + pathDir: "...", + pathFile: ".file", + }, + } { + t.Run(name, func(t *testing.T) { + req := require.New(t) + + expectedPath := simpleJoin(t, LocalizeDir, "host", tFSys.pathDir, tFSys.pathFile) + req.Equal(expectedPath, locFilePath("https://host"+tFSys.urlPath)) + + fSys := filesys.MakeFsOnDisk() + targetDir := simpleJoin(t, t.TempDir(), tFSys.pathDir) + req.NoError(fSys.Mkdir(targetDir)) + req.NoError(fSys.WriteFile(simpleJoin(t, targetDir, tFSys.pathFile), []byte("test"))) + + content, err := fSys.ReadFile(simpleJoin(t, targetDir, tFSys.pathFile)) + req.NoError(err) + req.Equal([]byte("test"), content) + }) + } +} + +func TestLocRootPath(t *testing.T) { + // kustomize does not support ports yet + // UIs seem to constrain special characters in org, repo + // path/to/root is actually path instead of url + // git limits special characters in ref + // TODO(annasong): fix skipped tests + for name, tSamePath := range map[string]struct { + skip bool + urlf, path string + }{ + "ssh_non-github": { + // RepoSpec bug + skip: true, + urlf: "ssh://git@gitlab.com/org/repo//%s?ref=value", + path: simpleJoin(t, "gitlab.com", "org", "repo", "value"), + }, + "rel_ssh_non-github": { + // RepoSpec bug + skip: true, + urlf: "git@gitlab.com:org/repo//%s?ref=value", + path: simpleJoin(t, "gitlab.com", "org", "repo", "value"), + }, + "https_.git_suffix": { + urlf: "https://gitlab.com/org/repo.git//%s?ref=value", + path: simpleJoin(t, "gitlab.com", "org", "repo", "value"), + }, + "gh_shorthand": { + urlf: "gh:org/repo//%s?ref=value", + path: simpleJoin(t, "github.com", "org", "repo", "value"), + }, + "illegal_windows_dir": { + urlf: "https://gitlab.com/org./repo..git//%s?ref=value", + path: simpleJoin(t, "gitlab.com", "org.", "repo.", "value"), + }, + "ref_has_slash": { + urlf: "https://gitlab.com/org/repo//%s?ref=group/version/kind", + path: simpleJoin(t, "gitlab.com", "org", "repo", "group", "version", "kind"), + }, + } { + t.Run(name, func(t *testing.T) { + if tSamePath.skip { + t.Skip() + } + req := require.New(t) + fSys := filesys.MakeFsOnDisk() + + url := fmt.Sprintf(tSamePath.urlf, "path/to/root") + path := simpleJoin(t, LocalizeDir, tSamePath.path, "path", "to", "root") + + testDir := t.TempDir() + repoDir := simpleJoin(t, testDir, "repo-random_hash") + req.NoError(fSys.Mkdir(repoDir)) + rootDir := simpleJoin(t, repoDir, "path", "to", "root") + req.NoError(fSys.MkdirAll(rootDir)) + + actual := locRootPath(url, filesys.ConfirmedDir(repoDir), filesys.ConfirmedDir(rootDir)) + req.Equal(path, actual) + + req.NoError(fSys.MkdirAll(simpleJoin(t, testDir, path))) + }) + } +} + +func TestLocRootPathRepo(t *testing.T) { + for name, test := range map[string]struct { + skip bool + url, path string + }{ + "simple": { + url: "https://github.com/org/repo?ref=value", + path: simpleJoin(t, LocalizeDir, "github.com", "org", "repo", "value"), + }, + "long_org_path": { + url: "https://github.com/parent-org/child-org/repo.git?ref=value", + path: simpleJoin(t, LocalizeDir, "github.com", "parent-org", "child-org", "repo", "value"), + }, + "ref_slash": { + skip: true, + url: "https://github.com/org/repo?ref=group/version", + path: simpleJoin(t, LocalizeDir, "github.com", "org", "repo", "group", "version"), + }, + } { + t.Run(name, func(t *testing.T) { + if test.skip { + t.Skip() + } + testDir := t.TempDir() + repoDir := filesys.ConfirmedDir(testDir) + require.Equal(t, test.path, locRootPath(test.url, repoDir, repoDir)) + }) + } +} + +func TestLocRootPathSymlinkPath(t *testing.T) { + req := require.New(t) + fSys := filesys.MakeFsOnDisk() + + const url = "https://github.com/org/repo//symlink?ref=value" + repoDir := t.TempDir() + rootDir := simpleJoin(t, repoDir, "actual-root") + req.NoError(fSys.Mkdir(rootDir)) + req.NoError(os.Symlink(rootDir, simpleJoin(t, repoDir, "symlink"))) + + expected := simpleJoin(t, LocalizeDir, "github.com", "org", "repo", "value", "actual-root") + req.Equal(expected, locRootPath(url, filesys.ConfirmedDir(repoDir), filesys.ConfirmedDir(rootDir))) +} + +func TestDefaultNewDirRepo(t *testing.T) { + for name, test := range map[string]struct { + skip bool + url, dst string + }{ + "simple": { + url: "https://github.com/org/repo?ref=value", + dst: "localized-repo-value", + }, + // TODO(annasong): Fix test + // RepoSpec bug + "slashed_ref": { + skip: true, + url: "https://github.com/org/repo?ref=group/version", + dst: "localized-repo-group-version", + }, + } { + t.Run(name, func(t *testing.T) { + if test.skip { + t.Skip() + } + repoSpec, err := git.NewRepoSpecFromURL(test.url) + require.NoError(t, err) + require.Equal(t, test.dst, defaultNewDir(&fakeLoader{t.TempDir()}, repoSpec)) + }) + } +} + +type fakeLoader struct { + root string +} + +func (fl *fakeLoader) Root() string { + return fl.root +} +func (fl *fakeLoader) Repo() (string, bool) { + return fl.root, true +} +func (fl *fakeLoader) Load(_ string) ([]byte, error) { + return []byte{}, nil +} +func (fl *fakeLoader) New(path string) (ifc.Loader, error) { + return &fakeLoader{path}, nil +} +func (fl *fakeLoader) Cleanup() error { + return nil } diff --git a/api/kv/kv.go b/api/kv/kv.go index 303aede866..bcc2055d34 100644 --- a/api/kv/kv.go +++ b/api/kv/kv.go @@ -177,10 +177,10 @@ func (kvl *loader) keyValuesFromLine(line []byte, currentLine int) (types.Pair, // ParseFileSource parses the source given. // -// Acceptable formats include: -// 1. source-path: the basename will become the key name -// 2. source-name=source-path: the source-name will become the key name and -// source-path is the path to the key file. +// Acceptable formats include: +// 1. source-path: the basename will become the key name +// 2. source-name=source-path: the source-name will become the key name and +// source-path is the path to the key file. // // Key names cannot include '='. func parseFileSource(source string) (keyName, filePath string, err error) {