From 399ca353a025918c98e95c9a7380f451213c8d8a Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Wed, 27 Nov 2024 14:40:50 +0100 Subject: [PATCH] Feature: unified include/exclude file (#224) * refactor: Simplify config/flag/default behavior. The new `pickFirst()` function returns the first non-empty string, making the code simpler more readable. * feat(pattern_file): Add the `pattern_file` config option and flag. Issue: #223 * test(pattern_file): Add integration test for the `pattern_file` option. * chore(pattern_file): Remove the `pattern_file` option and use `include` instead. * fix: Allow the `doublestar` option to overwrite the default match type. * refactor: Rename `PatternFile` to `PatternFileCollector`. * refactor: Use pointer receiver for `PatternFileCollector`. * fix: Copy loop variable for Go 1.18. * docs(match_type): Document the `match_type` option and the `gitignore` mode. * docs: Improve markup. Co-authored-by: Braydon Kains <93549768+braydonk@users.noreply.github.com> * chore: Regenerate integration test outputs. ```shell make integrationtest_update ``` --------- Co-authored-by: Braydon Kains <93549768+braydonk@users.noreply.github.com> --- cmd/yamlfmt/config.go | 24 +++- cmd/yamlfmt/flags.go | 1 + command/command.go | 35 ++++-- docs/command-usage.md | 1 + docs/config-file.md | 1 + docs/paths.md | 33 +++++- integrationtest/command/command_test.go | 8 ++ .../testdata/pattern_file/after/a.yaml | 1 + .../command/testdata/pattern_file/after/b.yml | 1 + .../pattern_file/after/excluded_dir/a.yaml | 1 + .../pattern_file/after/excluded_file.yaml | 1 + .../pattern_file/after/included_dir/a.yaml | 1 + .../pattern_file/after/yamlfmt.patterns | 7 ++ .../testdata/pattern_file/before/a.yaml | 1 + .../testdata/pattern_file/before/b.yml | 1 + .../pattern_file/before/excluded_dir/a.yaml | 1 + .../pattern_file/before/excluded_file.yaml | 1 + .../pattern_file/before/included_dir/a.yaml | 1 + .../pattern_file/before/yamlfmt.patterns | 7 ++ .../testdata/pattern_file/stdout/stderr.txt | 0 .../testdata/pattern_file/stdout/stdout.txt | 0 .../print_conf_file/stdout/stdout.txt | 1 + .../print_conf_flags/stdout/stdout.txt | 1 + .../stdout/stdout.txt | 1 + internal/tempfile/golden.go | 8 +- path_collector.go | 104 ++++++++++++++++++ path_collector_test.go | 89 +++++++++++++++ 27 files changed, 311 insertions(+), 20 deletions(-) create mode 100755 integrationtest/command/testdata/pattern_file/after/a.yaml create mode 100755 integrationtest/command/testdata/pattern_file/after/b.yml create mode 100644 integrationtest/command/testdata/pattern_file/after/excluded_dir/a.yaml create mode 100755 integrationtest/command/testdata/pattern_file/after/excluded_file.yaml create mode 100755 integrationtest/command/testdata/pattern_file/after/included_dir/a.yaml create mode 100755 integrationtest/command/testdata/pattern_file/after/yamlfmt.patterns create mode 100644 integrationtest/command/testdata/pattern_file/before/a.yaml create mode 100644 integrationtest/command/testdata/pattern_file/before/b.yml create mode 100644 integrationtest/command/testdata/pattern_file/before/excluded_dir/a.yaml create mode 100644 integrationtest/command/testdata/pattern_file/before/excluded_file.yaml create mode 100644 integrationtest/command/testdata/pattern_file/before/included_dir/a.yaml create mode 100644 integrationtest/command/testdata/pattern_file/before/yamlfmt.patterns create mode 100644 integrationtest/command/testdata/pattern_file/stdout/stderr.txt create mode 100644 integrationtest/command/testdata/pattern_file/stdout/stdout.txt diff --git a/cmd/yamlfmt/config.go b/cmd/yamlfmt/config.go index 27e4ddf..1174e84 100644 --- a/cmd/yamlfmt/config.go +++ b/cmd/yamlfmt/config.go @@ -27,6 +27,7 @@ import ( "github.com/braydonk/yaml" "github.com/google/yamlfmt" "github.com/google/yamlfmt/command" + "github.com/google/yamlfmt/engine" "github.com/google/yamlfmt/internal/collections" "github.com/google/yamlfmt/internal/logger" "github.com/mitchellh/mapstructure" @@ -278,12 +279,14 @@ func makeCommandConfigFromData(configData map[string]any) (*command.Config, erro if !config.GitignoreExcludes { config.GitignoreExcludes = *flagGitignoreExcludes } - if config.GitignorePath == "" { - config.GitignorePath = *flagGitignorePath - } - if config.OutputFormat == "" { - config.OutputFormat = getOutputFormatFromFlag() + config.GitignorePath = pickFirst(config.GitignorePath, *flagGitignorePath) + config.OutputFormat = pickFirst(config.OutputFormat, getOutputFormatFromFlag(), engine.EngineOutputDefault) + + defaultMatchType := yamlfmt.MatchTypeStandard + if config.Doublestar { + defaultMatchType = yamlfmt.MatchTypeDoublestar } + config.MatchType = pickFirst(config.MatchType, yamlfmt.MatchType(*flagMatchType), defaultMatchType) // Overwrite config if includes are provided through args if len(flag.Args()) > 0 { @@ -297,6 +300,17 @@ func makeCommandConfigFromData(configData map[string]any) (*command.Config, erro return &config, nil } +// pickFirst returns the first string in ss that is not empty. +func pickFirst[T ~string](ss ...T) T { + for _, s := range ss { + if s != "" { + return s + } + } + + return "" +} + func parseFormatterConfigFlag(flagValues []string) (map[string]any, error) { formatterValues := map[string]any{} flagErrors := collections.Errors{} diff --git a/cmd/yamlfmt/flags.go b/cmd/yamlfmt/flags.go index c6fdf96..7137723 100644 --- a/cmd/yamlfmt/flags.go +++ b/cmd/yamlfmt/flags.go @@ -41,6 +41,7 @@ operation without performing it.`) flagGitignoreExcludes *bool = flag.Bool("gitignore_excludes", false, "Use a gitignore file for excludes") flagGitignorePath *string = flag.String("gitignore_path", ".gitignore", "Path to gitignore file to use") flagOutputFormat *string = flag.String("output_format", "default", "The engine output format") + flagMatchType *string = flag.String("match_type", "", "The file discovery method to use. Valid values: standard, doublestar, gitignore") flagExclude = arrayFlag{} flagFormatter = arrayFlag{} flagExtensions = arrayFlag{} diff --git a/command/command.go b/command/command.go index 3f75c40..3224f46 100644 --- a/command/command.go +++ b/command/command.go @@ -40,6 +40,7 @@ func NewFormatterConfig() *FormatterConfig { type Config struct { Extensions []string `mapstructure:"extensions"` + MatchType yamlfmt.MatchType `mapstructure:"match_type"` Include []string `mapstructure:"include"` Exclude []string `mapstructure:"exclude"` RegexExclude []string `mapstructure:"regex_exclude"` @@ -190,7 +191,11 @@ func (c *Command) getFormatter() (yamlfmt.Formatter, error) { } func (c *Command) collectPaths() ([]string, error) { - collector := c.makePathCollector() + collector, err := c.makePathCollector() + if err != nil { + return nil, err + } + return collector.CollectPaths() } @@ -203,17 +208,31 @@ func (c *Command) analyzePaths(paths []string) ([]string, error) { return includePaths, err } -func (c *Command) makePathCollector() yamlfmt.PathCollector { - if c.Config.Doublestar { +func (c *Command) makePathCollector() (yamlfmt.PathCollector, error) { + switch c.Config.MatchType { + case yamlfmt.MatchTypeDoublestar: return &yamlfmt.DoublestarCollector{ Include: c.Config.Include, Exclude: c.Config.Exclude, + }, nil + case yamlfmt.MatchTypeGitignore: + files := c.Config.Include + if len(files) == 0 { + files = []string{yamlfmt.DefaultPatternFile} } - } - return &yamlfmt.FilepathCollector{ - Include: c.Config.Include, - Exclude: c.Config.Exclude, - Extensions: c.Config.Extensions, + + patternFile, err := yamlfmt.NewPatternFileCollector(files...) + if err != nil { + return nil, fmt.Errorf("NewPatternFile(%q): %w", files, err) + } + + return patternFile, nil + default: + return &yamlfmt.FilepathCollector{ + Include: c.Config.Include, + Exclude: c.Config.Exclude, + Extensions: c.Config.Extensions, + }, nil } } diff --git a/docs/command-usage.md b/docs/command-usage.md index 4d9d283..3ae3e96 100644 --- a/docs/command-usage.md +++ b/docs/command-usage.md @@ -81,6 +81,7 @@ The string array flags can be a bit confusing. See the [String Array Flags](#str | Global Config | `-global_conf` | bool | `yamlfmt -global_conf` | Force yamlfmt to use the configuration file from the system config directory. | | Disable Global Config | `-no_global_conf` | bool | `yamlfmt -no_global_conf` | Disable looking for the configuration file from the system config directory. | | Doublestar | `-dstar` | bool | `yamlfmt -dstar "**/*.yaml"` | Enable [Doublestar](./paths.md#doublestar) path collection mode. Note that doublestar patterns should be specified with quotes in bash to prevent shell expansion. | +| Match type | `-match_type` | string | `yamlfmt -match_type standard` | Controls how `include` and `exclude` are interpreted. See [Specifying Paths](./paths.md) for more details. | | Exclude | `-exclude` | []string | `yamlfmt -exclude ./not/,these_paths.yaml` | Patterns to exclude from path collection. These are in addition to the exclude patterns specified in the [config file](./config-file.md) | | Gitignore Excludes | `-gitignore_excludes` | bool | `yamlfmt -gitignore_excludes` | Use a gitignore file to exclude paths. This is in addition to otherwise specified exclude patterns. | | Gitignore Path | `-gitignore_path` | string | `yamlfmt -gitignore_path .special_gitignore` | Specify a path to a gitignore file to use. Defaults to `.gitignore` (in working directory). | diff --git a/docs/config-file.md b/docs/config-file.md index 16404c2..a0f9efb 100644 --- a/docs/config-file.md +++ b/docs/config-file.md @@ -39,6 +39,7 @@ The command package defines the main command engine that `cmd/yamlfmt` uses. It | `line_ending` | `lf` or `crlf` | `crlf` on Windows, `lf` otherwise | Parse and write the file with "lf" or "crlf" line endings. This global setting will override any formatter `line_ending` options. | | `doublestar` | bool | false | Use [doublestar](https://github.com/bmatcuk/doublestar) for include and exclude paths. (This was the default before 0.7.0) | | `continue_on_error` | bool | false | Continue formatting and don't exit with code 1 when there is an invalid yaml file found. | +| `match_type` | string | `standard` | Controls how `include` and `exclude` are interpreted. See [Specifying Paths][] for more details. | | `include` | []string | [] | The paths for the command to include for formatting. See [Specifying Paths][] for more details. | | `exclude` | []string | [] | The paths for the command to exclude from formatting. See [Specifying Paths][] for more details. | | `gitignore_excludes` | bool | false | Use gitignore files for exclude paths. This is in addition to the patterns from the `exclude` option. | diff --git a/docs/paths.md b/docs/paths.md index a98236c..f6c971c 100644 --- a/docs/paths.md +++ b/docs/paths.md @@ -1,6 +1,6 @@ # Paths -`yamlfmt` can collect paths in two modes: Standard, or Doublestar. +`yamlfmt` can collect paths in three modes: Standard, Doublestar, and Gitignore. The `match_type` option allows you to select between the modes, using the values `standard`, `doublestar`, and `gitignore`. ## Standard (default) @@ -12,14 +12,41 @@ This mode does *not* support wildcards, aka. globbing. That means with `*.yaml` In Doublestar mode, paths are specified using wildcard patterns explained in the [doublestar](https://github.com/bmatcuk/doublestar) package. It is almost identical to bash and git's style of glob pattern specification. -To enable the doublestar mode, set `doublestar: true` in the config file or use the `-dstar` command line flag. +To enable the doublestar mode, set `match_type: doublestar` in the config file or use the `-match_type doublestar` command line flag. + +## Gitignore + +This mode allows you to use a file (or files) using the [gitignore](https://git-scm.com/docs/gitignore) syntax to determine which files to include and exclude. + +Despite having "ignore" in the name, yamlfmt will format all files that match the patterns listed in the file, unless the patterns are negated. For example, the following file will format all `*.yaml` and `*.yml` files, while ignoring other files such as `README.md` as well as files in `testdata` directories: + +```gitignore +# Include +*.yaml +*.yml + +# Exclude +!testdata/ +``` + +Please read the [gitignore manpage](https://git-scm.com/docs/gitignore) for the full syntax description and further examples. + +To use the `gitignore` mode, set `match_type: gitignore` in the config file or use the `-match_type gitignore` command line flag. +In this mode, positional arguments on the command line and files listed in the `include` config option are considered to be files using the gitignore syntax. +If no file is specified, yamlfmt will look for a file called `yamlfmt.patterns` in the working directory. + +The `exclude` option is ignored in this mode. + ## Include and Exclude -In both modes, `yamlfmt` will allow you to configure include and exclude paths. These can be paths to files in Standard or Doublestar modes, paths to directories in Standard mode, and valid doublestar patterns in Doublestar mode. These paths should be specified **relative to the working directory of `yamlfmt`**. They will work as absolute paths if both the includes and excludes are specified as absolute paths or if both are relative paths, however it will not work as expected if they are mixed together. It usually easier to reason about includes and excludes when always specifying both as relative paths from the directory `yamlfmt` is going to be run in. +In the `standard` and `doublestar` modes, `yamlfmt` will allow you to configure include and exclude paths. These can be paths to files in Standard or Doublestar modes, paths to directories in Standard mode, and valid doublestar patterns in Doublestar mode. These paths should be specified **relative to the working directory of `yamlfmt`**. They will work as absolute paths if both the includes and excludes are specified as absolute paths or if both are relative paths, however it will not work as expected if they are mixed together. It usually easier to reason about includes and excludes when always specifying both as relative paths from the directory `yamlfmt` is going to be run in. + +In the `gitignore` mode, files specified on the command line or via the `include` config option are expected to contain patterns following the [gitignore](https://git-scm.com/docs/gitignore) syntax. Exclude paths can be specified on the command line using the `-exclude` flag. Paths excluded from the command line are **added* to excluded paths from the config file. +The `gitignore` mode ignores the `exclude` option. Include paths can be specified on the command line via the positional arguments, i.e. there is no flag for it. Paths from the command line take precedence over and **replace** any paths configured in the config file. diff --git a/integrationtest/command/command_test.go b/integrationtest/command/command_test.go index 46b162f..1fff369 100644 --- a/integrationtest/command/command_test.go +++ b/integrationtest/command/command_test.go @@ -155,3 +155,11 @@ func TestGitLabOutput(t *testing.T) { Update: *updateFlag, }.Run(t) } + +func TestPatternFile(t *testing.T) { + TestCase{ + Dir: "pattern_file", + Command: yamlfmtWithArgs("-match_type gitignore yamlfmt.patterns"), + Update: *updateFlag, + }.Run(t) +} diff --git a/integrationtest/command/testdata/pattern_file/after/a.yaml b/integrationtest/command/testdata/pattern_file/after/a.yaml new file mode 100755 index 0000000..28e1bff --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/after/a.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/after/b.yml b/integrationtest/command/testdata/pattern_file/after/b.yml new file mode 100755 index 0000000..28e1bff --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/after/b.yml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/after/excluded_dir/a.yaml b/integrationtest/command/testdata/pattern_file/after/excluded_dir/a.yaml new file mode 100644 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/after/excluded_dir/a.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/after/excluded_file.yaml b/integrationtest/command/testdata/pattern_file/after/excluded_file.yaml new file mode 100755 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/after/excluded_file.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/after/included_dir/a.yaml b/integrationtest/command/testdata/pattern_file/after/included_dir/a.yaml new file mode 100755 index 0000000..28e1bff --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/after/included_dir/a.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/after/yamlfmt.patterns b/integrationtest/command/testdata/pattern_file/after/yamlfmt.patterns new file mode 100755 index 0000000..915fd24 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/after/yamlfmt.patterns @@ -0,0 +1,7 @@ +# vim: syntax=gitignore + +*.yaml +*.yml + +!excluded_file.yaml +!excluded_dir/ diff --git a/integrationtest/command/testdata/pattern_file/before/a.yaml b/integrationtest/command/testdata/pattern_file/before/a.yaml new file mode 100644 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/before/a.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/before/b.yml b/integrationtest/command/testdata/pattern_file/before/b.yml new file mode 100644 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/before/b.yml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/before/excluded_dir/a.yaml b/integrationtest/command/testdata/pattern_file/before/excluded_dir/a.yaml new file mode 100644 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/before/excluded_dir/a.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/before/excluded_file.yaml b/integrationtest/command/testdata/pattern_file/before/excluded_file.yaml new file mode 100644 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/before/excluded_file.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/before/included_dir/a.yaml b/integrationtest/command/testdata/pattern_file/before/included_dir/a.yaml new file mode 100644 index 0000000..0eea297 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/before/included_dir/a.yaml @@ -0,0 +1 @@ +formatted: true diff --git a/integrationtest/command/testdata/pattern_file/before/yamlfmt.patterns b/integrationtest/command/testdata/pattern_file/before/yamlfmt.patterns new file mode 100644 index 0000000..915fd24 --- /dev/null +++ b/integrationtest/command/testdata/pattern_file/before/yamlfmt.patterns @@ -0,0 +1,7 @@ +# vim: syntax=gitignore + +*.yaml +*.yml + +!excluded_file.yaml +!excluded_dir/ diff --git a/integrationtest/command/testdata/pattern_file/stdout/stderr.txt b/integrationtest/command/testdata/pattern_file/stdout/stderr.txt new file mode 100644 index 0000000..e69de29 diff --git a/integrationtest/command/testdata/pattern_file/stdout/stdout.txt b/integrationtest/command/testdata/pattern_file/stdout/stdout.txt new file mode 100644 index 0000000..e69de29 diff --git a/integrationtest/command/testdata/print_conf_file/stdout/stdout.txt b/integrationtest/command/testdata/print_conf_file/stdout/stdout.txt index 2a6fae2..a1172e5 100644 --- a/integrationtest/command/testdata/print_conf_file/stdout/stdout.txt +++ b/integrationtest/command/testdata/print_conf_file/stdout/stdout.txt @@ -9,6 +9,7 @@ gitignore_excludes: false gitignore_path: .my_gitignore include: [] line_ending: crlf +match_type: doublestar output_format: default regex_exclude: [] formatter: diff --git a/integrationtest/command/testdata/print_conf_flags/stdout/stdout.txt b/integrationtest/command/testdata/print_conf_flags/stdout/stdout.txt index af2ab8f..9c73e36 100644 --- a/integrationtest/command/testdata/print_conf_flags/stdout/stdout.txt +++ b/integrationtest/command/testdata/print_conf_flags/stdout/stdout.txt @@ -8,6 +8,7 @@ gitignore_excludes: false gitignore_path: .gitignore include: [] line_ending: lf +match_type: standard output_format: default regex_exclude: [] formatter: diff --git a/integrationtest/command/testdata/print_conf_flags_and_file/stdout/stdout.txt b/integrationtest/command/testdata/print_conf_flags_and_file/stdout/stdout.txt index a8f4615..01be6e7 100644 --- a/integrationtest/command/testdata/print_conf_flags_and_file/stdout/stdout.txt +++ b/integrationtest/command/testdata/print_conf_flags_and_file/stdout/stdout.txt @@ -9,6 +9,7 @@ gitignore_excludes: false gitignore_path: .my_gitignore include: [] line_ending: crlf +match_type: doublestar output_format: default regex_exclude: [] formatter: diff --git a/internal/tempfile/golden.go b/internal/tempfile/golden.go index b14b46b..624fa07 100644 --- a/internal/tempfile/golden.go +++ b/internal/tempfile/golden.go @@ -43,7 +43,7 @@ func (g GoldenCtx) CompareGoldenFile(path string, gotContent []byte) error { // If we are not updating, check that the content is the same. expectedContent, err := os.ReadFile(g.goldenPath(path)) if err != nil { - return err + return fmt.Errorf("os.ReadFile(%q): %w", g.goldenPath(path), err) } // Edge case for empty stdout. if gotContent == nil { @@ -84,11 +84,11 @@ func (g GoldenCtx) CompareDirectory(resultPath string) error { for path := range resultPaths { gotContent, err := os.ReadFile(filepath.Join(resultPath, path)) if err != nil { - return fmt.Errorf("%s: %w", path, err) + return fmt.Errorf("os.ReadFile(%q): %w", path, err) } err = g.CompareGoldenFile(path, gotContent) if err != nil { - return fmt.Errorf("%s: %w", path, err) + return fmt.Errorf("CompareGoldenFile(%q): %w", path, err) } } // If there are no errors this will be nil, otherwise will be a @@ -118,7 +118,7 @@ func (g GoldenCtx) updateGoldenDirectory(resultPath string) error { func readAllPaths(dirPath string) (collections.Set[string], error) { paths := collections.Set[string]{} allNamesButCurrentDirectory := func(path string, d fs.DirEntry, err error) error { - if path == dirPath { + if d.IsDir() { return nil } paths.Add(d.Name()) diff --git a/path_collector.go b/path_collector.go index 2b8d516..7967f3c 100644 --- a/path_collector.go +++ b/path_collector.go @@ -15,8 +15,11 @@ package yamlfmt import ( + "bufio" + "bytes" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -28,6 +31,14 @@ import ( ignore "github.com/sabhiram/go-gitignore" ) +type MatchType string + +const ( + MatchTypeStandard MatchType = "standard" + MatchTypeDoublestar MatchType = "doublestar" + MatchTypeGitignore MatchType = "gitignore" +) + type PathCollector interface { CollectPaths() ([]string, error) } @@ -230,3 +241,96 @@ func ExcludeWithGitignore(gitignorePath string, paths []string) ([]string, error logger.Debug(logger.DebugCodePaths, "paths to format: %s", pathsToFormat) return pathsToFormat, nil } + +const DefaultPatternFile = "yamlfmt.patterns" + +// PatternFileCollector determines which files to format and which to ignore based on a pattern file in gitignore(5) syntax. +type PatternFileCollector struct { + fs fs.FS + matcher *ignore.GitIgnore +} + +// NewPatternFileCollector initializes a new PatternFile using the provided file(s). +// If multiple files are provided, their content is concatenated in order. +// All patterns are relative to the current working directory. +func NewPatternFileCollector(files ...string) (*PatternFileCollector, error) { + r, err := cat(files...) + if err != nil { + return nil, err + } + + wd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("os.Getwd: %w", err) + } + + return NewPatternFileCollectorFS(r, os.DirFS(wd)), nil +} + +// cat concatenates the contents of all files in its argument list. +func cat(files ...string) (io.Reader, error) { + var b bytes.Buffer + + for _, f := range files { + fh, err := os.Open(f) + if err != nil { + return nil, err + } + defer fh.Close() + + if _, err := io.Copy(&b, fh); err != nil { + return nil, fmt.Errorf("copying %q: %w", f, err) + } + fh.Close() + + // Append a newline to avoid issues with files lacking a newline at end-of-file. + fmt.Fprintln(&b) + } + + return &b, nil +} + +// NewPatternFileCollectorFS reads a pattern file from r and uses fs for file lookups. +// It is used by NewPatternFile and primarily public because it is useful for testing. +func NewPatternFileCollectorFS(r io.Reader, fs fs.FS) *PatternFileCollector { + var lines []string + + s := bufio.NewScanner(r) + for s.Scan() { + lines = append(lines, s.Text()) + } + + return &PatternFileCollector{ + fs: fs, + matcher: ignore.CompileIgnoreLines(lines...), + } +} + +// CollectPaths implements the PathCollector interface. +func (c *PatternFileCollector) CollectPaths() ([]string, error) { + var files []string + + err := fs.WalkDir(c.fs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + ok, pattern := c.matcher.MatchesPathHow(path) + switch { + case ok && pattern.Negate && d.IsDir(): + return fs.SkipDir + case ok && pattern.Negate: + return nil + case ok && d.Type().IsRegular(): + files = append(files, path) + } + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("WalkDir: %w", err) + } + + return files, nil +} diff --git a/path_collector_test.go b/path_collector_test.go index 0ccf06e..0c79e45 100644 --- a/path_collector_test.go +++ b/path_collector_test.go @@ -15,12 +15,16 @@ package yamlfmt_test import ( + "bytes" "fmt" "os" "path/filepath" "strings" "testing" + "testing/fstest" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/yamlfmt" "github.com/google/yamlfmt/internal/collections" "github.com/google/yamlfmt/internal/tempfile" @@ -449,3 +453,88 @@ func useDoublestarCollector(tc testCase, path string) yamlfmt.PathCollector { Exclude: tc.excludePatterns.allPatterns(path), } } + +func TestPatternFile(t *testing.T) { + t.Parallel() + + makePatterns := func(patterns ...string) []byte { + var b bytes.Buffer + + fmt.Fprintln(&b, "# Comment followed by empty line") + fmt.Fprintln(&b) + for _, p := range patterns { + fmt.Fprintln(&b, p) + } + + return b.Bytes() + } + + cases := []struct { + name string + patterns []byte + haveFiles []string + wantFiles []string + }{ + { + name: "yaml and yml files", + patterns: makePatterns("*.yaml", "*.yml"), + haveFiles: []string{"x.yaml", "y.yml", "README.md"}, + wantFiles: []string{"x.yaml", "y.yml"}, + }, + { + name: "ignore pattern", + patterns: makePatterns("*.yaml", "*.yml", "!test_input.yaml"), + haveFiles: []string{"x.yaml", "y.yml", "test_input.yaml"}, + wantFiles: []string{"x.yaml", "y.yml"}, + }, + { + name: "descent into directories", + patterns: makePatterns("*.yaml", "*.yml"), + haveFiles: []string{"a/x.yaml", "b/y.yml"}, + wantFiles: []string{"a/x.yaml", "b/y.yml"}, + }, + { + name: "exclude directories", + patterns: makePatterns("*.yaml", "*.yml", "!a/"), + haveFiles: []string{"a/x.yaml", "b/y.yml"}, + wantFiles: []string{"b/y.yml"}, + }, + { + name: "matches are rooted at the working directory", + patterns: makePatterns("*.yaml", "!/x.yaml"), + haveFiles: []string{"x.yaml", "a/x.yaml"}, + wantFiles: []string{"a/x.yaml"}, + }, + } + + for _, tc := range cases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + fs := make(fstest.MapFS) + for _, f := range tc.haveFiles { + fs[f] = &fstest.MapFile{ + Data: []byte("test"), + } + } + + patternFile := yamlfmt.NewPatternFileCollectorFS(bytes.NewReader(tc.patterns), fs) + + gotFiles, err := patternFile.CollectPaths() + if err != nil { + t.Fatal(err) + } + + // Ignore the order of files in tc.wantFiles and gotFiles. + opts := []cmp.Option{ + cmpopts.SortSlices(func(a, b string) bool { return a < b }), + } + + if diff := cmp.Diff(tc.wantFiles, gotFiles, opts...); diff != "" { + t.Errorf("PatternFile.CollectPaths() differs (-want/+got):\n%s", diff) + } + }) + } +}