diff --git a/args/args.go b/args/args.go index 49cc76da..1adebfe4 100644 --- a/args/args.go +++ b/args/args.go @@ -137,16 +137,8 @@ func (g *GeneratorArgs) NewBuilder() (*parser.Builder, error) { // Ignore all auto-generated files. b.AddBuildTags(g.GeneratedBuildTag) - for _, d := range g.InputDirs { - var err error - if strings.HasSuffix(d, "/...") { - err = b.AddDirRecursive(strings.TrimSuffix(d, "/...")) - } else { - err = b.AddDir(d) - } - if err != nil { - return nil, fmt.Errorf("unable to add directory %q: %v", d, err) - } + if err := b.AddPackagePatterns(g.InputDirs...); err != nil { + return nil, err } return b, nil } diff --git a/go.mod b/go.mod index ebbd68ad..9ae0b810 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module k8s.io/gengo -go 1.13 +go 1.16 require ( github.com/davecgh/go-spew v1.1.1 @@ -8,13 +8,9 @@ require ( github.com/google/gofuzz v1.1.0 github.com/kr/pretty v0.2.0 // indirect github.com/spf13/pflag v1.0.5 - golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8 + golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55 // indirect + golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect k8s.io/klog/v2 v2.2.0 sigs.k8s.io/yaml v1.2.0 ) - -replace ( - golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // pinned to release-branch.go1.13 - golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 // pinned to release-branch.go1.13 -) diff --git a/go.sum b/go.sum index 43d91b63..fc2377aa 100644 --- a/go.sum +++ b/go.sum @@ -13,16 +13,35 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55 h1:rw6UNGRMfarCepjI8qOepea/SXwIBVfTKjztZ5gBbq4= +golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7 h1:PVCvyir09Xgta5zksNZDkrL+eSm/Y+gQxRG3IfqNQ3A= -golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff h1:VX/uD7MK0AHXGiScH3fsieUQUcpmRERPDYtqZdJnA+Q= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/parser/parse.go b/parser/parse.go index 58850e63..0479b10a 100644 --- a/parser/parse.go +++ b/parser/parse.go @@ -33,6 +33,8 @@ import ( "sort" "strings" + "golang.org/x/tools/go/packages" + "k8s.io/gengo/types" "k8s.io/klog/v2" ) @@ -48,11 +50,18 @@ type Builder struct { // If true, include *_test.go IncludeTestFiles bool + // Cache of import path to filesystem directory. + // Used to optimize loading of individual packages using ImportDir instead of Import. + // For context, see https://github.com/golang/go/issues/31087 + pkgDirs map[string]string + // Map of package names to more canonical information about the package. // This might hold the same value for multiple names, e.g. if someone // referenced ./pkg/name or in the case of vendoring, which canonicalizes // differently that what humans would type. - buildPackages map[string]*build.Package + // + // This must only be accessed via getLoadedBuildPackage and setLoadedBuildPackage + buildPackages map[importPathString]*build.Package fset *token.FileSet // map of package path to list of parsed files @@ -102,7 +111,8 @@ func New() *Builder { c.CgoEnabled = false return &Builder{ context: &c, - buildPackages: map[string]*build.Package{}, + pkgDirs: map[string]string{}, + buildPackages: map[importPathString]*build.Package{}, typeCheckedPackages: map[importPathString]*tc.Package{}, fset: token.NewFileSet(), parsed: map[importPathString][]parsedFile{}, @@ -118,11 +128,37 @@ func (b *Builder) AddBuildTags(tags ...string) { b.context.BuildTags = append(b.context.BuildTags, tags...) } +func (b *Builder) getLoadedBuildPackage(importPath string) (*build.Package, bool) { + canonicalized := canonicalizeImportPath(importPath) + if string(canonicalized) != string(importPath) { + klog.V(5).Infof("getLoadedBuildPackage: %s normalized to %s", importPath, canonicalized) + } + buildPkg, ok := b.buildPackages[canonicalizeImportPath(importPath)] + return buildPkg, ok +} +func (b *Builder) setLoadedBuildPackage(importPath string, buildPkg *build.Package) { + canonicalizedImportPath := canonicalizeImportPath(importPath) + if string(canonicalizedImportPath) != string(importPath) { + klog.V(5).Infof("setLoadedBuildPackage: importPath %s normalized to %s", importPath, canonicalizedImportPath) + } + + canonicalizedBuildPkgImportPath := canonicalizeImportPath(buildPkg.ImportPath) + if string(canonicalizedBuildPkgImportPath) != string(buildPkg.ImportPath) { + klog.V(5).Infof("setLoadedBuildPackage: buildPkg.ImportPath %s normalized to %s", buildPkg.ImportPath, canonicalizedBuildPkgImportPath) + } + + if string(canonicalizedImportPath) != string(canonicalizedBuildPkgImportPath) { + klog.V(5).Infof("setLoadedBuildPackage: normalized importPath (%s) differs from buildPkg.ImportPath (%s)", canonicalizedImportPath, canonicalizedBuildPkgImportPath) + } + b.buildPackages[canonicalizedImportPath] = buildPkg + b.buildPackages[canonicalizedBuildPkgImportPath] = buildPkg +} + // Get package information from the go/build package. Automatically excludes // e.g. test files and files for other platforms-- there is quite a bit of // logic of that nature in the build package. func (b *Builder) importBuildPackage(dir string) (*build.Package, error) { - if buildPkg, ok := b.buildPackages[dir]; ok { + if buildPkg, ok := b.getLoadedBuildPackage(dir); ok { return buildPkg, nil } // This validates the `package foo // github.com/bar/foo` comments. @@ -142,17 +178,7 @@ func (b *Builder) importBuildPackage(dir string) (*build.Package, error) { // Remember it under the user-provided name. klog.V(5).Infof("saving buildPackage %s", dir) - b.buildPackages[dir] = buildPkg - canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) - if dir != string(canonicalPackage) { - // Since `dir` is not the canonical name, see if we knew it under another name. - if buildPkg, ok := b.buildPackages[string(canonicalPackage)]; ok { - return buildPkg, nil - } - // Must be new, save it under the canonical name, too. - klog.V(5).Infof("saving buildPackage %s", canonicalPackage) - b.buildPackages[string(canonicalPackage)] = buildPkg - } + b.setLoadedBuildPackage(dir, buildPkg) return buildPkg, nil } @@ -212,18 +238,121 @@ func (b *Builder) addFile(pkgPath importPathString, path string, src []byte, use return nil } -// AddDir adds an entire directory, scanning it for go files. 'dir' should have -// a single go package in it. GOPATH, GOROOT, and the location of your go -// binary (`which go`) will all be searched if dir doesn't literally resolve. -func (b *Builder) AddDir(dir string) error { - _, err := b.importPackage(dir, true) +// getResolvedPackageDir returns the resolved absolute path to the directory +// containing the source for the specified package import path. +// The returned dir is suitable to pass to build.Context#ImportDir. +func (b *Builder) getResolvedPackageDir(pkgPath string) (string, bool) { + if len(pkgPath) == 0 || !strings.Contains(pkgPath, "/") { + return "", false + } + dir, ok := b.pkgDirs[pkgPath] + if ok { + return dir, len(dir) > 0 + } + parentDir, ok := b.getResolvedPackageDir(path.Dir(pkgPath)) + if !ok { + return "", false + } + return filepath.Join(parentDir, filepath.FromSlash(path.Base(pkgPath))), true +} + +func (b *Builder) resolvePackageDirs(patterns []string) { + // make packages canonical before loading + canonicalPatterns := make([]string, 0, len(patterns)) + for _, p := range patterns { + canonicalPattern := string(canonicalizeImportPath(p)) + if _, ok := b.getResolvedPackageDir(canonicalPattern); ok { + // we already had a module dir resolved for this package + continue + } + // append the pattern + canonicalPatterns = append(canonicalPatterns, canonicalPattern) + // if this is a recursive pattern, also append the specific root to ensure it resolves (package expansion doesn't work on symlinked paths) + if strings.HasSuffix(canonicalPattern, "/...") { + canonicalPatterns = append(canonicalPatterns, strings.TrimSuffix(canonicalPattern, "/...")) + } + } + + // name + // module info to get root dir + // deps to get downstream dirs to speed up type checking + // files to get source location + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedModule | packages.NeedImports | packages.NeedDeps, + } + if len(b.context.BuildTags) > 0 { + cfg.BuildFlags = []string{"-tags", strings.Join(b.context.BuildTags, ",")} + } + if resolvedWorkingDir, err := os.Getwd(); err != nil { + klog.V(2).Infof("error resolving working dir: %v", err) + } else if symlinkResolvedWorkingDir, err := filepath.EvalSymlinks(resolvedWorkingDir); err != nil { + klog.V(2).Infof("error resolving working dir: %v", err) + } else { + cfg.Dir = symlinkResolvedWorkingDir + } + pkgs, err := packages.Load(cfg, canonicalPatterns...) + if err != nil { + klog.V(2).Infof("Error resolving packages: %v", err) + return + } + packages.Visit(pkgs, func(pkg *packages.Package) bool { + if len(pkg.Errors) > 0 { + klog.V(2).Info(pkg.Errors) + } + if pkg.Module != nil { + if len(pkg.Module.Dir) > 0 { + b.pkgDirs[pkg.Module.Path] = pkg.Module.Dir + } + if pkg.Module.Replace != nil && len(pkg.Module.Replace.Dir) > 0 { + b.pkgDirs[pkg.Module.Path] = pkg.Module.Replace.Dir + } + if pkg.Module.Error != nil { + klog.V(2).Infof("%#v\n", *pkg.Module.Error) + } + } + // use any file info we got to identify the package dir + if len(pkg.GoFiles) > 0 { + b.pkgDirs[pkg.PkgPath] = filepath.Dir(pkg.GoFiles[0]) + } else if len(pkg.OtherFiles) > 0 { + b.pkgDirs[pkg.PkgPath] = filepath.Dir(pkg.OtherFiles[0]) + } else if len(pkg.IgnoredFiles) > 0 { + b.pkgDirs[pkg.PkgPath] = filepath.Dir(pkg.IgnoredFiles[0]) + } + return true + }, nil) +} + +// AddPackagePatterns adds the specified patterns, +// which may be individual import paths as used in import directives, +// or recursive paths like `example.com/...`. +func (b *Builder) AddPackagePatterns(patterns ...string) error { + // resolve patterns to speed up load time + b.resolvePackageDirs(patterns) + + for _, d := range patterns { + var err error + if strings.HasSuffix(d, "/...") { + err = b.addPackageRecursive(strings.TrimSuffix(d, "/...")) + } else { + err = b.addPackage(d) + } + if err != nil { + return fmt.Errorf("unable to add directory %q: %v", d, err) + } + } + return nil +} + +// addPackage adds an entire package, scanning it for go files. +func (b *Builder) addPackage(importPackage string) error { + _, err := b.importPackage(importPackage, true) return err } -// AddDirRecursive is just like AddDir, but it also recursively adds -// subdirectories; it returns an error only if the path couldn't be resolved; +// addPackageRecursive is just like addPackage, but it also recursively adds +// subpackages; it returns an error only if the path couldn't be resolved; // any directories recursed into without go source are ignored. -func (b *Builder) AddDirRecursive(dir string) error { +func (b *Builder) addPackageRecursive(dir string) error { // Add the root. if _, err := b.importPackage(dir, true); err != nil { klog.Warningf("Ignoring directory %v: %v", dir, err) @@ -231,7 +360,11 @@ func (b *Builder) AddDirRecursive(dir string) error { // filepath.Walk does not follow symlinks. We therefore evaluate symlinks and use that with // filepath.Walk. - realPath, err := filepath.EvalSymlinks(b.buildPackages[dir].Dir) + buildPkg, ok := b.getLoadedBuildPackage(dir) + if !ok { + return fmt.Errorf("no loaded build package for %s", dir) + } + realPath, err := filepath.EvalSymlinks(buildPkg.Dir) if err != nil { return err } @@ -241,7 +374,11 @@ func (b *Builder) AddDirRecursive(dir string) error { rel := filepath.ToSlash(strings.TrimPrefix(filePath, realPath)) if rel != "" { // Make a pkg path. - pkg := path.Join(string(canonicalizeImportPath(b.buildPackages[dir].ImportPath)), rel) + buildPkg, ok := b.getLoadedBuildPackage(dir) + if !ok { + return fmt.Errorf("no loaded build package for %s", dir) + } + pkg := path.Join(string(canonicalizeImportPath(buildPkg.ImportPath)), rel) // Add it. if _, err := b.importPackage(pkg, true); err != nil { @@ -269,7 +406,7 @@ func (b *Builder) AddDirTo(dir string, u *types.Universe) error { if _, err := b.importPackage(dir, true); err != nil { return err } - pkg, ok := b.buildPackages[dir] + pkg, ok := b.getLoadedBuildPackage(dir) if !ok { return fmt.Errorf("no such package: %q", dir) } @@ -287,8 +424,8 @@ func (b *Builder) AddDirectoryTo(dir string, u *types.Universe) (*types.Package, if _, err := b.importPackage(dir, true); err != nil { return nil, err } - pkg, ok := b.buildPackages[dir] - if !ok { + pkg, ok := b.getLoadedBuildPackage(dir) + if !ok || pkg == nil { return nil, fmt.Errorf("no such package: %q", dir) } path := canonicalizeImportPath(pkg.ImportPath) @@ -355,10 +492,11 @@ func isErrPackageNotFound(err error) bool { // needs to import a go package. 'path' is the import path. func (b *Builder) importPackage(dir string, userRequested bool) (*tc.Package, error) { klog.V(5).Infof("importPackage %s", dir) + var pkgPath = importPathString(dir) // Get the canonical path if we can. - if buildPkg := b.buildPackages[dir]; buildPkg != nil { + if buildPkg, _ := b.getLoadedBuildPackage(dir); buildPkg != nil { canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) klog.V(5).Infof("importPackage %s, canonical path is %s", dir, canonicalPackage) pkgPath = canonicalPackage @@ -382,7 +520,7 @@ func (b *Builder) importPackage(dir string, userRequested bool) (*tc.Package, er } // Get the canonical path now that it has been added. - if buildPkg := b.buildPackages[dir]; buildPkg != nil { + if buildPkg, _ := b.getLoadedBuildPackage(dir); buildPkg != nil { canonicalPackage := canonicalizeImportPath(buildPkg.ImportPath) klog.V(5).Infof("importPackage %s, canonical path is %s", dir, canonicalPackage) pkgPath = canonicalPackage @@ -588,7 +726,7 @@ func (b *Builder) findTypesIn(pkgPath importPathString, u *types.Universe) error return nil } -func (b *Builder) importWithMode(dir string, mode build.ImportMode) (*build.Package, error) { +func (b *Builder) importWithMode(pkgPath string, mode build.ImportMode) (*build.Package, error) { // This is a bit of a hack. The srcDir argument to Import() should // properly be the dir of the file which depends on the package to be // imported, so that vendoring can work properly and local paths can @@ -600,7 +738,21 @@ func (b *Builder) importWithMode(dir string, mode build.ImportMode) (*build.Pack if err != nil { return nil, fmt.Errorf("unable to get current directory: %v", err) } - buildPkg, err := b.context.Import(filepath.ToSlash(dir), cwd, mode) + + // normalize to drop /vendor/ if present + pkgPath = string(canonicalizeImportPath(pkgPath)) + + if pkgDir, ok := b.getResolvedPackageDir(pkgPath); ok { + buildPkg, err := b.context.ImportDir(pkgDir, mode) + if err != nil { + return nil, err + } + // ensure the ImportPath is the canonical one + buildPkg.ImportPath = pkgPath + return buildPkg, nil + } + + buildPkg, err := b.context.Import(pkgPath, cwd, mode) if err != nil { return nil, err }