diff --git a/common/lib/dependabot/shared_helpers.rb b/common/lib/dependabot/shared_helpers.rb index c67c4b92c9..326c3e9a7c 100644 --- a/common/lib/dependabot/shared_helpers.rb +++ b/common/lib/dependabot/shared_helpers.rb @@ -280,10 +280,10 @@ def self.reset_global_git_config(backup_path) FileUtils.mv(backup_path, GIT_CONFIG_GLOBAL_PATH) end - def self.run_shell_command(command, allow_unsafe_shell_command: false) + def self.run_shell_command(command, allow_unsafe_shell_command: false, env: {}) start = Time.now cmd = allow_unsafe_shell_command ? command : escape_command(command) - stdout, process = Open3.capture2e(cmd) + stdout, process = Open3.capture2e(env || {}, cmd) time_taken = Time.now - start # Raise an error with the output from the shell session if the diff --git a/common/spec/dependabot/shared_helpers_spec.rb b/common/spec/dependabot/shared_helpers_spec.rb index 1f6a16c3e6..54c1bc82b6 100644 --- a/common/spec/dependabot/shared_helpers_spec.rb +++ b/common/spec/dependabot/shared_helpers_spec.rb @@ -187,9 +187,10 @@ def existing_tmp_folders describe ".run_shell_command" do let(:command) { File.join(spec_root, "helpers/test/run_bash") + " output" } + let(:env) { nil } subject(:run_shell_command) do - Dependabot::SharedHelpers.run_shell_command(command) + Dependabot::SharedHelpers.run_shell_command(command, env: env) end context "when the subprocess is successful" do @@ -221,6 +222,14 @@ def existing_tmp_folders end end + context "with an environment variable" do + let(:env) { { "TEST_ENV" => "prefix:" } } + + it "is available to the command" do + expect(run_shell_command).to eq("prefix:output\n") + end + end + context "when the subprocess exits" do let(:command) { File.join(spec_root, "helpers/test/error_bash") } diff --git a/common/spec/helpers/test/run_bash b/common/spec/helpers/test/run_bash index 73c0846b33..0acdceabc8 100755 --- a/common/spec/helpers/test/run_bash +++ b/common/spec/helpers/test/run_bash @@ -2,4 +2,4 @@ set -e -echo "$@" +echo "$TEST_ENV$@" diff --git a/go_modules/helpers/go.mod b/go_modules/helpers/go.mod index 56cf3f640a..dafd260f8b 100644 --- a/go_modules/helpers/go.mod +++ b/go_modules/helpers/go.mod @@ -2,8 +2,4 @@ module github.com/dependabot/dependabot-core/go_modules/helpers go 1.16 -require ( - github.com/Masterminds/vcs v1.13.1 - github.com/dependabot/gomodules-extracted v1.4.2 - golang.org/x/mod v0.5.1 -) +require github.com/Masterminds/vcs v1.13.1 diff --git a/go_modules/helpers/go.sum b/go_modules/helpers/go.sum index dcd6cdb6e9..240000ee2c 100644 --- a/go_modules/helpers/go.sum +++ b/go_modules/helpers/go.sum @@ -1,20 +1,2 @@ github.com/Masterminds/vcs v1.13.1 h1:NL3G1X7/7xduQtA2sJLpVpfHTNBALVNSjob6KEjPXNQ= github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/dependabot/gomodules-extracted v1.4.2 h1:3IxvHARuuSojSNUHguc6kzWgs+uQN3fdRCowJMU1kDE= -github.com/dependabot/gomodules-extracted v1.4.2/go.mod h1:cpzrmDX1COyhSDQXHfkRMw0STb0vmguBFqmrkr51h1I= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go_modules/helpers/main.go b/go_modules/helpers/main.go index f146b30692..4f8d6e8747 100644 --- a/go_modules/helpers/main.go +++ b/go_modules/helpers/main.go @@ -7,7 +7,6 @@ import ( "os" "github.com/dependabot/dependabot-core/go_modules/helpers/importresolver" - "github.com/dependabot/dependabot-core/go_modules/helpers/updatechecker" ) type HelperParams struct { @@ -32,10 +31,6 @@ func main() { funcErr error ) switch helperParams.Function { - case "getVersions": - var args updatechecker.Args - parseArgs(helperParams.Args, &args) - funcOut, funcErr = updatechecker.GetVersions(&args) case "getVcsRemoteForImport": var args importresolver.Args parseArgs(helperParams.Args, &args) diff --git a/go_modules/helpers/updatechecker/main.go b/go_modules/helpers/updatechecker/main.go deleted file mode 100644 index 9689da08d6..0000000000 --- a/go_modules/helpers/updatechecker/main.go +++ /dev/null @@ -1,93 +0,0 @@ -package updatechecker - -import ( - "context" - "errors" - "io/ioutil" - - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch" - "github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modload" - "golang.org/x/mod/modfile" - "golang.org/x/mod/semver" -) - -type Dependency struct { - Name string `json:"name"` - Version string `json:"version"` -} - -type Args struct { - Dependency *Dependency `json:"dependency"` -} - -// GetVersions returns a list of versions for the given dependency that -// are within the same major version. -func GetVersions(args *Args) (interface{}, error) { - if args.Dependency == nil { - return nil, errors.New("Expected args.dependency to not be nil") - } - - currentVersion := args.Dependency.Version - - modload.DisallowWriteGoMod() - _ = modload.LoadModFile(context.Background()) - - repo := modfetch.Lookup("direct", args.Dependency.Name) - versions, err := repo.Versions("") - if err != nil { - return nil, err - } - - excludes, err := goModExcludes(args.Dependency.Name) - if err != nil { - return nil, err - } - - currentMajor := semver.Major(currentVersion) - - var candidateVersions []string - -Outer: - for _, v := range versions { - if semver.Major(v) != currentMajor { - continue - } - - for _, exclude := range excludes { - if v == exclude { - continue Outer - } - } - - candidateVersions = append(candidateVersions, v) - } - - return candidateVersions, nil -} - -func goModExcludes(dependency string) ([]string, error) { - data, err := ioutil.ReadFile("go.mod") - if err != nil { - return nil, err - } - - var f *modfile.File - // TODO library detection - don't consider exclude etc for libraries - if "library" == "true" { - f, err = modfile.ParseLax("go.mod", data, nil) - } else { - f, err = modfile.Parse("go.mod", data, nil) - } - if err != nil { - return nil, err - } - - var excludes []string - for _, e := range f.Exclude { - if e.Mod.Path == dependency { - excludes = append(excludes, e.Mod.Version) - } - } - - return excludes, nil -} diff --git a/go_modules/lib/dependabot/go_modules/update_checker/latest_version_finder.rb b/go_modules/lib/dependabot/go_modules/update_checker/latest_version_finder.rb index ad90fa3453..b35fa305a1 100644 --- a/go_modules/lib/dependabot/go_modules/update_checker/latest_version_finder.rb +++ b/go_modules/lib/dependabot/go_modules/update_checker/latest_version_finder.rb @@ -20,9 +20,11 @@ class LatestVersionFinder /404 Not Found/, /Repository not found/, /unrecognized import path/, + /malformed module path/, # (Private) module could not be fetched /module .*: git ls-remote .*: exit status 128/m.freeze ].freeze + INVALID_VERSION_REGEX = /version "[^"]+" invalid/m.freeze PSEUDO_VERSION_REGEX = /\b\d{14}-[0-9a-f]{12}$/.freeze def initialize(dependency:, dependency_files:, credentials:, @@ -73,23 +75,22 @@ def fetch_lowest_security_fix_version def available_versions SharedHelpers.in_a_temporary_directory do SharedHelpers.with_git_configured(credentials: credentials) do - File.write("go.mod", go_mod.content) + manifest = parse_manifest + + # Set up an empty go.mod so 'go list -m' won't attempt to download dependencies. This + # appears to be a side effect of operating with GOPRIVATE=*. We'll retain any exclude + # directives to omit those versions. + File.write("go.mod", "module dummy\n") + manifest["Exclude"]&.each do |r| + SharedHelpers.run_shell_command("go mod edit -exclude=#{r['Path']}@#{r['Version']}") + end # Turn off the module proxy for now, as it's causing issues with # private git dependencies env = { "GOPRIVATE" => "*" } - version_strings = SharedHelpers.run_helper_subprocess( - command: NativeHelpers.helper_path, - env: env, - function: "getVersions", - args: { - dependency: { - name: dependency.name, - version: "v" + dependency.version - } - } - ) + versions_json = SharedHelpers.run_shell_command("go list -m -versions -json #{dependency.name}", env: env) + version_strings = JSON.parse(versions_json)["Versions"] return [version_class.new(dependency.version)] if version_strings.nil? @@ -108,6 +109,8 @@ def available_versions def handle_subprocess_error(error) if RESOLVABILITY_ERROR_REGEXES.any? { |rgx| error.message =~ rgx } ResolvabilityErrors.handle(error.message, credentials: credentials) + elsif INVALID_VERSION_REGEX =~ error.message + raise Dependabot::DependencyFileNotResolvable, error.message end raise @@ -123,6 +126,15 @@ def go_mod @go_mod ||= dependency_files.find { |f| f.name == "go.mod" } end + def parse_manifest + SharedHelpers.in_a_temporary_directory do + File.write("go.mod", go_mod.content) + json = SharedHelpers.run_shell_command("go mod edit -json") + + JSON.parse(json) || {} + end + end + def filter_prerelease_versions(versions_array) return versions_array if wants_prerelease? diff --git a/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb b/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb index 6aeeb067aa..7bc0f90642 100644 --- a/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb +++ b/go_modules/spec/dependabot/go_modules/update_checker/latest_version_finder_spec.rb @@ -66,11 +66,11 @@ module foobar end context "when already on the latest version" do - let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib/v2" } - let(:dependency_version) { "2.0.0" } + let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib/v3" } + let(:dependency_version) { "3.0.0" } it "returns the current version" do - expect(finder.latest_version).to eq(Dependabot::GoModules::Version.new("2.0.0")) + expect(finder.latest_version).to eq(Dependabot::GoModules::Version.new("3.0.0")) end end @@ -186,6 +186,34 @@ module foobar end end + context "when the dependency's major version is invalid because it's not specified in its go.mod" do + let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib/v2" } + let(:dependency_version) { "2.0.0" } + + it "raises a DependencyFileNotResolvable error" do + error_class = Dependabot::DependencyFileNotResolvable + expect { finder.latest_version }. + to raise_error(error_class) do |error| + expect(error.message).to include("github.com/dependabot-fixtures/go-modules-lib/v2") + expect(error.message).to include("version \"v2.0.0\" invalid") + end + end + end + + context "when the dependency's major version is invalid because not properly imported" do + let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-lib" } + let(:dependency_version) { "3.0.0" } + + it "raises a DependencyFileNotResolvable error" do + error_class = Dependabot::DependencyFileNotResolvable + expect { finder.latest_version }. + to raise_error(error_class) do |error| + expect(error.message).to include("github.com/dependabot-fixtures/go-modules-lib") + expect(error.message).to include("version \"v3.0.0\" invalid") + end + end + end + context "when the module is unreachable" do let(:dependency_files) { [go_mod] } let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-private" } @@ -212,7 +240,7 @@ module foobar # latest release v1.0.1 is retracted let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-retracted" } - pending "doesn't return the retracted version" do + it "doesn't return the retracted version" do expect(finder.latest_version).to eq(Dependabot::GoModules::Version.new("1.0.0")) end end