download: add list from go cli

cmd/olympus/actions/app.go

@@ -13,6 +13,7 @@ import (
 	"github.com/gomods/athens/pkg/cdn/metadata/azurecdn"
 	"github.com/gomods/athens/pkg/config/env"
 	"github.com/gomods/athens/pkg/download"
+	"github.com/gomods/athens/pkg/download/goget"
 	"github.com/gomods/athens/pkg/eventlog"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/storage"
@@ -114,8 +115,9 @@ func App(config *AppConfig) (*buffalo.App, error) {
 		app.POST("/cachemiss", cachemissHandler(w))
 		app.POST("/push", pushNotificationHandler(w))
 
+		dp := goget.New()
 		// Download Protocol
-		app.GET(download.PathList, download.ListHandler(config.Storage, renderEng))
+		app.GET(download.PathList, download.ListHandler(dp, lggr, renderEng))
 		app.GET(download.PathVersionInfo, download.VersionInfoHandler(config.Storage, renderEng))
 		app.GET(download.PathVersionModule, download.VersionModuleHandler(config.Storage, renderEng))
 		app.GET(download.PathVersionZip, download.VersionZipHandler(config.Storage, renderEng, lggr))

cmd/proxy/actions/app_proxy.go

@@ -3,6 +3,7 @@ package actions
 import (
 	"github.com/gobuffalo/buffalo"
 	"github.com/gomods/athens/pkg/download"
+	"github.com/gomods/athens/pkg/download/goget"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/storage"
@@ -16,8 +17,9 @@ func addProxyRoutes(
 ) error {
 	app.GET("/", proxyHomeHandler)
 
+	dp := download.New(goget.New(), storage)
 	// Download Protocol
-	app.GET(download.PathList, download.ListHandler(storage, proxy))
+	app.GET(download.PathList, download.ListHandler(dp, lggr, proxy))
 	app.GET(download.PathVersionInfo, cacheMissHandler(download.VersionInfoHandler(storage, proxy), app.Worker, mf, lggr))
 	app.GET(download.PathVersionModule, cacheMissHandler(download.VersionModuleHandler(storage, proxy), app.Worker, mf, lggr))
 	app.GET(download.PathVersionZip, cacheMissHandler(download.VersionZipHandler(storage, proxy, lggr), app.Worker, mf, lggr))

pkg/config/env/go.go

@@ -21,3 +21,9 @@ func GoPath() (string, error) {
 
 	return env, nil
 }
+
+// GoBinPath returns the path to Go's executable binary
+// this binary must have Go Modules enabled.
+func GoBinPath() string {
+	return envy.Get("GO_BIN_PATH", "vgo")
+}

pkg/download/download.go

@@ -0,0 +1,108 @@
+package download
+
+import (
+	"context"
+	"io"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/storage"
+)
+
+// Protocol is the download protocol which mirrors
+// the http requests that cmd/go makes to the proxy.
+type Protocol interface {
+	// List implements GET /{module}/@v/list
+	List(ctx context.Context, mod string) ([]string, error)
+
+	// Info implements GET /{module}/@v/{version}.info
+	Info(ctx context.Context, mod, ver string) ([]byte, error)
+
+	// Latest implements GET /{module}/@latest
+	Latest(ctx context.Context, mod string) (*storage.RevInfo, error)
+
+	// GoMod implements GET /{module}/@v/{version}.mod
+	GoMod(ctx context.Context, mod, ver string) ([]byte, error)
+
+	// Zip implements GET /{module}/@v/{version}.zip
+	Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error)
+
+	// Version is a helper method to get Info, GoMod, and Zip together.
+	Version(ctx context.Context, mod, ver string) (*storage.Version, error)
+}
+
+type protocol struct {
+	s  storage.Backend
+	dp Protocol
+}
+
+// New takes an upstream Protocol and storage
+// it always prefers storage, otherwise it goes to upstream
+// and fills the storage with the results.
+func New(dp Protocol, s storage.Backend) Protocol {
+	return &protocol{dp: dp, s: s}
+}
+
+func (p *protocol) List(ctx context.Context, mod string) ([]string, error) {
+	return p.dp.List(ctx, mod)
+}
+
+func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
+	const op errors.Op = "protocol.Info"
+	v, err := p.s.Get(mod, ver)
+	if errors.ErrNotFound(err) {
+		v, err = p.fillCache(ctx, mod, ver)
+	}
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+
+	return v.Info, nil
+}
+
+func (p *protocol) fillCache(ctx context.Context, mod, ver string) (*storage.Version, error) {
+	const op errors.Op = "protocol.fillCache"
+	v, err := p.dp.Version(ctx, mod, ver)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	err = p.s.Save(ctx, mod, ver, v.Mod, v.Zip, v.Info)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+
+	return v, nil
+}
+
+func (p *protocol) Latest(ctx context.Context, mod string) (*storage.RevInfo, error) {
+	return p.dp.Latest(ctx, mod)
+}
+
+func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
+	const op errors.Op = "protocol.GoMod"
+	v, err := p.s.Get(mod, ver)
+	if errors.ErrNotFound(err) {
+		v, err = p.fillCache(ctx, mod, ver)
+	}
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+
+	return v.Mod, nil
+}
+
+func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
+	const op errors.Op = "protocol.Zip"
+	v, err := p.s.Get(mod, ver)
+	if errors.ErrNotFound(err) {
+		v, err = p.fillCache(ctx, mod, ver)
+	}
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+
+	return v.Zip, nil
+}
+
+func (p *protocol) Version(ctx context.Context, mod, ver string) (*storage.Version, error) {
+	return p.dp.Version(ctx, mod, ver)
+}

pkg/download/goget/goget.go

@@ -0,0 +1,153 @@
+package goget
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/config/env"
+	"github.com/gomods/athens/pkg/download"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/module"
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/spf13/afero"
+)
+
+// New returns a download protocol by using
+// go get. You must have a modules supported
+// go binary for this to work.
+func New() download.Protocol {
+	return &goget{
+		goBinPath: env.GoBinPath(),
+		fs:        afero.NewOsFs(),
+	}
+}
+
+type goget struct {
+	goBinPath string
+	fs        afero.Fs
+}
+
+func (gg *goget) List(ctx context.Context, mod string) ([]string, error) {
+	const op errors.Op = "goget.List"
+	lr, err := gg.list(op, mod)
+	if err != nil {
+		return nil, err
+	}
+
+	return lr.Versions, nil
+}
+
+type listResp struct {
+	Path     string
+	Version  string
+	Versions []string `json:"omitempty"`
+	Time     time.Time
+}
+
+func (gg *goget) Info(ctx context.Context, mod string, ver string) ([]byte, error) {
+	const op errors.Op = "goget.Info"
+	v, err := gg.Version(ctx, mod, ver)
+	if err != nil {
+		return nil, errors.E(op)
+	}
+	v.Zip.Close()
+
+	return v.Info, nil
+}
+
+func (gg *goget) Latest(ctx context.Context, mod string) (*storage.RevInfo, error) {
+	const op errors.Op = "goget.Latest"
+	lr, err := gg.list(op, mod)
+	if err != nil {
+		return nil, err
+	}
+
+	pseudoInfo := strings.Split(lr.Version, "-")
+	if len(pseudoInfo) < 3 {
+		return nil, errors.E(op, fmt.Errorf("malformed pseudoInfo %v", lr.Version))
+	}
+	return &storage.RevInfo{
+		Name:    pseudoInfo[2],
+		Short:   pseudoInfo[2],
+		Time:    lr.Time,
+		Version: lr.Version,
+	}, nil
+}
+
+func (gg *goget) list(op errors.Op, mod string) (*listResp, error) {
+	hackyPath, err := afero.TempDir(gg.fs, "", "hackymod")
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	defer gg.fs.RemoveAll(hackyPath)
+	err = module.Dummy(gg.fs, hackyPath)
+	cmd := exec.Command(
+		gg.goBinPath,
+		"list", "-m", "-versions", "-json",
+		config.FmtModVer(mod, "latest"),
+	)
+	cmd.Dir = hackyPath
+
+	bts, err := cmd.CombinedOutput()
+	if err != nil {
+		errFmt := fmt.Errorf("%v: %s", err, bts)
+		return nil, errors.E(op, errFmt)
+	}
+
+	// ugly hack until go cli implements -quiet flag.
+	// https://github.com/golang/go/issues/26628
+	if bytes.HasPrefix(bts, []byte("go: finding")) {
+		bts = bts[bytes.Index(bts, []byte{'\n'}):]
+	}
+
+	var lr listResp
+	err = json.Unmarshal(bts, &lr)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+
+	return &lr, nil
+}
+
+func (gg *goget) GoMod(ctx context.Context, mod string, ver string) ([]byte, error) {
+	const op errors.Op = "goget.Info"
+	v, err := gg.Version(ctx, mod, ver)
+	if err != nil {
+		return nil, errors.E(op)
+	}
+	v.Zip.Close()
+
+	return v.Mod, nil
+}
+
+func (gg *goget) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
+	const op errors.Op = "goget.Info"
+	v, err := gg.Version(ctx, mod, ver)
+	if err != nil {
+		return nil, errors.E(op)
+	}
+
+	return v.Zip, nil
+}
+
+func (gg *goget) Version(ctx context.Context, mod, ver string) (*storage.Version, error) {
+	const op errors.Op = "goget.Version"
+	fetcher, _ := module.NewGoGetFetcher(gg.goBinPath, gg.fs) // TODO: remove err from func call
+	ref, err := fetcher.Fetch(mod, ver)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	v, err := ref.Read()
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+
+	return v, nil
+}

pkg/download/goget/goget_test.go

@@ -0,0 +1,37 @@
+package goget
+
+import (
+	"context"
+	"testing"
+)
+
+type testCase struct {
+	name    string
+	mod     string
+	version string
+}
+
+// TODO(marwan): we should create Test Repos under github.com/gomods
+// so we can get reproducible results from live VCS repos.
+// For now, I cannot test that github.com/pkg/errors returns v0.8.0
+// from goget.Latest, because they could very well introduce a new tag
+// in the near future.
+var tt = []testCase{
+	{"basic list", "github.com/pkg/errors", "latest"},
+	{"list non tagged", "github.com/marwan-at-work/gowatch", "latest"},
+	{"list vanity", "golang.org/x/tools", "latest"},
+}
+
+func TestList(t *testing.T) {
+	dp := New()
+	ctx := context.Background()
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := dp.List(ctx, tc.mod) // TODO ensure list is correct per TODO above.
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}

pkg/download/list.go

@@ -7,29 +7,31 @@ import (
 	"github.com/bketelsen/buffet"
 	"github.com/gobuffalo/buffalo"
 	"github.com/gobuffalo/buffalo/render"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/paths"
-	"github.com/gomods/athens/pkg/storage"
-	errs "github.com/pkg/errors"
 )
 
 // PathList URL.
 const PathList = "/{module:.+}/@v/list"
 
 // ListHandler implements GET baseURL/module/@v/list
-func ListHandler(lister storage.Lister, eng *render.Engine) func(c buffalo.Context) error {
+func ListHandler(dp Protocol, lggr *log.Logger, eng *render.Engine) func(c buffalo.Context) error {
 	return func(c buffalo.Context) error {
 		sp := buffet.SpanFromContext(c)
 		sp.SetOperationName("listHandler")
 		mod, err := paths.GetModule(c)
 		if err != nil {
-			return err
+			lggr.SystemErr(err)
+			return c.Render(500, nil)
 		}
-		versions, err := lister.List(c, mod)
-		if storage.IsNotFoundError(err) {
-			return c.Render(http.StatusNotFound, eng.JSON(err.Error()))
-		} else if err != nil {
-			return errs.WithStack(err)
+
+		versions, err := dp.List(c, mod)
+		if err != nil {
+			lggr.SystemErr(err)
+			return c.Render(errors.Kind(err), eng.JSON(errors.KindText(err)))
 		}
+
 		return c.Render(http.StatusOK, eng.String(strings.Join(versions, "\n")))
 	}
 }

pkg/errors/kinds.go

@@ -0,0 +1,6 @@
+package errors
+
+// ErrNotFound helper function for KindNotFound
+func ErrNotFound(err error) bool {
+	return Kind(err) == KindNotFound
+}