From e9044a1e65fdd8ac810a0e2d56dd2b50d9dd1b60 Mon Sep 17 00:00:00 2001
From: goto1134 <1134togo@gmail.com>
Date: Sat, 24 Aug 2024 09:57:02 +0200
Subject: [PATCH 1/2] cmd/go: add benchmark for go list -m
Updates #63136
---
src/cmd/go/list_test.go | 190 ++++++++++++++++++++++++++
src/cmd/go/testdata/list/cmd/go.mod | 15 ++
src/cmd/go/testdata/list/cmd/go.sum | 16 +++
src/cmd/go/testdata/list/empty/go.mod | 3 +
4 files changed, 224 insertions(+)
create mode 100644 src/cmd/go/list_test.go
create mode 100644 src/cmd/go/testdata/list/cmd/go.mod
create mode 100644 src/cmd/go/testdata/list/cmd/go.sum
create mode 100644 src/cmd/go/testdata/list/empty/go.mod
diff --git a/src/cmd/go/list_test.go b/src/cmd/go/list_test.go
new file mode 100644
index 00000000000000..851b85c287187a
--- /dev/null
+++ b/src/cmd/go/list_test.go
@@ -0,0 +1,190 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main_test
+
+import (
+ "internal/testenv"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+var modulesForTest = []struct {
+ name string
+ testDataFolder string
+ url string
+ dependenciesToStrip []string
+}{
+ {
+ name: "Empty",
+ testDataFolder: "empty",
+ },
+ {
+ name: "Cmd",
+ testDataFolder: "cmd",
+ },
+ {
+ name: "K8S",
+ testDataFolder: "k8s",
+ url: "https://raw.githubusercontent.com/kubernetes/kubernetes/b33ef18bdfa62d3975f2c72743373f1d69740b99",
+ dependenciesToStrip: []string{
+ "k8s.io/api",
+ "k8s.io/apiextensions-apiserver",
+ "k8s.io/apimachinery",
+ "k8s.io/apiserver",
+ "k8s.io/cli-runtime",
+ "k8s.io/client-go",
+ "k8s.io/cloud-provider",
+ "k8s.io/cluster-bootstrap",
+ "k8s.io/code-generator",
+ "k8s.io/component-base",
+ "k8s.io/component-helpers",
+ "k8s.io/controller-manager",
+ "k8s.io/cri-api",
+ "k8s.io/cri-client",
+ "k8s.io/csi-translation-lib",
+ "k8s.io/dynamic-resource-allocation",
+ "k8s.io/endpointslice",
+ "k8s.io/kms",
+ "k8s.io/kube-aggregator",
+ "k8s.io/kube-controller-manager",
+ "k8s.io/kube-proxy",
+ "k8s.io/kube-scheduler",
+ "k8s.io/kubectl",
+ "k8s.io/kubelet",
+ "k8s.io/metrics",
+ "k8s.io/mount-utils",
+ "k8s.io/pod-security-admission",
+ "k8s.io/sample-apiserver",
+ "k8s.io/sample-cli-plugin",
+ "k8s.io/sample-controller",
+ },
+ },
+}
+
+func BenchmarkListModules(b *testing.B) {
+ testenv.MustHaveExec(b)
+ gotool, err := testenv.GoTool()
+ if err != nil {
+ b.Fatal(err)
+ }
+
+ tempDir := b.TempDir()
+ if err := prepareTestModules(tempDir); err != nil {
+ b.Fatal(err)
+ }
+
+ b.ResetTimer()
+ for _, m := range modulesForTest {
+ // Do not run in parallel. GOPROXY rate limits may affect parallel executions.
+ b.Run(m.name, func(b *testing.B) {
+ // We collect extra metrics.
+ var n, userTime, systemTime int64
+ modDir := filepath.Join(tempDir, m.testDataFolder, "go.mod")
+ for i := 0; i < b.N; i++ {
+ cmd := testenv.Command(b, gotool, "list", "-m", "-modfile="+modDir, "-mod=readonly", "all")
+
+ // Guarantees clean module cache for every execution.
+ gopath := b.TempDir()
+ cmd.Env = append(cmd.Env, "GOPATH="+gopath)
+
+ if err := cmd.Run(); err != nil {
+ b.Fatal(err)
+ }
+ atomic.AddInt64(&n, 1)
+ atomic.AddInt64(&userTime, int64(cmd.ProcessState.UserTime()))
+ atomic.AddInt64(&systemTime, int64(cmd.ProcessState.SystemTime()))
+
+ }
+ b.ReportMetric(float64(userTime)/float64(n), "user-ns/op")
+ b.ReportMetric(float64(systemTime)/float64(n), "sys-ns/op")
+ })
+ }
+}
+
+func prepareTestModules(tempDir string) error {
+ err := os.CopyFS(tempDir, os.DirFS(filepath.Join("testdata", "list")))
+ if err != nil {
+ return err
+ }
+ for _, m := range modulesForTest {
+ if m.url != "" {
+ modulePath := filepath.Join(tempDir, m.testDataFolder)
+ if err := downloadModule(modulePath, m.url, m.dependenciesToStrip); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func downloadModule(modulePath string, url string, dependenciesToStrip []string) error {
+
+ if err := os.Mkdir(modulePath, 0755); err != nil {
+ return err
+ }
+ sumText, err := fetchText(url + "/go.sum")
+ if err != nil {
+ return err
+ }
+
+ if err := writeFile(filepath.Join(modulePath, "go.sum"), sumText); err != nil {
+ return err
+ }
+
+ modText, err := fetchText(url + "/go.mod")
+ if err != nil {
+ return err
+ }
+
+ strippedModText := stripDependencies(modText, dependenciesToStrip)
+
+ return writeFile(filepath.Join(modulePath, "go.mod"), strippedModText)
+}
+
+func stripDependencies(goModText string, dependenciesToStrip []string) string {
+ var filteredLines []string
+ lines := strings.Split(goModText, "\n")
+ for _, line := range lines {
+ if !containsAny(line, dependenciesToStrip) {
+ filteredLines = append(filteredLines, line)
+ }
+ }
+ filteredText := strings.Join(filteredLines, "\n")
+ return filteredText
+}
+
+func containsAny(text string, options []string) bool {
+ for _, option := range options {
+ if strings.Contains(text, option) {
+ return true
+ }
+ }
+ return false
+}
+
+func writeFile(filePath, content string) error {
+ out, err := os.Create(filePath)
+ if err != nil {
+ return err
+ }
+ defer out.Close()
+ _, err = io.WriteString(out, content)
+ return err
+}
+
+func fetchText(url string) (string, error) {
+ resp, err := http.Get(url)
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ return string(body), err
+}
diff --git a/src/cmd/go/testdata/list/cmd/go.mod b/src/cmd/go/testdata/list/cmd/go.mod
new file mode 100644
index 00000000000000..7a426887b48cb0
--- /dev/null
+++ b/src/cmd/go/testdata/list/cmd/go.mod
@@ -0,0 +1,15 @@
+module cmd
+
+go 1.22
+
+require (
+ github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17
+ golang.org/x/arch v0.6.0
+ golang.org/x/mod v0.14.0
+ golang.org/x/sync v0.5.0
+ golang.org/x/sys v0.15.0
+ golang.org/x/term v0.15.0
+ golang.org/x/tools v0.16.2-0.20231218185909-83bceaf2424d
+)
+
+require github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab // indirect
diff --git a/src/cmd/go/testdata/list/cmd/go.sum b/src/cmd/go/testdata/list/cmd/go.sum
new file mode 100644
index 00000000000000..8ea3d75bd1518d
--- /dev/null
+++ b/src/cmd/go/testdata/list/cmd/go.sum
@@ -0,0 +1,16 @@
+github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17 h1:0h35ESZ02+hN/MFZb7XZOXg+Rl9+Rk8fBIf5YLws9gA=
+github.com/google/pprof v0.0.0-20230811205829-9131a7e9cc17/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
+github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab h1:BA4a7pe6ZTd9F8kXETBoijjFJ/ntaa//1wiH9BZu4zU=
+github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
+golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc=
+golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/tools v0.16.2-0.20231218185909-83bceaf2424d h1:9YOyUBubvYqtjjtZBnI62JT9/QB9jfPwOQ7xLeyuOIU=
+golang.org/x/tools v0.16.2-0.20231218185909-83bceaf2424d/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
diff --git a/src/cmd/go/testdata/list/empty/go.mod b/src/cmd/go/testdata/list/empty/go.mod
new file mode 100644
index 00000000000000..99d185d5525b85
--- /dev/null
+++ b/src/cmd/go/testdata/list/empty/go.mod
@@ -0,0 +1,3 @@
+module empty
+
+go 1.22
From 2b864bee0086330e79a79d603df50a0a5f43ac92 Mon Sep 17 00:00:00 2001
From: goto1134 <1134togo@gmail.com>
Date: Sat, 24 Aug 2024 10:13:41 +0200
Subject: [PATCH 2/2] cmd/go: remove redundant writeFile function
Updates #63136
---
src/cmd/go/list_test.go | 26 ++++++++------------------
1 file changed, 8 insertions(+), 18 deletions(-)
diff --git a/src/cmd/go/list_test.go b/src/cmd/go/list_test.go
index 851b85c287187a..08fa42a0296fc1 100644
--- a/src/cmd/go/list_test.go
+++ b/src/cmd/go/list_test.go
@@ -134,7 +134,7 @@ func downloadModule(modulePath string, url string, dependenciesToStrip []string)
return err
}
- if err := writeFile(filepath.Join(modulePath, "go.sum"), sumText); err != nil {
+ if err := os.WriteFile(filepath.Join(modulePath, "go.sum"), sumText, 0644); err != nil {
return err
}
@@ -145,19 +145,19 @@ func downloadModule(modulePath string, url string, dependenciesToStrip []string)
strippedModText := stripDependencies(modText, dependenciesToStrip)
- return writeFile(filepath.Join(modulePath, "go.mod"), strippedModText)
+ return os.WriteFile(filepath.Join(modulePath, "go.mod"), strippedModText, 0644)
}
-func stripDependencies(goModText string, dependenciesToStrip []string) string {
+func stripDependencies(goModText []byte, dependenciesToStrip []string) []byte {
var filteredLines []string
- lines := strings.Split(goModText, "\n")
+ lines := strings.Split(string(goModText), "\n")
for _, line := range lines {
if !containsAny(line, dependenciesToStrip) {
filteredLines = append(filteredLines, line)
}
}
filteredText := strings.Join(filteredLines, "\n")
- return filteredText
+ return []byte(filteredText)
}
func containsAny(text string, options []string) bool {
@@ -169,22 +169,12 @@ func containsAny(text string, options []string) bool {
return false
}
-func writeFile(filePath, content string) error {
- out, err := os.Create(filePath)
- if err != nil {
- return err
- }
- defer out.Close()
- _, err = io.WriteString(out, content)
- return err
-}
-
-func fetchText(url string) (string, error) {
+func fetchText(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
- return "", err
+ return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
- return string(body), err
+ return body, err
}