diff --git a/Gopkg.lock b/Gopkg.lock index cdc07e883648..9d5b485e1e52 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1028,6 +1028,13 @@ pruneopts = "UT" revision = "9f7362b77ad333b26c01c99de52a11bdb650ded2" +[[projects]] + digest = "1:2eeeebccad4f052e6037527e86b8114c6bfd184a97f84d4449a5ea6ad202c216" + name = "github.com/maruel/panicparse" + packages = ["stack"] + pruneopts = "UT" + revision = "f20d4c4d746f810c9110e21928d4135e1f2a3efa" + [[projects]] digest = "1:130d1249f8a867da8115c3de6cddf0ac29ca405117c88aa1541db690384a51c6" name = "github.com/marusama/semaphore" @@ -1786,6 +1793,7 @@ "github.com/lightstep/lightstep-tracer-go", "github.com/linkedin/goavro", "github.com/lufia/iostat", + "github.com/maruel/panicparse/stack", "github.com/marusama/semaphore", "github.com/mattn/go-isatty", "github.com/mattn/goveralls", diff --git a/Gopkg.toml b/Gopkg.toml index f10472fba056..7585e776e716 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -36,6 +36,10 @@ ignored = [ name = "github.com/docker/docker" branch = "master" +[[constraint]] + name = "github.com/maruel/panicparse" + revision = "f20d4c4d746f810c9110e21928d4135e1f2a3efa" + # https://github.com/getsentry/raven-go/pull/139 [[constraint]] name = "github.com/getsentry/raven-go" diff --git a/pkg/server/debug/goroutineui/dump.go b/pkg/server/debug/goroutineui/dump.go new file mode 100644 index 000000000000..a66227d11c08 --- /dev/null +++ b/pkg/server/debug/goroutineui/dump.go @@ -0,0 +1,100 @@ +// Copyright 2019 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package goroutineui + +import ( + "bytes" + "io" + "io/ioutil" + "runtime" + "sort" + "strings" + "time" + + "github.com/maruel/panicparse/stack" +) + +// stacks is a wrapper for runtime.Stack that attempts to recover the data for all goroutines. +func stacks() []byte { + // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. + var trace []byte + for n := 1 << 20; /* 1mb */ n <= (1 << 29); /* 512mb */ n *= 2 { + trace = make([]byte, n) + nbytes := runtime.Stack(trace, true /* all */) + if nbytes < len(trace) { + return trace[:nbytes] + } + } + return trace +} + +// A Dump wraps a goroutine dump with functionality to output through panicparse. +type Dump struct { + err error + + now time.Time + buckets []*stack.Bucket +} + +// NewDump grabs a goroutine dump and associates it with the supplied time. +func NewDump(now time.Time) Dump { + return NewDumpFromBytes(now, stacks()) +} + +// NewDumpFromBytes is like NewDump, but treats the supplied bytes as a goroutine +// dump. +func NewDumpFromBytes(now time.Time, b []byte) Dump { + c, err := stack.ParseDump(bytes.NewReader(b), ioutil.Discard, true /* guesspaths */) + if err != nil { + return Dump{err: err} + } + return Dump{now: now, buckets: stack.Aggregate(c.Goroutines, stack.AnyValue)} +} + +// SortCountDesc rearranges the goroutine buckets such that higher multiplicities +// appear earlier. +func (d Dump) SortCountDesc() { + sort.Slice(d.buckets, func(i, j int) bool { + a, b := d.buckets[i], d.buckets[j] + return len(a.IDs) > len(b.IDs) + }) +} + +// SortWaitDesc rearranges the goroutine buckets such that goroutines that have +// longer wait times appear earlier. +func (d Dump) SortWaitDesc() { + sort.Slice(d.buckets, func(i, j int) bool { + a, b := d.buckets[i], d.buckets[j] + return a.SleepMax > b.SleepMax + }) +} + +// HTML writes the rendered output of panicparse into the supplied Writer. +func (d Dump) HTML(w io.Writer) error { + if d.err != nil { + return d.err + } + return writeToHTML(w, d.buckets, d.now) +} + +// HTMLString is like HTML, but returns a string. If an error occurs, its string +// representation is returned. +func (d Dump) HTMLString() string { + var w strings.Builder + if err := d.HTML(&w); err != nil { + return err.Error() + } + return w.String() +} diff --git a/pkg/server/debug/goroutineui/dump_test.go b/pkg/server/debug/goroutineui/dump_test.go new file mode 100644 index 000000000000..7e2074da5976 --- /dev/null +++ b/pkg/server/debug/goroutineui/dump_test.go @@ -0,0 +1,98 @@ +// Copyright 2019 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package goroutineui + +import ( + "io/ioutil" + "testing" + "time" + + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/stretchr/testify/assert" +) + +func TestDumpHTML(t *testing.T) { + defer leaktest.AfterTest(t)() + + now := time.Time{} + dump := NewDumpFromBytes(now, []byte(fixture)) + dump.SortWaitDesc() // noop + dump.SortCountDesc() // noop + act := dump.HTMLString() + + if false { + _ = ioutil.WriteFile("test.html", []byte(act), 0644) + } + assert.Equal(t, exp, act) +} + +// This is the output of debug.PrintStack() on the go playground. +const fixture = `goroutine 1 [running]: +runtime/debug.Stack(0x434070, 0xddb11, 0x0, 0x40e0f8) + /usr/local/go/src/runtime/debug/stack.go:24 +0xc0 +runtime/debug.PrintStack() + /usr/local/go/src/runtime/debug/stack.go:16 +0x20 +main.main() + /tmp/sandbox157492124/main.go:6 +0x20` + +const exp = ` + +PanicParse + + +
Generated on 0001-01-01 00:00:00 +0000 UTC. +
+
+ +

Running Routine

+ 1: running + +

Stack

+ + - stack.go:24 Stack({[{4407408 } {908049 } {0 } {4251896 }] [] false})
+ - stack.go:16 PrintStack({[] [] false})
+ - .:0 main({[] [] false})
+ + +
+` diff --git a/pkg/server/debug/goroutineui/html.go b/pkg/server/debug/goroutineui/html.go new file mode 100644 index 000000000000..0f68cdd623c4 --- /dev/null +++ b/pkg/server/debug/goroutineui/html.go @@ -0,0 +1,202 @@ +// Copyright 2019 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Copyright 2017 Marc-Antoine Ruel. All rights reserved. +// Use of this source code is governed under the Apache License, Version 2.0 +// that can be found in the LICENSE file. + +// NB: this file's original lives at: +// https://github.com/maruel/panicparse/blob/master/internal/html.go +// +// Please modify this file only when absolutely necessary so that we can +// pick up updates easily. + +package goroutineui + +import ( + "html/template" + "io" + "time" + + "github.com/maruel/panicparse/stack" +) + +func writeToHTML(w io.Writer, buckets []*stack.Bucket, now time.Time) error { + m := template.FuncMap{ + "funcClass": funcClass, + "notoColorEmoji1F4A3": notoColorEmoji1F4A3, + } + if len(buckets) > 1 { + m["routineClass"] = routineClass + } else { + m["routineClass"] = func(bucket *stack.Bucket) template.HTML { return "Routine" } + } + t, err := template.New("htmlTpl").Funcs(m).Parse(htmlTpl) + if err != nil { + return err + } + data := struct { + Buckets []*stack.Bucket + Now time.Time + }{buckets, now.Truncate(time.Second)} + + return t.Execute(w, data) +} + +func funcClass(line *stack.Call) template.HTML { + if line.IsStdlib { + if line.Func.IsExported() { + return "FuncStdLibExported" + } + return "FuncStdLib" + } else if line.IsPkgMain() { + return "FuncMain" + } else if line.Func.IsExported() { + return "FuncOtherExported" + } + return "FuncOther" +} + +func routineClass(bucket *stack.Bucket) template.HTML { + if bucket.First { + return "RoutineFirst" + } + return "Routine" +} + +const htmlTpl = ` +{{- define "RenderCall" -}} +{{.SrcLine}} {{.Func.Name}}({{.Args}}) +{{- end -}} + + +PanicParse + + +
Generated on {{.Now.String}}. +
+
+{{range .Buckets}} +

{{if .First}}Running {{end}}Routine

+ {{len .IDs}}: {{.State}} + {{if .SleepMax -}} + {{- if ne .SleepMin .SleepMax}} [{{.SleepMin}}~{{.SleepMax}} minutes] + {{- else}} [{{.SleepMax}} minutes] + {{- end -}} + {{- end}} + {{if .Locked}} [locked] + {{- end -}} + {{- if .CreatedBy.SrcPath}} [Created by {{template "RenderCall" .CreatedBy}}] + {{- end -}} +

Stack

+ {{range .Signature.Stack.Calls}} + - {{template "RenderCall" .}}
+ {{- end}} + {{if .Stack.Elided}}(...)
{{end}} +{{end}} +
+` + +// notoColorEmoji1F4A3 is the bomb emoji U+1F4A3 in Noto Emoji as a PNG. +// +// Source: https://www.google.com/get/noto/help/emoji/smileys-people.html +// License: http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=OFL +// +// Created with: +// python -c "import base64;a=base64.b64encode(open('emoji_u1f4a3.png','rb').read()); print '\n'.join(a[i:i+70] for i in range(0,len(a),70))" +func notoColorEmoji1F4A3() template.HTML { + return "" + + "iVBORw0KGgoAAAANSUhEUgAAAIgAAACACAMAAADnN9ENAAAA5FBMVEVMaXFvTjQhISEhIS" + + "EhISEhISEhISEhISEhISEhISEhISEhISHRbBTRbBTRbBQhISHRbBTRbBTRbBTRbBQ6OjrR" + + "bBTZcBTsehbzfhf1fxfidRVOTk75oiL7uCr90zL3kB3/6zv2hhlHMR5OTk5KSkpKSkpRUV" + + "FKSkpKSkpMTExBQUE2NjYpKSlOTk5EREQ8PDw4ODgtLS1RUVFQUFAmJiYkJCQ0NDRLS0tT" + + "U1MoKCgyMjJHR0dTU1NdXV0rKytXV1dVVVUvLy9eXl5aWlpcXFxcXFxeXl5fX186OjphYW" + + "E/Pz9KSkrdB5CTAAAATHRSTlMAEUZggJjf/6/vcL86e1nP2//vmSC7//////9E////////" + + "/2WPpYHp////////////////////z/////8w6P////+p/8f///////+/QBb3BQAACWJJRE" + + "FUeAHslgWC6yAURUOk1N3d3d29+9/SzzxCStvQEfp9zgZycu8DnvTNN78YJCuqqsry77WQ" + + "NRum2BX0u7JQiYWJw/l7NBz4AderQkFut8frdn+kFBu2wodeYeHxBwjBkFd6StiFOYiboF" + + "CAxf9MRXHc9KGpqurDBpqghzcYuCOCeMp21oKelTBVCQt5kDiisXhCJ5aMQkFu6+lg4tDY" + + "r2rikRCPKFgQYlGeiYpN7OHbpMj8OgQ8PAGdWIIlnnwbFKYdtgDAJj+MDgZSX/Zwsx4mby" + + "YR6RbntRZVegQDX7/tI1YexMTNObQ+y992EUWRQJIJQjqTzWRyRjtRiMTqKtWQJCTCD4TM" + + "aS6bBzLGxLKRKNer1MELX0wEmYHk8pSMGUmIlMI+LC7uTeEQmhEvnZBC/kqGTolfkmSn72" + + "NPbFhoWOHswmczeYYC7YaV4MfBHl+BEYmCSJYVSUM3ukgRM5C7g4edqAqLMBq0m1sRh8oe" + + "Fl4zzp8swtFApXKlmkIkECD8+mpYEZ8iWZGq1YFKKazg582IDiu8bk7Ob5YaOnVCs9UWu+" + + "C99D7LWR5fVeY/YpVOp9Po9rp1g25/AIGIXmhp0yMLgSTIhcYDVYaj0ag1Hk/a0yZ1qVVK" + + "6KsmfnMVSbMepBkv32M2nw87rfZioatMxsveirqUBLoxHr1CJpvNZmBQSSB+icd6M9dFpt" + + "tt21CZ4EHfKKny9Ug4awA/kNRmt993loPBAFSICcbtFpRUeuViFIPFiENpp9NYHg6HexW8" + + "1Suaiaysscc8gsg6jdTxpFNf6oALUaEmBz2SsMBKEkjeL88Bt8VFWj1fCKvWdDolKmYoYD" + + "L5ejcSApNoMsZqBH+0byfbiStNEICbFZt/uIN3WtpASWIUwgOiZWgMyPL7v8+tCuIkBdlw" + + "W4WWHS/AdyKzRCEfXy7Iw+PHntlti+k0TZ1FKCJJgteV0wHGhr/1Lvp4/HGQvGdJVVVTZy" + + "HFl6TGDEIM3FiUIvnrv53zkXyH4PNzm5mknrhUNo7CUjAeSIbGmNW3Vih/gHGKY1jE53uc" + + "1BJYSOF4KCmM6YcqaPmvy/86l88MKPbzcXIKLKSwFJFUPMDtpvPDKT63cZKMJSCRglJE4k" + + "7x0hjTaZHAOpzjYBIKCpsThpQLSc4D3GaOdWQ0+CEFEo7HSkpIxjzAraXz4RxbURhJQQtK" + + "gYSdYE2mPMBtZYWxjMggIRYLKSiFks0Gw9nExjy16XBnxYBxNHghRSTcE65JEXM4rTl2qI" + + "OKkRdnIcWXcDgzXktam8s76zBQzOcZ4q6IsIBCSVVhYTmckpJ2HWBA8XoMMEri1oQnJ89L" + + "x++3cF6U46hYI8CQ4ks4HBzhYdHK0+TDc5ABxDsCCygi4ZpIJYvF0EIGqzsdffvtskschA" + + "4wLGHrcrRoCYfDSnBVe+nc5Yis4zAWRwYHFQwpFxKBuEq6Uyt5untBCs8hjB1DCiVnlfDg" + + "5PkCdzUT3TmYDINxezqH46jY7+3FxF0Vd75EVcLZdPPCmEH4cFb202RBfIdT2K4ONo5CyQ" + + "iSE+T5NJvu8q4z/LE/fBYWwsHY/Xgn45NxGEr8SvyDc4R06zt+W0S7wyGrk1MhdJAhkr2T" + + "rEXy09ng/toLL2SfcENkMHRoiYVkrERBnKQKriSynzmqORlAlMObjgyHs+G5kSXp5sGV/N" + + "jZQmQyKMQOxnNIpBI1m40sCSoJOjgPux0LKW4XQgkOjpqNBwm9wPYtJFGToeNaJaPjbPS2" + + "utRh98bvu/1rLZAMEFWISAjxl0RBNkE//Fb2U4sTBI5bkJ2/JBqCFCHfOH27mBMHKXzI4R" + + "pk/1PI8zmkCpnNx3aXaYh1NICkF5BZwKOkYz+1aBUS+OYmsp9aa8i+wWjUjuDc9BqvyPa9" + + "AkSW9b5Tg0yNeWm8ItusmkwuniP6wUrHBSTREFmSpk+R7dZMq4l6sh6uP1kdBLc09WQVyL" + + "D5Rc1eCGtAkiPk5pLIZDKuiH7EM40hD4R426oqufmlNwHEOs4hSdN7WmQhqYOoLxslEYd8" + + "1ejTK5C6MWTtIN6SuHPD4fgSOvxCrnznBZxfQtbPrOTi7kyJdkghNyCpMV+NIP31+4gQVs" + + "Itubg9yzVerqxyePWKhEHWDiLnhpVQAgoCBh08u7qQuyCv69FSKrmQgLLDm3jHEIdXiJ5M" + + "IOTxdZ0B4h0cSvzfnFswzh1SiJ5MACSyn7h89iuhJIMEFCoc49KhJxN2aghxlSiJvJdg1n" + + "DMnUMV4k0m9Dmysp/3zErOJKDAwrxahnKcCuFkAp6sjP20qauEwzmTgIJAAYbvuCjEhzT/" + + "0htkr/VmyX0VCSnOwoABR0EHBnOlkLw55Ct7HTvImUQoFsN4L1rVS0VVSMh9pD/P4pmTYD" + + "giUW98Y4/BPvRgJGnzG1pk698oiX4HblwK5ZC3rHSE31kf7FJOZxtfQgotEmGIQw1GEvLr" + + "dzCaJ6WthJKKElAQGs4Y1x0Bv2uYp9E8Ls8kpIhFEI5Bx5SO44nxIcG/9CL7GF2WM5FgPK" + + "DAwlBBBs4LHbqQgN++yCAeJcMzCSiwiCahAgyM5YZjFvZn4Kd4FJdOgo0VCizEwAAG69AO" + + "P5Ow9yOrOB6lw2FZniSWIhbJBAphYE+VI+yNEfMVxyZ/o8SjwMKIAgzUoR1MFfpH4EcTx8" + + "9HiVBgAeaUqTDoGMKhCgl/0TowcZFbCRcFFFokQJwx4FCM0PesrMTE6QISjwKLRBSWIWOh" + + "o4VCmBdjTL7IheIsxEioEMYVB9/FByYyxtQLSkiBBaFAFML4mWN235+OesaYzcKnwEIMEV" + + "CAcd2x4N9rQtMZGFPkXVJoIQYBAgrFUIOJghkcTtJ1ElBgAcZLSYVmSFK1qUHDqbqk0AIM" + + "AwQUYNChF4SDCU/HnZxNlxRY3qBhiPAUOsOC33Z3ZTWgxLOAg8AAhWJI2vhLONeEElqoEY" + + "JSKAdP7r15FIlgJBqhHVzUtiTLblBKOlqUVIsAx9LQ0aYkGTZlLCYtO3h2iobjKceG56XF" + + "PLwYm7pBKYupsRlE39rOk3GZLn51Owpj89VpmyH/lVCkv0LZYCoDjqXtdPoGqf5lQHlaGN" + + "Tx0L6BeegZJFnmV1djg6OitqPtRKSYZDrTMyrTOuC/BIJbGRhmXKfLktmkk8QwphfQRkA6" + + "jz1zIy+PAbsRbInQi8qgF6C4H9PvfQ2E8NXrR7z9tJvf+Z1/ANt+S+GBXoDpAAAAAElFTk" + + "SuQmCC" +} diff --git a/pkg/server/debug/pprofui/server.go b/pkg/server/debug/pprofui/server.go index b9f3d80a5dcc..9e52980923ed 100644 --- a/pkg/server/debug/pprofui/server.go +++ b/pkg/server/debug/pprofui/server.go @@ -44,12 +44,29 @@ type Server struct { storage Storage profileSem syncutil.Mutex profileTypes map[string]http.HandlerFunc + hook func(profile string, do func()) } -// NewServer creates a new Server backed by the supplied Storage. -func NewServer(storage Storage) *Server { +// NewServer creates a new Server backed by the supplied Storage and optionally +// a hook which is called when a new profile is created. The closure passed to +// the hook will carry out the work involved in creating the profile and must +// be called by the hook. The intention is that hook will be a method such as +// this: +// +// func hook(profile string, do func()) { +// if profile == "profile" { +// something.EnableProfilerLabels() +// defer something.DisableProfilerLabels() +// do() +// } +// } +func NewServer(storage Storage, hook func(profile string, do func())) *Server { + if hook == nil { + hook = func(_ string, do func()) { do() } + } s := &Server{ storage: storage, + hook: hook, } s.profileTypes = map[string]http.HandlerFunc{ @@ -195,7 +212,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { rw := &responseBridge{target: w} - fetchHandler(rw, req) + s.hook(profileName, func() { fetchHandler(rw, req) }) if rw.statusCode != http.StatusOK && rw.statusCode != 0 { return errors.Errorf("unexpected status: %d", rw.statusCode) @@ -215,8 +232,21 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + + // If this is a request issued by `go tool pprof`, just return the profile + // directly. This is convenient because it avoids having to expose the pprof + // endpoints separately, and also allows inserting hooks around CPU profiles + // in the future. + isGoPProf := strings.Contains(r.Header.Get("User-Agent"), "Go-http-client") origURL.Path = path.Join(origURL.Path, id, "flamegraph") - http.Redirect(w, r, origURL.String(), http.StatusTemporaryRedirect) + if !isGoPProf { + http.Redirect(w, r, origURL.String(), http.StatusTemporaryRedirect) + } else { + _ = s.storage.Get(id, func(r io.Reader) error { + _, err := io.Copy(w, r) + return err + }) + } } type fetcherFn func(_ string, _, _ time.Duration) (*profile.Profile, string, error) diff --git a/pkg/server/debug/pprofui/server_test.go b/pkg/server/debug/pprofui/server_test.go index 56a921b48481..4b0677e58db0 100644 --- a/pkg/server/debug/pprofui/server_test.go +++ b/pkg/server/debug/pprofui/server_test.go @@ -24,7 +24,7 @@ import ( func TestServer(t *testing.T) { storage := NewMemStorage(1, 0) - s := NewServer(storage) + s := NewServer(storage, nil) for i := 0; i < 3; i++ { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { diff --git a/pkg/server/debug/server.go b/pkg/server/debug/server.go index 984c115271c0..69968d63298f 100644 --- a/pkg/server/debug/server.go +++ b/pkg/server/debug/server.go @@ -23,11 +23,13 @@ import ( "net/http/pprof" "strings" + "github.com/cockroachdb/cockroach/pkg/server/debug/goroutineui" "github.com/cockroachdb/cockroach/pkg/server/debug/pprofui" "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/stop" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/pkg/errors" "github.com/rcrowley/go-metrics" "github.com/rcrowley/go-metrics/exp" @@ -118,9 +120,37 @@ func NewServer(st *cluster.Settings) *Server { } mux.HandleFunc("/debug/logspy", spy.handleDebugLogSpy) - ps := pprofui.NewServer(pprofui.NewMemStorage(1, 0)) + ps := pprofui.NewServer(pprofui.NewMemStorage(1, 0), func(profile string, do func()) { + tBegin := timeutil.Now() + + extra := "" + if profile == "profile" { + extra = " (enabling profiler labels)" + st.SetCPUProfiling(true) + defer st.SetCPUProfiling(false) + } + log.Infof(context.Background(), "pprofui: recording %s%s", profile, extra) + + do() + + log.Infof(context.Background(), "pprofui: recorded %s in %.2fs", profile, timeutil.Since(tBegin).Seconds()) + }) mux.Handle("/debug/pprof/ui/", http.StripPrefix("/debug/pprof/ui", ps)) + mux.HandleFunc("/debug/pprof/goroutineui/", func(w http.ResponseWriter, req *http.Request) { + dump := goroutineui.NewDump(timeutil.Now()) + + _ = req.ParseForm() + switch req.Form.Get("sort") { + case "count": + dump.SortCountDesc() + case "wait": + dump.SortWaitDesc() + default: + } + _ = dump.HTML(w) + }) + return &Server{ st: st, mux: mux, diff --git a/pkg/settings/cluster/settings.go b/pkg/settings/cluster/settings.go index a30b50f33abc..3aaafea5dc3a 100644 --- a/pkg/settings/cluster/settings.go +++ b/pkg/settings/cluster/settings.go @@ -71,6 +71,27 @@ type Settings struct { ExternalIODir string Initialized bool + + // Set to 1 if a profile is active (if the profile is being grabbed through + // the `pprofui` server as opposed to the raw endpoint). + cpuProfiling int32 // atomic +} + +// IsCPUProfiling returns true if a pprofui CPU profile is being recorded. This can +// be used by moving parts across the system to add profiler labels which are +// too expensive to be enabled at all times. +func (s *Settings) IsCPUProfiling() bool { + return atomic.LoadInt32(&s.cpuProfiling) == 1 +} + +// SetCPUProfiling is called from the pprofui to inform the system that a CPU +// profile is being recorded. +func (s *Settings) SetCPUProfiling(to bool) { + i := int32(0) + if to { + i = 1 + } + atomic.StoreInt32(&s.cpuProfiling, i) } // NoSettings is used when a func requires a Settings but none is available diff --git a/pkg/sql/conn_executor_exec.go b/pkg/sql/conn_executor_exec.go index f3828b35a51a..b4a17206f67b 100644 --- a/pkg/sql/conn_executor_exec.go +++ b/pkg/sql/conn_executor_exec.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Cockroach Authors. +// Copyright 2018 The Cockroach Authors.D // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package sql import ( "context" "fmt" + "runtime/pprof" "strings" "time" @@ -90,7 +91,17 @@ func (ex *connExecutor) execStmt( case stateNoTxn: ev, payload = ex.execStmtInNoTxnState(ctx, stmt) case stateOpen: - ev, payload, err = ex.execStmtInOpenState(ctx, stmt, pinfo, res) + if ex.server.cfg.Settings.IsCPUProfiling() { + labels := pprof.Labels( + "stmt.tag", stmt.AST.StatementTag(), + "stmt.anonymized", stmt.AnonymizedStr, + ) + pprof.Do(ctx, labels, func(ctx context.Context) { + ev, payload, err = ex.execStmtInOpenState(ctx, stmt, pinfo, res) + }) + } else { + ev, payload, err = ex.execStmtInOpenState(ctx, stmt, pinfo, res) + } switch ev.(type) { case eventNonRetriableErr: ex.recordFailure() diff --git a/pkg/ui/src/views/reports/containers/debug/index.tsx b/pkg/ui/src/views/reports/containers/debug/index.tsx index 8a62309a4e94..8299c40a4607 100644 --- a/pkg/ui/src/views/reports/containers/debug/index.tsx +++ b/pkg/ui/src/views/reports/containers/debug/index.tsx @@ -218,7 +218,7 @@ export default function Debug() { - + @@ -226,14 +226,13 @@ export default function Debug() { - - - - - - - - + + + + + + + diff --git a/vendor b/vendor index d8a496f8197d..1c5942a982b5 160000 --- a/vendor +++ b/vendor @@ -1 +1 @@ -Subproject commit d8a496f8197df890ec9aa21097af56e97fdbb37b +Subproject commit 1c5942a982b5e1b0b399a6fb4ae183d7651179ab