Random crashes

[matt-richardson]

Running into random crashes:

$ ./tools/azure-cli/azure-extensions-cli_darwin_amd64 replication-status --subscription-id XXXXXXXXXX --subscription-cert azure-management-cert.pem --namespace "XXXXXXX" --name "XXXXXXX" --version 2.0.1

failed MSpanList_Insert 0x8a8000 0x1c33b966f2cd 0x0 0x0
fatal error: MSpanList_Insert

runtime stack:
runtime.throw(0x5ac9d0, 0x10)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/panic.go:530 +0x90 fp=0x7fff5fbff040 sp=0x7fff5fbff028
runtime.(*mSpanList).insert(0x7e8948, 0x8a8000)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:933 +0x293 fp=0x7fff5fbff070 sp=0x7fff5fbff040
runtime.(*mheap).freeSpanLocked(0x7e8140, 0x8a8000, 0x100, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:809 +0x4be fp=0x7fff5fbff0d8 sp=0x7fff5fbff070
runtime.(*mheap).grow(0x7e8140, 0x8, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:675 +0x2a0 fp=0x7fff5fbff130 sp=0x7fff5fbff0d8
runtime.(*mheap).allocSpanLocked(0x7e8140, 0x1, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:553 +0x4e3 fp=0x7fff5fbff188 sp=0x7fff5fbff130
runtime.(*mheap).alloc_m(0x7e8140, 0x1, 0x15, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:437 +0x119 fp=0x7fff5fbff1b8 sp=0x7fff5fbff188
runtime.(*mheap).alloc.func1()
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:502 +0x41 fp=0x7fff5fbff1e8 sp=0x7fff5fbff1b8
runtime.systemstack(0x7fff5fbff208)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/asm_amd64.s:307 +0xab fp=0x7fff5fbff1f0 sp=0x7fff5fbff1e8
runtime.(*mheap).alloc(0x7e8140, 0x1, 0x10000000015, 0x1eb0f)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mheap.go:503 +0x63 fp=0x7fff5fbff238 sp=0x7fff5fbff1f0
runtime.(*mcentral).grow(0x7e9d40, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mcentral.go:209 +0x93 fp=0x7fff5fbff2a0 sp=0x7fff5fbff238
runtime.(*mcentral).cacheSpan(0x7e9d40, 0x7e2858)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mcentral.go:89 +0x47d fp=0x7fff5fbff2e0 sp=0x7fff5fbff2a0
runtime.(*mcache).refill(0x8a4000, 0x15, 0x7fff5fbff348)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/mcache.go:119 +0xcc fp=0x7fff5fbff318 sp=0x7fff5fbff2e0
runtime.mallocgc.func2()
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/malloc.go:642 +0x2b fp=0x7fff5fbff338 sp=0x7fff5fbff318
runtime.systemstack(0x7fff5fbff3d8)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/asm_amd64.s:307 +0xab fp=0x7fff5fbff340 sp=0x7fff5fbff338
runtime.mallocgc(0x180, 0x554a80, 0x0, 0x800000000)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/malloc.go:643 +0x869 fp=0x7fff5fbff418 sp=0x7fff5fbff340
runtime.newobject(0x554a80, 0x7e2e10)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/malloc.go:781 +0x42 fp=0x7fff5fbff440 sp=0x7fff5fbff418
runtime.malg(0x8000, 0x7e31c0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/proc.go:2634 +0x27 fp=0x7fff5fbff478 sp=0x7fff5fbff440
runtime.mpreinit(0x7e39c0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/os1_darwin.go:140 +0x1f fp=0x7fff5fbff490 sp=0x7fff5fbff478
runtime.mcommoninit(0x7e39c0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/proc.go:494 +0x105 fp=0x7fff5fbff4d8 sp=0x7fff5fbff490
runtime.schedinit()
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/proc.go:434 +0x79 fp=0x7fff5fbff520 sp=0x7fff5fbff4d8
runtime.rt0_go(0x7fff5fbff550, 0xc, 0x7fff5fbff550, 0x0, 0xc, 0x7fff5fbff768, 0x7fff5fbff79c, 0x7fff5fbff7af, 0x7fff5fbff7c1, 0x7fff5fbff7e6, ...)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/asm_amd64.s:138 +0x132 fp=0x7fff5fbff528 sp=0x7fff5fbff520

If I re-run it, it succeeds.

[boumenot]

Please see golang/go#17182 for details, and a fix.

[matt-richardson]

Well, I'm not trying to build a go distribution, so I'm not sure how thats relevant.

But on the assumption you were talking about adding an environment variable called GOROOT_BOOTSTRAP, nup, that doesn't work either.

The relevant snippet from export:

declare -x GOPATH="/Users/mattr/gopath"
declare -x GOROOT="/usr/local/go"
declare -x GOROOT_BOOTSTRAP="/usr/local/go"

gives me:

panic: sync: inconsistent mutex state

goroutine 30 [running]:
panic(0x3f6160, 0xc820450dc0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/panic.go:464 +0x3e6
sync.(*Mutex).Lock(0xc8200740c0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/sync/mutex.go:75 +0x16d
math/rand.(*lockedSource).Int63(0xc8200740c0, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:258 +0x28
math/rand.(*Rand).Int63(0xc82007a350, 0x6c175)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:46 +0x39
math/rand.(*Rand).Int(0xc82007a350, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:56 +0x21
math/rand.Int(0xc82009a060)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:198 +0x23
net.exchange(0xc820450c30, 0xa, 0xc8203ab080, 0x1c, 0xc82045001c, 0x12a05f200, 0x0, 0x0, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:151 +0x405
net.tryOneName(0xc820153600, 0xc8203ab080, 0x1c, 0x1c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:181 +0x2f3
net.goLookupIPOrder.func1(0xc820153600, 0xc820450a50, 0xc8200c0720, 0x1c)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:462 +0x59
created by net.goLookupIPOrder
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:464 +0x3d0
panic: sync: inconsistent mutex state

goroutine 29 [running]:
panic(0x3f6160, 0xc820424000)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/runtime/panic.go:464 +0x3e6
sync.(*Mutex).Lock(0xc8200740c0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/sync/mutex.go:75 +0x16d
math/rand.(*lockedSource).Int63(0xc8200740c0, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:258 +0x28
math/rand.(*Rand).Int63(0xc82007a350, 0x6c175)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:46 +0x39
math/rand.(*Rand).Int(0xc82007a350, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:56 +0x21
math/rand.Int(0xc82015e000)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/math/rand/rand.go:198 +0x23
net.exchange(0xc82041b410, 0xa, 0xc8203ab080, 0x1c, 0xc820410001, 0x12a05f200, 0x0, 0x0, 0x0)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:151 +0x405
net.tryOneName(0xc820153600, 0xc8203ab080, 0x1c, 0xc820290001, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:181 +0x2f3
net.goLookupIPOrder.func1(0xc820153600, 0xc820450a50, 0xc8200c0720, 0x1)
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:462 +0x59
created by net.goLookupIPOrder
	/home/travis/.gimme/versions/go1.6.linux.amd64/src/net/dnsclient_unix.go:464 +0x3d0

[boumenot]

Keep reading. This issue was caused by an update in Mac Sierra.

[matt-richardson]

Yep, I've kept reading. I've read it all, forwards, backwards and even sideways. The only fix I can see in that thread is to set the GO_BOOTSTRAP environment variable, which as you can see, I've done. But it makes no difference.

[boumenot]

I don't know then. I cannot reproduce the issue on the OSs that I have access too (Windows and Linux). If you're able to fix it please respond back to the thread.

My only recommendation is to try the latest 1.7.x Go compiler, or the just released 1.8 Go compiler to see if that addresses the issue.

The PowerShell cmdlets for publishing are the best alternative for azure-extensions-cli.