failsuite: add package to create corpus for failed test output

This package serves to make it easy to improve test coverage for
go-test-teamcity in the future as problems arise, by opening a PR that
edits the CI scripts to include the COCKROACH_FAILSUITE environment
variable, triggering various interesting test failures and data races in
these newly introduced packages. The created artifacts can then be
incorporated into the test suite for go-test-teamcity (as was already
done in cockroachdb/go-test-teamcity#9).

Release note: None

pkg/testutils/failsuite/failsuitepanic/main.go

@@ -0,0 +1,16 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License. See the AUTHORS file
+// for names of contributors.
+
+package failsuitepanic

pkg/testutils/failsuite/failsuitepanic/main_test.go

@@ -0,0 +1,38 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License. See the AUTHORS file
+// for names of contributors.
+
+package failsuitepanic
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
+)
+
+func TestMain(m *testing.M) {
+	if !envutil.EnvOrDefaultBool("COCKROACH_FAILSUITE", false) {
+		return
+	}
+	os.Exit(m.Run())
+}
+
+func TestZPanics(t *testing.T) {
+	fmt.Println("first output something")
+	t.Log("then log something")
+	t.Error("an error, why not")
+	panic("boom")
+}

pkg/testutils/failsuite/main.go

@@ -0,0 +1,20 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License. See the AUTHORS file
+// for names of contributors.
+
+// Package failsuite contains no real code. It exists to easily generate
+// interesting test failures to jog our CI pipeline. "Activate" the test
+// failures by setting the COCKROACH_FAILSUITE env variable while running the
+// tests for this package (or its subpackages).
+package failsuite

pkg/testutils/failsuite/main_test.go

@@ -0,0 +1,88 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License. See the AUTHORS file
+// for names of contributors.
+
+package failsuite
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
+)
+
+func TestMain(m *testing.M) {
+	if !envutil.EnvOrDefaultBool("COCKROACH_FAILSUITE", false) {
+		return
+	}
+	os.Exit(m.Run())
+}
+
+func TestThatPasses(t *testing.T) {
+	fmt.Println("output of", t.Name())
+}
+
+func TestThatFails(t *testing.T) {
+	t.Error("an error occurs")
+	t.Fatal("then the test fatals")
+}
+
+func TestThatGetsSkipped(t *testing.T) {
+	t.Log("logging something")
+	t.Skip("skipped")
+}
+
+func TestParallelOneWithDataRaceWithoutFailures(t *testing.T) {
+	testParallelImpl(t, false)
+}
+func TestParallelWithDataRaceAndFailures(t *testing.T) {
+	testParallelImpl(t, true)
+}
+
+func testParallelImpl(t *testing.T, withFailures bool) {
+	var dataracingGlobal bool
+	sleep := func() {
+		time.Sleep(time.Duration(rand.Intn(1E9)))
+	}
+	// Parallel tests are interesting because they interleave.
+	// Make the parent test parallel as well, to interleave it
+	// with another copy of itself.
+	fmt.Println("output before calling t.Parallel()")
+	t.Parallel()
+	for i := 0; i < 10; i++ {
+		i := i
+		fails := withFailures && (i%3) == 0
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			t.Parallel()
+			// Add a data race. Go pretty much catches this every time.
+			dataracingGlobal = !dataracingGlobal
+			sleep()
+			// NB: access to stdout isn't actually handled properly with parallel tests.
+			// In effect, this output will just be put into whatever test happens to
+			// announce that it's running most recently. So we can't do anything useful
+			// here and avoid doing this in the first place. fmt.Printf("subtest %d
+			// outputs this\n", i)
+			if fails {
+				t.Error("subtest failed")
+			} else {
+				t.Log("something gets logged")
+			}
+		})
+	}
+
+}