testing: Add SkipTestOnError to test steps

helper/acctest/skip_on_error.go

@@ -0,0 +1,32 @@
+package acctest
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+)
+
+// SkipOnErrorContains helps skip tests based on a portion of an error message
+// which is contained in a complete error message.
+func SkipOnErrorContains(s string) resource.SkipOnErrorFunc {
+	return func(err error) bool {
+		if err == nil {
+			return false
+		}
+
+		return strings.Contains(err.Error(), s)
+	}
+}
+
+// SkipOnErrorRegexp helps skip tests based on a regexp that matches an error
+// message.
+func SkipOnErrorRegexp(re *regexp.Regexp) resource.SkipOnErrorFunc {
+	return func(err error) bool {
+		if err == nil {
+			return false
+		}
+
+		return re.MatchString(err.Error())
+	}
+}

helper/acctest/skip_on_error_test.go

@@ -0,0 +1,67 @@
+package acctest
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+)
+
+func TestSkipOnErrorContains(t *testing.T) {
+	testCases := []struct {
+		s        string
+		err      error
+		expected bool
+	}{
+		{
+			s:        "Operations related to PublicDNS are not supported in this aws partition",
+			err:      fmt.Errorf("Error: error creating Route53 Hosted Zone: InvalidInput: Operations related to PublicDNS are not supported in this aws partition.\nstatus code: 400, request id: 395ef7ef-be89-48a1-98ec-0bcb0a517825"),
+			expected: true,
+		},
+		{
+			s:        "Operations related to PublicDNS are not supported in this aws partition",
+			err:      fmt.Errorf("Error: unrelated error: Smog Patrol. Had your emissions checked?\nstatus code: 400, request id: 395ef7ef-be89-48a1-98ec-0bcb0a517825"),
+			expected: false,
+		},
+		{
+			s:        "Operations related to PublicDNS are not supported in this aws partition",
+			expected: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		f := SkipOnErrorContains(tc.s)
+		if f(tc.err) != tc.expected {
+			t.Fatalf("expected test case %d to be %v but was %v (error portion %s, error message %s)", i, tc.expected, f(tc.err), tc.s, tc.err)
+		}
+	}
+}
+
+func TestSkipOnErrorRegexp(t *testing.T) {
+	testCases := []struct {
+		re       *regexp.Regexp
+		err      error
+		expected bool
+	}{
+		{
+			re:       regexp.MustCompile(`Operations related to [a-zA-Z]+ are not supported in this aws partition`),
+			err:      fmt.Errorf("Error: error creating Route53 Hosted Zone: InvalidInput: Operations related to PublicDNS are not supported in this aws partition.\nstatus code: 400, request id: 395ef7ef-be89-48a1-98ec-0bcb0a517825"),
+			expected: true,
+		},
+		{
+			re:       regexp.MustCompile(`Operations related to [a-zA-Z]+ are not supported in this aws partition`),
+			err:      fmt.Errorf("Error: unrelated error, You on a scavenger hunt, or did I forget to pay my dinner check?\nstatus code: 400, request id: 395ef7ef-be89-48a1-98ec-0bcb0a517825"),
+			expected: false,
+		},
+		{
+			re:       regexp.MustCompile(`Operations related to [a-zA-Z]+ are not supported in this aws partition`),
+			expected: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		f := SkipOnErrorRegexp(tc.re)
+		if f(tc.err) != tc.expected {
+			t.Fatalf("expected test case %d to be %v but was %v (regexp %s, error message %s)", i, tc.expected, f(tc.err), tc.re, tc.err)
+		}
+	}
+}

helper/resource/testing.go

@@ -258,6 +258,10 @@ type ImportStateCheckFunc func([]*terraform.InstanceState) error
 // generation for ImportState tests.
 type ImportStateIdFunc func(*terraform.State) (string, error)
 
+// SkipOnErrorFunc is a function to determine whether an error should cause a
+// test to be skipped.
+type SkipOnErrorFunc func(error) bool
+
 // TestCase is a single acceptance test case used to test the apply/destroy
 // lifecycle of a resource in a specific configuration.
 //
@@ -323,6 +327,11 @@ type TestCase struct {
 	// to allow the tester to test that the resource is truly gone.
 	CheckDestroy TestCheckFunc
 
+	// SkipOnError allows the construction of tests that we want to skip if
+	// they fail with particular errors. The error is passed to a function that
+	// determines whether to skip the test.
+	SkipOnError SkipOnErrorFunc
+
 	// Steps are the apply sequences done within the context of the
 	// same state. Each step can have its own check to verify correctness.
 	Steps []TestStep

helper/resource/testing_new.go

@@ -113,6 +113,8 @@ func runNewTest(t testing.T, c TestCase, helper *plugintest.Helper) {
 				if !step.ExpectError.MatchString(err.Error()) {
 					t.Fatalf("Step %d/%d error running import, expected an error with pattern (%s), no match on: %s", i+1, len(c.Steps), step.ExpectError.String(), err)
 				}
+			} else if err != nil && c.SkipOnError != nil && c.SkipOnError(err) {
+				t.Skipf("[WARN] Skipping test, step %d/%d error passed SkipOnError: %s", i+1, len(c.Steps), err)
 			} else {
 				if err != nil {
 					t.Fatalf("Step %d/%d error running import: %s", i+1, len(c.Steps), err)
@@ -130,6 +132,8 @@ func runNewTest(t testing.T, c TestCase, helper *plugintest.Helper) {
 				if !step.ExpectError.MatchString(err.Error()) {
 					t.Fatalf("Step %d/%d, expected an error with pattern, no match on: %s", i+1, len(c.Steps), err)
 				}
+			} else if err != nil && c.SkipOnError != nil && c.SkipOnError(err) {
+				t.Skipf("[WARN] Skipping test, step %d/%d error passed SkipOnError: %s", i+1, len(c.Steps), err)
 			} else {
 				if err != nil {
 					t.Fatalf("Step %d/%d error: %s", i+1, len(c.Steps), err)