From f9530793a8b10448d3814a65619a8d382e3dbfe0 Mon Sep 17 00:00:00 2001 From: The Magician Date: Mon, 22 Mar 2021 14:56:30 -0700 Subject: [PATCH] Add operation retry for exceeded quota group OperationReadGroup (#4599) (#3077) Signed-off-by: Modular Magician --- .changelog/4599.txt | 3 +++ google-beta/common_operation.go | 2 +- google-beta/error_retry_predicates.go | 11 +++++++++++ google-beta/error_retry_predicates_test.go | 11 +++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .changelog/4599.txt diff --git a/.changelog/4599.txt b/.changelog/4599.txt new file mode 100644 index 0000000000..40282d5321 --- /dev/null +++ b/.changelog/4599.txt @@ -0,0 +1,3 @@ +```release-note:bug +compute: fixed an issue where exceeding the operation rate limit would fail without retrying +``` diff --git a/google-beta/common_operation.go b/google-beta/common_operation.go index bbd724b8c2..624c37e83f 100644 --- a/google-beta/common_operation.go +++ b/google-beta/common_operation.go @@ -111,7 +111,7 @@ func CommonRefreshFunc(w Waiter) resource.StateRefreshFunc { op, err := w.QueryOp() if err != nil { // Retry 404 when getting operation (not resource state) - if isRetryableError(err, isNotFoundRetryableError("GET operation")) { + if isRetryableError(err, isNotFoundRetryableError("GET operation"), isOperationReadQuotaError) { log.Printf("[DEBUG] Dismissed retryable error on GET operation %q: %s", w.OpName(), err) return nil, "done: false", nil } diff --git a/google-beta/error_retry_predicates.go b/google-beta/error_retry_predicates.go index 3d2b188c93..438bfd21e9 100644 --- a/google-beta/error_retry_predicates.go +++ b/google-beta/error_retry_predicates.go @@ -216,6 +216,17 @@ func isBigqueryIAMQuotaError(err error) (bool, string) { return false, "" } +// Retry if operation returns a 403 with the message for +// exceeding the quota limit for 'OperationReadGroup' +func isOperationReadQuotaError(err error) (bool, string) { + if gerr, ok := err.(*googleapi.Error); ok { + if gerr.Code == 403 && strings.Contains(gerr.Body, "Quota exceeded for quota group 'OperationReadGroup'") { + return true, "Waiting for quota to refresh" + } + } + return false, "" +} + // Retry if Monitoring operation returns a 409 with a specific message for // concurrent operations. func isMonitoringConcurrentEditError(err error) (bool, string) { diff --git a/google-beta/error_retry_predicates_test.go b/google-beta/error_retry_predicates_test.go index 5806aa0d09..0fbf558f0a 100644 --- a/google-beta/error_retry_predicates_test.go +++ b/google-beta/error_retry_predicates_test.go @@ -78,3 +78,14 @@ func TestIsCommonRetryableErrorCode_otherError(t *testing.T) { t.Errorf("Error incorrectly detected as retryable") } } + +func TestIsOperationReadQuotaError_quotaExceeded(t *testing.T) { + err := googleapi.Error{ + Code: 403, + Body: "Quota exceeded for quota group 'OperationReadGroup' and limit 'Operation read requests per 100 seconds' of service 'compute.googleapis.com' for consumer 'project_number:11111111'.", + } + isRetryable, _ := isOperationReadQuotaError(&err) + if !isRetryable { + t.Errorf("Error not detected as retryable") + } +}