diff --git a/src/grpc/infer_handler.h b/src/grpc/infer_handler.h
index 0e1091feb8..a976ccff02 100644
--- a/src/grpc/infer_handler.h
+++ b/src/grpc/infer_handler.h
@@ -666,7 +666,11 @@ class InferHandlerState {
 
     bool IsCancelled()
     {
-      return received_notification_ ? ctx_->IsCancelled() : false;
+      std::lock_guard<std::recursive_mutex> lock(mu_);
+      return received_notification_
+                 ? (ctx_->IsCancelled() ||
+                    gRPCErrorTracker_->CheckAndUpdateGRPCError())
+                 : false;
     }
 
     // Increments the ongoing request counter