Reduce logging of HTTP status codes (#7993)

# Description

This change reduces the amount of logging we do for an HTTP response in
non-error cases by moving those logs to debug instead of info.

In particular we generate a TON of logs for for asynchronous operations
where the API is being polled at 1 second intervals by the CLI. Since
it's hitting the operation status endpoint, the status code is always
200 and not especially meaningful.

This change simplifies the logging by applying the following rules:

- Non-2XX status codes continue to be logged at info level.
- 2XX status code are now logged at debug level (off by default).

My motivation for this change is to make the logs more readable by
making the more significant events visible. Right now it's hard to find
significant events in our logs because there is a flood of "responded
with 200" type events due to the nature of async operations. When an
async operation is taking a long time to process, the relevant
information might be 100s of lines back in the history.

I've been testing this out locally for a few days of work and I feel
like it's really improved my ability to diagnose issues.

## Type of change

- This pull request is a minor refactor, code cleanup, test improvement,
or other maintenance task and doesn't change the functionality of Radius
(issue link optional).

## Contributor checklist
Please verify that the PR meets the following requirements, where
applicable:

- [ ] An overview of proposed schema changes is included in a linked
GitHub issue.
- [ ] A design document PR is created in the [design-notes
repository](https://github.com/radius-project/design-notes/), if new
APIs are being introduced.
- [ ] If applicable, design document has been reviewed and approved by
Radius maintainers/approvers.
- [ ] A PR for the [samples
repository](https://github.com/radius-project/samples) is created, if
existing samples are affected by the changes in this PR.
- [ ] A PR for the [documentation
repository](https://github.com/radius-project/docs) is created, if the
changes in this PR affect the documentation or any user facing updates
are made.
- [ ] A PR for the [recipes
repository](https://github.com/radius-project/recipes) is created, if
existing recipes are affected by the changes in this PR.

Signed-off-by: Ryan Nowak <[email protected]>

pkg/armrpc/rest/results.go

@@ -72,7 +72,7 @@ func NewOKResponseWithHeaders(body any, headers map[string]string) Response {
 // code of 200. If an error occurs while marshaling the body or writing the response, an error is returned.
 func (r *OKResponse) Apply(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
 	logger := ucplog.FromContextOrDiscard(ctx)
-	logger.Info(fmt.Sprintf("responding with status code: %d", http.StatusOK), logging.LogHTTPStatusCode, http.StatusOK)
+	logger.V(ucplog.LevelDebug).Info(fmt.Sprintf("responding with status code: %d", http.StatusOK), logging.LogHTTPStatusCode, http.StatusOK)
 
 	bytes, err := json.MarshalIndent(r.Body, "", "  ")
 	if err != nil {
@@ -114,7 +114,7 @@ func NewCreatedResponse(body any) Response {
 // 201 Created response code and returns an error if any of these steps fail.
 func (r *CreatedResponse) Apply(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
 	logger := ucplog.FromContextOrDiscard(ctx)
-	logger.Info(fmt.Sprintf("responding with status code: %d", http.StatusCreated), logging.LogHTTPStatusCode, http.StatusCreated)
+	logger.V(ucplog.LevelDebug).Info(fmt.Sprintf("responding with status code: %d", http.StatusCreated), logging.LogHTTPStatusCode, http.StatusCreated)
 
 	bytes, err := json.MarshalIndent(r.Body, "", "  ")
 	if err != nil {
@@ -148,7 +148,7 @@ func NewCreatedAsyncResponse(body any, location string, scheme string) Response
 // Apply renders Created HTTP Response into http.ResponseWriter with Location header for asynchronous operation and returns an error if it fails.
 func (r *CreatedAsyncResponse) Apply(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
 	logger := ucplog.FromContextOrDiscard(ctx)
-	logger.Info(fmt.Sprintf("responding with status code: %d", http.StatusCreated), logging.LogHTTPStatusCode, http.StatusCreated)
+	logger.V(ucplog.LevelDebug).Info(fmt.Sprintf("responding with status code: %d", http.StatusCreated), logging.LogHTTPStatusCode, http.StatusCreated)
 
 	bytes, err := json.MarshalIndent(r.Body, "", "  ")
 	if err != nil {
@@ -199,7 +199,7 @@ func NewAcceptedAsyncResponse(body any, location string, scheme string) Response
 // Apply renders Accepted HTTP Response into http.ResponseWriter with Location header for asynchronous operation and returns an error if it fails.
 func (r *AcceptedAsyncResponse) Apply(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
 	logger := ucplog.FromContextOrDiscard(ctx)
-	logger.Info(fmt.Sprintf("responding with status code: %d", http.StatusAccepted), logging.LogHTTPStatusCode, http.StatusAccepted)
+	logger.V(ucplog.LevelDebug).Info(fmt.Sprintf("responding with status code: %d", http.StatusAccepted), logging.LogHTTPStatusCode, http.StatusAccepted)
 
 	bytes, err := json.MarshalIndent(r.Body, "", "  ")
 	if err != nil {
@@ -729,7 +729,7 @@ func NewAsyncOperationResultResponse(headers map[string]string) Response {
 // Apply sets the response headers and status code to http.StatusAccepted and returns nil.
 func (r *AsyncOperationResultResponse) Apply(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
 	logger := ucplog.FromContextOrDiscard(ctx)
-	logger.Info(fmt.Sprintf("responding with status code: %d", http.StatusAccepted), logging.LogHTTPStatusCode, http.StatusAccepted)
+	logger.V(ucplog.LevelDebug).Info(fmt.Sprintf("responding with status code: %d", http.StatusAccepted), logging.LogHTTPStatusCode, http.StatusAccepted)
 
 	w.Header().Add("Content-Type", "application/json")