Enable smoke-test-watcher with logging (#39169)

SmokeTestWatcherTestSuiteIT.testMonitorClusterHealth has failed a few
times with various causes (not all of which we have logs for).

This change enables the test again.

1. The fix from #39092 should resolve any issues in assertWatchCount
2. In at least 1 case, getWatchHistoryEntry failed due to a
   ResponseException, which is not caught by assertBusy.
   This commit catches those and calls "fail" so that assertBusy will
   sleep and retry
3. Additional logging has been included to help diagnose any other
   failures causes.

x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java

@@ -7,6 +7,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -107,7 +108,6 @@ protected Settings restAdminSettings() {
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32299")
     public void testMonitorClusterHealth() throws Exception {
         String watchId = "cluster_health_watch";
 
@@ -168,6 +168,7 @@ private void indexWatch(String watchId, XContentBuilder builder) throws Exceptio
         Response response = client().performRequest(request);
         Map<String, Object> responseMap = entityAsMap(response);
         assertThat(responseMap, hasEntry("_id", watchId));
+        logger.info("Successfully indexed watch with id [{}]", watchId);
     }
 
     private void deleteWatch(String watchId) throws IOException {
@@ -181,7 +182,14 @@ private void deleteWatch(String watchId) throws IOException {
     private ObjectPath getWatchHistoryEntry(String watchId) throws Exception {
         final AtomicReference<ObjectPath> objectPathReference = new AtomicReference<>();
         assertBusy(() -> {
-            client().performRequest(new Request("POST", "/.watcher-history-*/_refresh"));
+            logger.info("Refreshing watcher history");
+            try {
+                client().performRequest(new Request("POST", "/.watcher-history-*/_refresh"));
+            } catch (ResponseException e) {
+                final String err = "Failed to perform refresh of watcher history - " + e;
+                logger.info(err);
+                fail(err);
+            }
 
             try (XContentBuilder builder = jsonBuilder()) {
                 builder.startObject();
@@ -193,16 +201,23 @@ private ObjectPath getWatchHistoryEntry(String watchId) throws Exception {
                     .endObject().endArray();
                 builder.endObject();
 
+                logger.info("Searching watcher history");
                 Request searchRequest = new Request("POST", "/.watcher-history-*/_search");
                 searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
                 searchRequest.setJsonEntity(Strings.toString(builder));
                 Response response = client().performRequest(searchRequest);
                 ObjectPath objectPath = ObjectPath.createFromResponse(response);
                 int totalHits = objectPath.evaluate("hits.total");
+                logger.info("Found [{}] hits in watcher history", totalHits);
                 assertThat(totalHits, is(greaterThanOrEqualTo(1)));
-                String watchid = objectPath.evaluate("hits.hits.0._source.watch_id");
-                assertThat(watchid, is(watchId));
+                String foundWatchId = objectPath.evaluate("hits.hits.0._source.watch_id");
+                logger.info("Watch hit 0 has id [{}] (expecting [{}])", foundWatchId,  watchId);
+                assertThat("watch_id for hit 0 in watcher history", foundWatchId, is(watchId));
                 objectPathReference.set(objectPath);
+            } catch (ResponseException e) {
+                final String err = "Failed to perform search of watcher history - " + e;
+                logger.info(err);
+                fail(err);
             }
         });
         return objectPathReference.get();
@@ -212,6 +227,7 @@ private void assertWatchCount(int expectedWatches) throws IOException {
         Response watcherStatsResponse = adminClient().performRequest(new Request("GET", "/_watcher/stats"));
         ObjectPath objectPath = ObjectPath.createFromResponse(watcherStatsResponse);
         int watchCount = objectPath.evaluate("stats.0.watch_count");
-        assertThat(watchCount, is(expectedWatches));
+        assertThat("Watch count (from _watcher/stats)", watchCount, is(expectedWatches));
+        logger.info("Watch count is [{}]", watchCount);
     }
 }