diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java
index 37a52e97e2fe3..2a99bd9381229 100644
--- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java
+++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java
@@ -9,6 +9,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
+import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
@@ -45,6 +46,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
+@LuceneTestCase.AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/44814")
 public class WatchBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     private final StringEntity entity = new StringEntity(watchBuilder()
@@ -169,12 +171,6 @@ protected Settings restClientSettings() {
                 .build();
     }
 
-    public void testDummy() {
-        // Dummy test so the suite doesn't fail with a no executed tests error.
-        // Remove when the awaits fixes below (#44814) are resolved
-        assertTrue(true);
-    }
-
     @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/44814")
     public void testWatcherStats() throws Exception {
         executeAgainstAllNodes(client ->