Add setup and teardown for old cluster tests (#26713)

The tests recently got refactored to add a few tests depending on the
version the tests are run against. But since the tests are not run in
order, the writes that were in one test caused failures in others if run
out of order. The indices that this test use are also important as they
are validated post upgrade, so we cannot simply add a setup/teardown to
clean up.

The best way to keep these tests going was to duplicate the test so that
it can be run fully depending on the version, and the downstream tests
that depend on these upgrade indices do not have to be refactored.

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml

@@ -1,5 +1,10 @@
 ---
-"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade":
+"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade 5.6.0 and beyond":
+  - skip:
+      version: " - 5.5.99"
+      reason: stored search template API is deprecated in 5.6.0
+      features: "warnings"
+
   - do:
       indices.create:
         index: test_index
@@ -82,13 +87,9 @@
           }
   - match: { "acknowledged": true }
 
----
-"Put a template before 5.6.0":
-  - skip:
-      version: "5.6.0 - "
-      reason: stored search template API is deprecated in 5.6.0
-
   - do:
+      warnings:
+        - "The stored search template API is deprecated. Use stored scripts instead."
       put_template:
         id: test_search_template
         body:
@@ -104,16 +105,96 @@
           params:
             f1: v5_old
   - match: { hits.total: 1 }
+
 ---
-"Put a template 5.6.0 and beyond":
+"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade before 5.6.0":
   - skip:
-      version: " - 5.5.99"
+      version: "5.6.0 - "
       reason: stored search template API is deprecated in 5.6.0
-      features: "warnings"
 
   - do:
-      warnings:
-        - "The stored search template API is deprecated. Use stored scripts instead."
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
+  - do:
+      indices.create:
+        index: index_with_replicas # dummy index to ensure we can recover indices with replicas just fine
+        body:
+          # if the node with the replica is the first to be restarted, then delayed
+          # allocation will kick in, and the cluster health won't return to GREEN
+          # before timing out
+          index.unassigned.node_left.delayed_timeout: "100ms"
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v1_old", "f2": 0}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v2_old", "f2": 1}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v3_old", "f2": 2}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v4_old", "f2": 3}'
+          - '{"index": {"_index": "test_index", "_type": "doc"}}'
+          - '{"f1": "v5_old", "f2": 4}'
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+          - '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
+          - '{"f1": "d_old"}'
+
+  - do:
+      indices.flush:
+        index: test_index,index_with_replicas
+
+  - do:
+      search:
+        index: test_index
+
+  - match: { hits.total: 5 }
+
+  - do:
+      search:
+        index: index_with_replicas
+
+  - match: { hits.total: 5 }
+
+  - do:
+      snapshot.create_repository:
+        repository: my_repo
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test"
+  - match: { "acknowledged": true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+            ]
+          }
+  - match: { "acknowledged": true }
+
+  - do:
       put_template:
         id: test_search_template
         body:
@@ -129,3 +210,4 @@
           params:
             f1: v5_old
   - match: { hits.total: 1 }
+