Add ignore_missing_component_templates config option

docs/changelog/92436.yaml

@@ -0,0 +1,7 @@
+pr: 92436
+summary: Add `ignore_missing_component_templates` config option
+# The bot was creating `area: "Data streams, Indices APIs"` but this seems to be invalid
+area: Indices APIs
+type: enhancement
+issues:
+  - 92426

docs/reference/indices/ignore-missing-component-templates.asciidoc

@@ -0,0 +1,95 @@
+[[ignore_missing_component_templates]]
+== Config ignore_missing_component_templates
+
+The configuration option `ignore_missing_component_templates` can be used when an index template references a component template that might not exist. Every time a data stream is created based on the index template, the existence of the component template will be checked. If it exists, it will used to form the index's composite settings. If it does not exist, it is ignored.
+
+=== Usage example
+
+In the following, one component template and an index template are created. The index template references two component templates, but only the `@package` one exists.
+
+
+Create the component template `logs-foo_component1`. This has to be created before the index template as it is not optional:
+
+[source,console]
+----
+PUT _component_template/logs-foo_component1
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host.name": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+----
+
+Next, the index template will be created and it references two component templates:
+
+[source,JSON]
+----
+  "composed_of": ["logs-foo_component1", "logs-foo_component2"]
+----
+
+Before, only the `logs-foo_component1` compontent template was created, meaning the `logs-foo_component2` is missing. Because of this the following entry was added to the config:
+
+[source,JSON]
+----
+  "ignore_missing_component_templates": ["logs-foo_component2"],
+----
+
+During creation of the template, it will not validate that `logs-foo_component2` exists:
+
+
+[source,console]
+----
+PUT _index_template/logs-foo
+{
+  "index_patterns": ["logs-foo-*"],
+  "data_stream": { },
+  "composed_of": ["logs-foo_component1", "logs-foo_component2"],
+  "ignore_missing_component_templates": ["logs-foo_component2"],
+  "priority": 500
+}
+----
+// TEST[continued]
+
+The index template `logs-foo` was successfully created. A data stream can be created based on this template:
+
+[source,console]
+----
+PUT _data_stream/logs-foo-bar
+----
+// TEST[continued]
+
+Looking at the mappings of the data stream, it will contain the `host.name` field.
+
+At a later stage, the missing component template might be added:
+
+[source,console]
+----
+PUT _component_template/logs-foo_component2
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host.ip": {
+          "type": "ip"
+        }
+      }
+    }
+  }
+}
+----
+// TEST[continued]
+
+This will not have an immediate effect on the data stream. The mapping `host.ip` will only show up in the data stream mappings when the data stream is rolled over automatically next time or a manual rollover is triggered:
+
+[source,console]
+----
+POST logs-foo-bar/_rollover
+----
+// TEST[continued]
+// TEST[teardown:data_stream_cleanup]

docs/reference/indices/index-templates.asciidoc

@@ -161,3 +161,5 @@ DELETE _component_template/component_template1
 ////
 
 include::simulate-multi-component-templates.asciidoc[]
+
+include::ignore-missing-component-templates.asciidoc[]

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -286,3 +286,69 @@
 
   - is_false: purple-index.mappings.properties.nested.include_in_root
   - is_true: purple-index.mappings.properties.nested.include_in_parent
+
+---
+"Index template ignore_missing_component_template valid":
+  - skip:
+      version: " - 8.6.99"
+      reason: "index template v2 ignore_missing_component_template config not available before 8.7"
+      features: allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: red
+        body:
+          template:
+            mappings:
+              properties:
+                foo:
+                  type: keyword
+
+  - do:
+      allowed_warnings:
+        - "index template [blue] has index patterns [purple-index] matching patterns from existing older templates [global] with patterns (global => [*]); this template [blue] will take precedence during new index creation"
+      indices.put_index_template:
+        name: blue
+        body:
+          index_patterns: ["purple-index"]
+          composed_of: ["red", "blue"]
+          ignore_missing_component_templates: ["blue"]
+          data_stream: { }
+
+  - do:
+      indices.create_data_stream:
+        name: purple-index
+
+  - do:
+      indices.get:
+        index: purple-index
+
+  - match: {purple-index.mappings.properties.foo: {type: keyword}}
+
+---
+"Index template ignore_missing_component_template invalid":
+  - skip:
+      version: " - 8.6.99"
+      reason: "index template v2 ignore_missing_component_template config not available before 8.7"
+      features: allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: red
+        body:
+          template:
+            mappings:
+              properties:
+                foo:
+                  type: keyword
+
+  - do:
+      catch: /index_template \[blue\] invalid, cause \[index template \[blue\] specifies a missing component templates \[blue\] that does not exist/
+      indices.put_index_template:
+        name: blue
+        body:
+          index_patterns: ["purple-index"]
+          composed_of: ["red", "blue"]
+          ignore_missing_component_templates: ["foo"]
+          data_stream: { }
+

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -46,6 +46,7 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
     private static final ParseField METADATA = new ParseField("_meta");
     private static final ParseField DATA_STREAM = new ParseField("data_stream");
     private static final ParseField ALLOW_AUTO_CREATE = new ParseField("allow_auto_create");
+    private static final ParseField IGNORE_MISSING_COMPONENT_TEMPLATES = new ParseField("ignore_missing_component_templates");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<ComposableIndexTemplate, Void> PARSER = new ConstructingObjectParser<>(
@@ -59,7 +60,8 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
             (Long) a[4],
             (Map<String, Object>) a[5],
             (DataStreamTemplate) a[6],
-            (Boolean) a[7]
+            (Boolean) a[7],
+            (List<String>) a[8]
         )
     );
 
@@ -72,6 +74,7 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM);
         PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE);
+        PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), IGNORE_MISSING_COMPONENT_TEMPLATES);
     }
 
     private final List<String> indexPatterns;
@@ -89,6 +92,8 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
     private final DataStreamTemplate dataStreamTemplate;
     @Nullable
     private final Boolean allowAutoCreate;
+    @Nullable
+    private final List<String> ignoreMissingComponentTemplates;
 
     static Diff<ComposableIndexTemplate> readITV2DiffFrom(StreamInput in) throws IOException {
         return SimpleDiffable.readDiffFrom(ComposableIndexTemplate::new, in);
@@ -106,7 +111,7 @@ public ComposableIndexTemplate(
         @Nullable Long version,
         @Nullable Map<String, Object> metadata
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -118,7 +123,7 @@ public ComposableIndexTemplate(
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -130,6 +135,20 @@ public ComposableIndexTemplate(
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate,
         @Nullable Boolean allowAutoCreate
+    ) {
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null);
+    }
+
+    public ComposableIndexTemplate(
+        List<String> indexPatterns,
+        @Nullable Template template,
+        @Nullable List<String> componentTemplates,
+        @Nullable Long priority,
+        @Nullable Long version,
+        @Nullable Map<String, Object> metadata,
+        @Nullable DataStreamTemplate dataStreamTemplate,
+        @Nullable Boolean allowAutoCreate,
+        @Nullable List<String> ignoreMissingComponentTemplates
     ) {
         this.indexPatterns = indexPatterns;
         this.template = template;
@@ -139,6 +158,7 @@ public ComposableIndexTemplate(
         this.metadata = metadata;
         this.dataStreamTemplate = dataStreamTemplate;
         this.allowAutoCreate = allowAutoCreate;
+        this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates;
     }
 
     public ComposableIndexTemplate(StreamInput in) throws IOException {
@@ -154,6 +174,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException {
         this.metadata = in.readMap();
         this.dataStreamTemplate = in.readOptionalWriteable(DataStreamTemplate::new);
         this.allowAutoCreate = in.readOptionalBoolean();
+        if (in.getVersion().onOrAfter(Version.V_8_7_0)) {
+            this.ignoreMissingComponentTemplates = in.readOptionalStringList();
+        } else {
+            this.ignoreMissingComponentTemplates = null;
+        }
     }
 
     public List<String> indexPatterns() {
@@ -204,6 +229,11 @@ public Boolean getAllowAutoCreate() {
         return this.allowAutoCreate;
     }
 
+    @Nullable
+    public List<String> getIgnoreMissingComponentTemplates() {
+        return ignoreMissingComponentTemplates;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(this.indexPatterns);
@@ -219,6 +249,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(this.metadata);
         out.writeOptionalWriteable(dataStreamTemplate);
         out.writeOptionalBoolean(allowAutoCreate);
+        if (out.getVersion().onOrAfter(Version.V_8_7_0)) {
+            out.writeOptionalStringCollection(ignoreMissingComponentTemplates);
+        }
     }
 
     @Override
@@ -246,6 +279,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (this.allowAutoCreate != null) {
             builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate);
         }
+        if (this.ignoreMissingComponentTemplates != null) {
+            builder.stringListField(IGNORE_MISSING_COMPONENT_TEMPLATES.getPreferredName(), ignoreMissingComponentTemplates);
+        }
         builder.endObject();
         return builder;
     }
@@ -260,7 +296,8 @@ public int hashCode() {
             this.version,
             this.metadata,
             this.dataStreamTemplate,
-            this.allowAutoCreate
+            this.allowAutoCreate,
+            this.ignoreMissingComponentTemplates
         );
     }
 
@@ -280,7 +317,8 @@ && componentTemplatesEquals(this.componentTemplates, other.componentTemplates)
             && Objects.equals(this.version, other.version)
             && Objects.equals(this.metadata, other.metadata)
             && Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate)
-            && Objects.equals(this.allowAutoCreate, other.allowAutoCreate);
+            && Objects.equals(this.allowAutoCreate, other.allowAutoCreate)
+            && Objects.equals(this.ignoreMissingComponentTemplates, other.ignoreMissingComponentTemplates);
     }
 
     static boolean componentTemplatesEquals(List<String> c1, List<String> c2) {
@@ -420,6 +458,7 @@ public static class Builder {
         private Map<String, Object> metadata;
         private DataStreamTemplate dataStreamTemplate;
         private Boolean allowAutoCreate;
+        private List<String> ignoreMissingComponentTemplates;
 
         public Builder() {}
 
@@ -463,6 +502,11 @@ public Builder allowAutoCreate(Boolean allowAutoCreate) {
             return this;
         }
 
+        public Builder ignoreMissingComponentTemplates(List<String> ignoreMissingComponentTemplates) {
+            this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates;
+            return this;
+        }
+
         public ComposableIndexTemplate build() {
             return new ComposableIndexTemplate(
                 this.indexPatterns,
@@ -472,7 +516,8 @@ public ComposableIndexTemplate build() {
                 this.version,
                 this.metadata,
                 this.dataStreamTemplate,
-                this.allowAutoCreate
+                this.allowAutoCreate,
+                this.ignoreMissingComponentTemplates
             );
         }
     }