From 9de595980fc8055c3dff05c699d556a0701b0ca4 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Tue, 10 Nov 2020 12:02:35 +0100 Subject: [PATCH] Make elasticsearch/index_summary metricset work for Stack Monitoring without xpack.enabled flag (#20615) --- metricbeat/docs/fields.asciidoc | 260 ++++++++++++++++++ .../module/elasticsearch/_meta/fields.yml | 36 +++ .../module/elasticsearch/elasticsearch.go | 10 - .../elasticsearch_integration_test.go | 2 +- metricbeat/module/elasticsearch/fields.go | 2 +- .../index_summary/_meta/data-xpack.json | 60 ---- .../index_summary/_meta/data.json | 154 ++++++++--- .../index_summary/_meta/fields.yml | 96 +++++++ .../elasticsearch/index_summary/data.go | 111 +++++--- .../elasticsearch/index_summary/data_xpack.go | 100 ------- .../index_summary/index_summary.go | 15 +- 11 files changed, 588 insertions(+), 258 deletions(-) delete mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json delete mode 100644 metricbeat/module/elasticsearch/index_summary/data_xpack.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c2d105e8a9aa..3b3dd32dc39e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18134,6 +18134,58 @@ alias to: elasticsearch.node.stats.thread_pool.write.rejected.count -- + + + + +*`indices_stats._all.primaries.indexing.index_total`*:: ++ +-- +type: alias + +alias to: elasticsearch.index.summary.primaries.indexing.index.count + +-- + +*`indices_stats._all.primaries.indexing.index_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.index.summary.primaries.indexing.index.time.ms + +-- + + + +*`indices_stats._all.total.search.query_total`*:: ++ +-- +type: alias + +alias to: elasticsearch.index.summary.total.search.query.count + +-- + +*`indices_stats._all.total.search.query_time_in_millis`*:: ++ +-- +type: alias + +alias to: elasticsearch.index.summary.total.search.query.time.ms + +-- + + +*`indices_stats._all.total.indexing.index_total`*:: ++ +-- +type: alias + +alias to: elasticsearch.index.summary.total.indexing.index.count + +-- + [float] === elasticsearch @@ -18688,6 +18740,110 @@ format: bytes -- +*`elasticsearch.index.summary.primaries.indexing.index.count`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.primaries.indexing.index.time.ms`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.primaries.indexing.is_throttled`*:: ++ +-- +TODO + +type: boolean + +-- + +*`elasticsearch.index.summary.primaries.indexing.throttle_time.ms`*:: ++ +-- +TODO + +type: long + +-- + + + +*`elasticsearch.index.summary.primaries.search.query.count`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.primaries.search.query.time.ms`*:: ++ +-- +TODO + +type: long + +-- + + +*`elasticsearch.index.summary.primaries.bulk.operations.count`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.primaries.bulk.size.bytes`*:: ++ +-- +TODO + +type: long + +-- + + +*`elasticsearch.index.summary.primaries.bulk.time.count.ms`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.primaries.bulk.time.avg.ms`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.primaries.bulk.time.avg.bytes`*:: ++ +-- +TODO + +type: long + +-- + + *`elasticsearch.index.summary.total.docs.count`*:: + -- @@ -18742,6 +18898,110 @@ format: bytes -- + +*`elasticsearch.index.summary.total.indexing.index.count`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.total.indexing.index.time.ms`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.total.indexing.is_throttled`*:: ++ +-- +TODO + +type: boolean + +-- + +*`elasticsearch.index.summary.total.indexing.throttle_time.ms`*:: ++ +-- +TODO + +type: long + +-- + + + +*`elasticsearch.index.summary.total.search.query.count`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.total.search.query.time.ms`*:: ++ +-- +TODO + +type: long + +-- + + +*`elasticsearch.index.summary.total.bulk.operations.count`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.total.bulk.size.bytes`*:: ++ +-- +TODO + +type: long + +-- + + +*`elasticsearch.index.summary.total.bulk.time.count.ms`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.total.bulk.time.avg.ms`*:: ++ +-- +TODO + +type: long + +-- + +*`elasticsearch.index.summary.total.bulk.time.avg.bytes`*:: ++ +-- +TODO + +type: long + +-- + [float] === ml.job diff --git a/metricbeat/module/elasticsearch/_meta/fields.yml b/metricbeat/module/elasticsearch/_meta/fields.yml index 1ae2a10ed7a6..07b33ad5773d 100644 --- a/metricbeat/module/elasticsearch/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/_meta/fields.yml @@ -276,6 +276,42 @@ - name: rejected path: elasticsearch.node.stats.thread_pool.write.rejected.count type: alias + - name: indices_stats + type: group + fields: + - name: _all + type: group + fields: + - name: primaries + type: group + fields: + - name: indexing + type: group + fields: + - name: index_total + type: alias + path: elasticsearch.index.summary.primaries.indexing.index.count + - name: index_time_in_millis + type: alias + path: elasticsearch.index.summary.primaries.indexing.index.time.ms + - name: total + type: group + fields: + - name: search + type: group + fields: + - name: query_total + type: alias + path: elasticsearch.index.summary.total.search.query.count + - name: query_time_in_millis + type: alias + path: elasticsearch.index.summary.total.search.query.time.ms + - name: indexing + type: group + fields: + - name: index_total + type: alias + path: elasticsearch.index.summary.total.indexing.index.count - name: elasticsearch type: group description: > diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index c97cfa65b23e..23b24eafa99d 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -29,8 +29,6 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" - s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/helper/elastic" @@ -119,14 +117,6 @@ type licenseWrapper struct { License License `json:"license"` } -var BulkStatsDict = c.Dict("bulk", s.Schema{ - "total_operations": c.Int("total_operations"), - "total_time_in_millis": c.Int("total_time_in_millis"), - "total_size_in_bytes": c.Int("total_size_in_bytes"), - "avg_time_in_millis": c.Int("avg_time_in_millis"), - "avg_size_in_bytes": c.Int("avg_size_in_bytes"), -}, c.DictOptional) - // GetClusterID fetches cluster id for given nodeID. func GetClusterID(http *helper.HTTP, uri string, nodeID string) (string, error) { // Check if cluster id already cached. If yes, return it. diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index 2838b141d346..4ee5c9531503 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -131,7 +131,7 @@ func TestXPackEnabled(t *testing.T) { "cluster_stats": []string{"cluster_stats"}, "enrich": []string{"enrich_coordinator_stats"}, "index_recovery": []string{"index_recovery"}, - "index_summary": []string{"indices_stats"}, + "index_summary": []string{}, // no longer indexed into .monitoring-es-* "ml_job": []string{"job_stats"}, "node_stats": []string{}, // no longer indexed into .monitoring-es-* } diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index 39e239c8e8f0..ebdfa00c478b 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded gzipped contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzsXU1v5DbSvvtXFHxKgBkB79WHNwtkk10HyCDYTPayWChsqbqbY0mUSarHzq9fkPpoSk1KlETZ7YF9CDJu66mnisUiWSpWf4QHfL4DzIiQNBFIeHK8AZBUZngHtz+Zv7+9AUhRJJyWkrLiDv7/BgCg9zeQs7TK8AaAY4ZE4B0cyA2AQClpcRB38J9bIbLbD3B7lLK8/a/67Mi4jBNW7OnhDvYkE+r5PcUsFXdaxEcoSI53IGmOQpK81L8FkM8l3gHJKBHNb0oij3dw+7fuL297AElWCYk8riqaTmD0TBI1D0bNYy2eYBVPMC5Yij24A2dVS9JUxHzW4GDn4eKipEXGsy2g+u9ywO7pDo6lGAtJpJit2l5c8DAfsz3aG2YmSdb7xIXiQjLRyInQjOwyjGkR754lios/HTGLNkC0F5Go8pzw56iDi1xYdsOfCVHWM+t6DW32GkccQzWRWYmcqMlu09PPbq22kWYZnRGjhFWFdOC6bDgkyJGkcXiWCjY01a+cStyAq8ZdQ7bzyyKlCa6burRI8YkWh2C+rQFjFc7V9M1pltEF87dRLWrpRWfYKJ8zhy3UHJNvISM9pK4R9GElj5xJmWF4m/WQ55mtWxjUaKdEkmD+kWPO+HMs6F9r4nura8cvqnEXx/jHCvlznJDkiFeqqsFwrbIcHysU8qrV7XFcq3C3Tw6jaT0UoeZr88kZdHmEazDWRbg+n0Xx7Wz3Q45FwJ2Ti4i3ajWfVQE7ZUl8IlmFIm5cfr23d8TO4Et9Hnrh+wnTeEdlLFBuwLaHH4JwvajqHRLfgK8JH4JueIYBSBWM51u4psYNQbBktJBbMKyBQ1AUknFM4zocbcC0hx+CsESexydMJONb8DXhQ9HdimcQgifkgrIizkm5AU0DfQnZluSXU77q8HdIAi7MWVa7xzZ5jSwdPSC7sKfwLSqocRk7kcP0eH855dEhic42iViWRmf80RM/eKQoHLQnd6Sh+Fs3qL7kW+LPrLrIO/Qxrn1UtQZvelwvNFg0ssZ2KFhAOaKKvORpRdhV6uaYRwoqysnTqgVB86kEpqEIKaxAjErkCS45F10SKhPfY1HvtUPvzYjPO5BxTLYuqZmUVTA/zBhJY3JCTg7DVMk48Bi4KeD/hnOm/ZkYOyaipKyiht8hcuJMzdrExn7FTqCsSDLiRWtsVQlywLggBVuYjVdG0wSihmakISNndt9nIl66Wxhtk72IHysmSZzThAdROUr2ItKYUbVEZeidkcj4W4sQy3dR5TvkMdvHmJGyCXaUpStWwb5B1O+iAXbQdfysgX7HHbdZedeGdpkGA+ygGujtRwc9Ov2Wkze4OyfjFPF+WmabOckKyVkWj/m2twGao58Ppu+kzGhO5dgWZQlBDercq8yhVwfwwPTqED6XXpeN4ixBcT0bjsWbuUYRPa3mb+PkUb+aLxnLVpliV2UPwWzxWGFl23VNWMLQJVJ8Io2zKunP8Qsm0hq055JpoRa/VTng8KlXtfAB5dUYWHFZbV/9luCaLKwJXY2NazarrRz+nexaM59ffF6FnZtPVxtav+66JjvXBVDXYuaazWwrtySGlbh261rKcN3VmG3tqrU+9AGfvzJu6mzFrn/6pb4NrpYSOaVakjQBZNLULVGNiTU5tEKuxrRKNWp/7WNlGxoTgA4dzkV2grD6+cRShPu/W+UMhj+EpP7Im8JyoobCKm7HWIakmCfuXoA8oja2/p8aX//7BzuBjCUPJLOXES+l0IICFmSXYQqs6Gj9cOmPCZ90jBGZP3ImxMfW4TmWGU10hScMq4f7pfbtz5jPZUhSx/AsLtJ0Bjm7f01o3/58IjkC2zeMHZLOXvcUC3yMC+YkkzHrSyIPJr+SJ5pXOQh8rLBIsMnEKHJd8W3rEA1bcSQDvbtKGpZl7OvbGoKW88QgaKWj2jgbDMOnzurK0FoYfKXySGvLj3M7V0nrqh2JwxgQluFZXM0LU/iunciYfg+0kEyz7kxb67PnLB/3Ixjm0wQtEozVmhXr4vWRmr/lmn2mOX4AWkAuPoCW2GevxMMeZXLECyWc9A8Z25EsTo6YPOiKmw2I/0PLgLMM0EVxarr2TT+6qZhO6oyF8yaQrw/eCuEi275iKTeZVQK+O3DE4gM8ozLMB+CYfm9f5NWy51+L6bGhEJoD1fuuyCsAjhdyjrjNpNN8ZpJkRozXyirfbz3CwcS681lH5RxQavCPmNED3WXoTcpSch+KkoKe4OG+YAJrPOa+hrtin2kU9hggHXwuQ/bYC4ZJNr/rtcRtneXl0avXkYGdau1HzGSm12lOOB2pcghHq5b1PE3v4nrLeBXeGhf7VQNDJdQKy7gh8mLpwoLTXkZs9pr1k0YIsGTVyRlB/7KfQC2mmFotzr5TH9Cb+x16lNSmo5ZoJcMxZxLj9olgW/Ck4tz+pmWtY/5YI5snjkoKSYqUFodGn84C7rkzfk80yIbXTQvwCZNKYtpsHPVRXkjCZVXax6l9IK6HGJsLj5t7kDwS2Uye9m0e4wKO5IR+SrjOZLPn3xBg/sQLm/S5V3xGsj5hL26nLJm6WRMu1qcsqXQxNZgHSfdc0uRSzNCeMA5Or5Y0m6a+GKAD70RtoIPqnvGcyDtwPeytiqLQHto1Z6WARh0h73e9KpyVa2Kd2GliXsX2W5v2zD83Nge75zpB0lC1mNseryKOCTthr97lFQKXIys+P87Xu+BWqV4K35Sn4MMFyoFQhWQX22wxrZIXZak/8wqBDvaudtlCXtagrtD5X622GtfiZqZsSfgBZRTy3cdnDVkvyc5RrsUembCXPKwWrJCBpClHIeC7hFVZCjuE+9+6XzKu/0jxcaRVGpJhl26TZH8Bt/uGbmwTdHx+15Dj49OIDTs+puAQ49OQDDs+Jkn7C9U6OjdtaF43OLsO4e97vSD03vd673u9Sf7z9nrwfkp7n7nvM/cNztxzQUv0he3WrPt5tsWJbNFu54+CPlYIeQZf2M69G5TEUWm4SOgvbFdD2qWlRJL6Tm9X+45prM5wPLXe9V16Iv2tBa/fneHp0o1tnGhxIhlN45RIDMrn89GsZakVFrqmApDKI3IgkFMhaHFQhLD2ErVRJvW/dcK03ksXTKr9dEm4wNRyxrhwa6/yuREFBs/Pd+umd0I4N9OVcQ2qfVT7fRbcansI++Xfv8J9sWd+bz6ntJ7S3INQS8pqALi4VFVfFabFyH2grcPzP5GUoBj0IrLSYXrtM5WYvhG+sQ45eVquQsGK1x+KT6z4GGA4Wl1ec0Q6VfxHZXCDLNqghHXfgoPCxlTXctVGs0bm9WVPv3flB0B2rJKAJDk2SaACiL28e90+ZWaVyVUdcPBJGUsttO0R4noPN1d0JHhrh4H2nDVxYoeAZ8RFZ0At1IOcq68rBLkkPTFkk6rD5ND1cqyX7YvDqDHVc9hDjwHUaDPNGXAenX0n0IxCH0ffW3hjrjDW1Pat6WK9GxlGiak2sx5aDKDWOPUw2m+isn9PVQ/lpxcVM8rOaI8aWPacVqeBRXs1Cg0s06/1Z2Chs7p4BpY9pyPnBqJfWuaMjpkrJRtJn643WrCzyUYn3BZenVs3CaEvNYutbUtmCbG1TxkOaclYFu68ae9NGmC7MeIrEMbuXqcMqO9Suo4UJuUSycPVcP4NyYMv6fiajK2J534WH+0i+dLE/6jTZqMnT1fX1/f58tqc3+fLSxP3mS+i4id6Yu6b2e9T5vU4v0+ZlybumjLmFq/X4jrkNs9olL3JdAyQGVyTCxw2A/8WdLQ6yT6cV1yW905jgp+7/0wzBPEsJOYjYnxsXqformEKdyPA0adj58sRmv6W0Jdi1SXw7N86CkFmoetm6TT6lARTiud3XIKPZc1Ev/93fc6EnvXVnCPYnQECXlbeqMm4b//40ZXD2kI+DL9Zfdonlw67FaeZTrHtyZhuse7BGAZ1lZNIY9R96JsC5/ZD99THFDG3YflCEb5txT3g+68Bt3cjj67g4FH/10edbuUNvob2WevMU8RUk+4AgocVWfZW2E4hPjlke59sWBXbLT2zxwHHQKH/hnakx+qoLfpAk+1RJ7BanMvW1d+oouPt/b4pVbctVrguXW29lL8RVW+GQG3vpBJ1f5hYEmFGqdmVpX/aAP/U6xyhhQACzQegPjCRzFTPkvpSgVzGjLs6qM6/gXGvIeES0rhwyziV9j4BS26gWOC6+advIlslrbjZHMHPjAM+kbzMlEKV/JiTshzW2vX2WrSIaze+yNAsvupCc13O2O863mk+aLM52yWb7p3aeVb52GZtIeSRCqB1M2WPFhHWLq5h7hlpJuPdKUJe/vqsq1mJRB/ZHDOWEKmCiq6CL8K2YjAbWTcNbIlohTadU6Ob/wUAAP//VN0ZjA==" + return "eJzsXV9v4zYSf8+nGOSpBXYF3Gsergf0z10KdFtc03s5HFRaGtvcSKJDUt6kn/5A6o8lmaQoiUqchfNQdOP4N78ZDsnhcEh+hEd8uQPMiJA0EUh4sr8BkFRmeAe3P3Z/f3sDkKJIOD1Iyoo7+PsNAEDvbyBnaZnhDQDHDInAO9iRGwCBUtJiJ+7gv7dCZLcf4HYv5eH2f+qzPeMyTlixpbs72JJMqO9vKWapuNMiPkJBcrwDSXMUkuQH/VsA+XLAOyAZJaL+zYHI/R3c/qP9y9seQJKVQiKPy5KmIxg9k0T1F6P6aw2eYCVPMC5Yij24HWdlQ7KrSPe7HQ5mHjYuSlrU+W4DqP47H7D9dgvHUoyFJFJMVm0rznh0v2b6aq+ZmSRZ7xMbig2pi0aOhGZkk2FMi3jzIlGc/anDLNoA0VZEosxzwl+iFi6yYZkNfyJEWc+syzU02cuN6ELtIrMDcqI6u0lPP7s12kaaZXRCjBJWFtKCa7PhkCBHksbhWSrY0FS/cCpxBa4adwnZ1i+LlCa4rOvSIsVnWuyC+bYGjNVwrrpvTrOMzui/tWpRQy86wUb5lD5soGbpfDMZ6Sa1taAPK7nnTMoMw9ushzzNbO3EoFo7JZIE848cc8ZfYkH/WjK+N7q2/KIKd/YY/1Qif4kTkuzxQlXtMFyqLMenEoW8aHV7HJcq3MbJYTStmiJUf60/OYHOH+FqjGUjXJ/PrPHtZPddjkXAyMlGxFu1is+iATtlSXwkWYkirl1+ube3xE7gc30eesP3M6bxhspYoFyBbQ8/BOFqUtUREl+Bbxc+BN3wDAOQKhjP13BNjRuC4IHRQq7BsAIOQVFIxjGNq+FoBaY9/BCEJfI8PmIiGV+Dbxc+FN21eAYheEQuKCvinBxWoNlBn0O2Ifn5mC9a/O2SgBNzllXusU5eI0udC2Qb9hi+QQXVLq4VOYy39+djHu2S6GSTiGVpdMJ3rvjBI0VhoT0akYbibwxQfck3xF9YeZZ36GNceqtqDd51u55pMKtlO+FQsAFlj2rkJc8Lhl2lbo55pKCinDwvmhA0n1JgGoqQwgrE6IA8wTnronNCh8R3WdTbdujtjPjsgbgx2bKkZnIog/lhxkgakyNyshumStzALvCugL8N+0zzM9J2TETJoYxqfrvIijPWaxMT+wWRwKEkicOLltiqFGSHcUEKNjMbr4ymCUQ1zUhDRtbsvk9HPHe3MNomWxE/lUySOKcJD6JylGxFpDGjco7K0FsjEfeuRYjpuyjzDfKYbWPMyKEe7ChLF8yCfYOo30UD7KDz+EkDvccdN1l5W0A7T4MBdlANdPjRQju733zyHe7WzjhGvJ+WWadPskJylsUu3/Y2QL3088H07ZQZzal0hShzCGpQa6wyhV41gAemVw3hU+m12SjOEhSXE3DMDuZqRXS3mh7Gyb3emj8wli0yxabMHoPZ4qnE0hR1jViio0uk+EQaZ1HSn+NnTKRx0J5KpoGavauyw+G33tTCO5QXY2DFZbF99S7BJVlYE7oYG1dsFls5/J7sUjOfNj4vws71p4sNrbe7LsnOVQHUpZi5YjPZyoMirJmFlzHJlk23B05zwulZ5LOwgOu8IswN6gI+A7cWKIDXwsTUttWY1JR8tkYZVGs5VkUTy9dWZ2qqA1mn3NY4DrshXbDgXY4CgWxYlah0q1Y82tmzhGdFjrZKn6+oH1ZaW/tgQ2Z4mMGsouEkg31cbcr/jSX2j/jyhfHutGHErn76pyVqXC0lsko15LkDyKSpXaKafYz59QVyNaZRauf4hLmtTE3TBaDDOdtGdoSw+vnEUoT7H4xyBs0fQlK/5bvCcqKawihuw1iGpJgm7l6A3KM2tv6fCl//+zszgYwlj/2AYjmFBhSwIJsMU2BFS+u7c39M+KhjOGR+z5kQHxuH53jIaKKL5GF4AKN/Wqn5cflchiS1NM/sMMk6Npv9a0T75ucTyRHYtmZskXTyuudY4FNcMCuZjBlnEQ8mv5Bnmpc5CHwqsUiwTmYrcu35hcYharZiTwZ6t8WILMvYl/fVBA3nkUbQSkeVcVZohk+t1ZWhtTD4QuWeVpZ3czsdNNGFjxKHY0BYhidxFS9M4ZumI2P6LdBCMs26NW2lz5az3O1HMNySELRIMFZzVqzP/zjKpudr9kBz/AC0gFx8AC2xz16Jhy3KZI9nSljp7zK2IVmc7DF51EWLKxD/p5YBJxmg64pVd+2b3hlUjOfFXcN5PZAvH7wVwtmG5YKpvMusFPDNjiMWH+AFlWE+AMf0W/Mkr6Y9/7W2R0AhNAeq467IawB018I73GbUaR5UvN4Z47Wyyvcbj7AwMUY+y6icBpQK/CNmdEc3GXqTMpxaCkVJQY/wsJ/RgyUec1/BXbDP1Ap7NJAefM6HbNfidpTN73ousVtn/gmTxfPIwE6V9g4zjef41qBVyXoZp3d2QtBdyLzExX7RwFAKNcMy3hF5NnVhwWkvmTV5zvpRIwSYsqr8tqB/mVegBlOMzRYn36kW6PUROd1KKuioJBrJcMyZxLj5RrAQPCk5N29WL3XM7yvk7oqjlEKSIqXFrtantYC977iP2gcJeO20AJ8xKSWmdeCol/JCEi7Lg7mdmi/EVRNjfWZ8dQ+SeyLrztMURDAuYE+O6KeEbU02uf8NAaZ3vLBJn3vFx5H1CZuMT1kydjgx3FifsqTU51Ggu5C09yVNLsUMzXtuwelVkibT1Ger9MA7Ul5tobplPCfyDmxf9lZFUWgW7ZqzUkCjOsj7nVANZ+WKWCt2nJjXeaW1TXvin3eCg81LlSCpqRrMbR6vIo4JO2KvZPANBi5LVnz6OF9FwY1SvRR+V56CDzdQDoQqJLPYOsQ0Sp6VpX7gJQIdxK5m2UKel/Ev0PnfjbYa1+BmXdmS8B3KKOTex4OGrKZkaytXYvdMmKvGFgtWyEDSlKMQ8E3CyiyFDcL9b+0vGdd/pPhY0io1ybBTd5dkfwI3+4a+Gyxo+/yuId3tU4sN2z5dwSHapyYZtn26JM0bqr3t5LcdnMMX2lxjvWusd4311ov1hlqtX1DjPHXT0/fh1x9+tfyd06hDgfbrgtYQOX54yRXFTWPVOZcwdomZhwo+Ul+rIM6pQZgj7z5nvxy2ggm+09+T9Tk/vliw46gJBGmmSXczLvK5rlTnLLaCPNVer+OLr+IVQ8nkuHszua5WDCT6mn9cnd41Jr3GpKP8rzHpNSa9xqTXmPQak4aQd41J32tMejqEEH1mm5sh7oRcbd4NZ4Ptos3KUP9R0KcSIc/gM9vYM/iSWA7YzhL6M9tUkGZpKZGkusquvfIB05hjwnhqvOJu7i7ibw14Ve+Ix/MAzcSJFkeS0TROicSgfB723fMHlcJC18EDUrlHDgRyKgQtdooQVl4CTP1e/1sXuVT7HwWTsEE4EC4wNewLnbm115EnhwKD70936/rK0HBupk8z1ajmVu1fL2pX20PYz//5Be6LLfOrVh3TekxzD0INKaMBugzq5Ya+IY8Wjmtw1l54/AvJARSD3lpD6TC+zOgqMX4R4so65OR5vgoFK96+KT6x4mOA5mh0ecsWaVXxb5XTjmh139AKxw63DTgobEz1+ZvKaMaReflRld/bknEgG1ZKQJLs6437Aoj5SO6yOGXiyYCLSt3hszKWmmib5Njlpu0uKNn13tJcTQZxQkZrafZzVnZTC/UgZ3vOCIIs0UeabFR1GG2617wbYklqy3DNxJKsRbi8Wy/pZXz/CN6ZK7jecnpvuqx/FU0Ip/Z5GGkEbjjar6Ky/1NCHsqPTyrdUXbCq0CBZU954SewaK/3cQLL9HvxJrDQSY/XBJY95SGaFUS/tswJD8UslNxJ+rRPAgRbm6y0wm3g1bp1lSH0tXqx8bbeSUJMtwYPm/TAWBZuvWl+kidAuOHwFQhjd69VBlT339iWFF3KBySPF8P5NySPvqTjSzK2Jp77Wdz5eMprE/+jSps5V562x46u/eWtOV/7y2sT9+kvouRHemT227SuXebtOF+7zGsTt3WZbojXe9ktZJjXeR9ule4YIDO4JBc4fAPva9DR6CTbcF5xfiRzHBP83P0nmiGIFyExd4jxsXmVoruELty2AMeF5XSBCZEjoRnZZG/Pqk3gsXi4mdunsaQXjl94HaK+0LNUE/yK7k6JfpJG60BXTyAsx24NEPCCqZXe1vN9NtE5cxhfTgzDb9LzhKNTh9mK40zH2PZkjL8s6MEYBnWVo0gu6j70uwKnPgPoqQ8MCpynvNM3U4Tva3oe8P1twPXdyOMxPPCo/+ujjr9gB76G9pnruquIsbfpAggeVmSZX4CzCvHJIZufh4NFY/tqZyXcTws5bdEHGn0VaASrwTl/se0rVdR9JftXpeq6xQqXpavpCbGvRNWbIVBz3+0B9Z2esSSiO0pNriz90wT4p57nCC0EEKg/APVBF6mb6plTXyqQy5hx26sX009g3GtIOIc8TUCUcSrNd7vNOYFigGv7n749yihpwW1UEfzEOOAzyQ+ZUqiUH3NyOAxr7XqxFi3iyo3PMjSzj7rQXJcz9h/bazUfPI0w2SXrFxe08yzysdWu8pN7KoBWD+B4XOtnfHkjzDkjzcR9o2DIw18PupqVSPSRzTFjCZFqUNFV8EXY6/O6jw/Vj44Q0QitX7uIbv4fAAD//3JZXJs=" } diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json b/metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json deleted file mode 100644 index ad68a4b786e7..000000000000 --- a/metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "_index": ".monitoring-es-6-2018.05.15", - "_type": "doc", - "_id": "CGdxY2MBLoB8ROglGWFD", - "_score": 1, - "_source": { - "cluster_uuid": "SFQoPJkbSc2-n_so4ZbUwA", - "timestamp": "2018-05-15T10:55:46.476Z", - "interval_ms": 10000, - "type": "indices_stats", - "source_node": { - "uuid": "YYSPA7QWSQyBQSYgIdOAKw", - "host": "127.0.0.1", - "transport_address": "127.0.0.1:9300", - "ip": "127.0.0.1", - "name": "YYSPA7Q", - "timestamp": "2018-05-15T10:55:46.360Z" - }, - "indices_stats": { - "_all": { - "primaries": { - "docs": { - "count": 509 - }, - "store": { - "size_in_bytes": 1377439 - }, - "indexing": { - "index_total": 531, - "index_time_in_millis": 5220, - "is_throttled": false, - "throttle_time_in_millis": 0 - }, - "search": { - "query_total": 908, - "query_time_in_millis": 10160 - } - }, - "total": { - "docs": { - "count": 509 - }, - "store": { - "size_in_bytes": 1377439 - }, - "indexing": { - "index_total": 531, - "index_time_in_millis": 5220, - "is_throttled": false, - "throttle_time_in_millis": 0 - }, - "search": { - "query_total": 908, - "query_time_in_millis": 10160 - } - } - } - } - } -} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/data.json b/metricbeat/module/elasticsearch/index_summary/_meta/data.json index 686b38e54cf4..3dadbd5b61b3 100644 --- a/metricbeat/module/elasticsearch/index_summary/_meta/data.json +++ b/metricbeat/module/elasticsearch/index_summary/_meta/data.json @@ -1,61 +1,153 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "@timestamp": "2020-08-26T20:27:04.201Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "event": { + "dataset": "elasticsearch.index.summary", + "module": "elasticsearch", + "duration": 143822781 + }, + "metricset": { + "name": "index_summary", + "period": 10000 }, "elasticsearch": { - "cluster": { - "id": "UziYVLPkTTmCzccc6102Bg", - "name": "elasticsearch" - }, "index": { "summary": { - "primaries": { - "docs": { - "count": 231, - "deleted": 56 + "total": { + "store": { + "size": { + "bytes": 12088959 + } }, "segments": { - "count": 16, "memory": { - "bytes": 105245 + "bytes": 111208 + }, + "count": 18 + }, + "indexing": { + "is_throttled": false, + "throttle_time": { + "ms": 0 + }, + "index": { + "count": 0, + "time": { + "ms": 0 + } } }, - "store": { + "bulk": { + "time": { + "count": { + "ms": 0 + }, + "avg": { + "ms": 0, + "bytes": 0 + } + }, "size": { - "bytes": 444882 + "bytes": 0 + }, + "operations": { + "count": 0 + } + }, + "search": { + "query": { + "count": 0, + "time": { + "ms": 0 + } } + }, + "docs": { + "count": 1297, + "deleted": 418 } }, - "total": { + "primaries": { "docs": { - "count": 231, - "deleted": 56 + "count": 1297, + "deleted": 418 + }, + "store": { + "size": { + "bytes": 12088959 + } }, "segments": { - "count": 16, "memory": { - "bytes": 105245 + "bytes": 111208 + }, + "count": 18 + }, + "indexing": { + "index": { + "time": { + "ms": 0 + }, + "count": 0 + }, + "is_throttled": false, + "throttle_time": { + "ms": 0 } }, - "store": { + "bulk": { + "time": { + "avg": { + "bytes": 0, + "ms": 0 + }, + "count": { + "ms": 0 + } + }, "size": { - "bytes": 444882 + "bytes": 0 + }, + "operations": { + "count": 0 + } + }, + "search": { + "query": { + "time": { + "ms": 0 + }, + "count": 0 } } } } + }, + "cluster": { + "name": "docker-cluster", + "id": "giRKn5pZQ0afY0YrRiXdxQ" } }, - "metricset": { - "host": "127.0.0.1:9200", - "module": "elasticsearch", - "name": "index_summary", - "namespace": "elasticsearch.index.summary", - "rtt": 115 + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "7cca1b46-be04-4e3b-aa72-048c9e91fa6e", + "id": "803dfdba-e638-4590-a2de-80cb1cebe78d" }, "service": { - "name": "elasticsearch" + "type": "elasticsearch", + "name": "elasticsearch", + "address": "localhost:9200" } -} \ No newline at end of file +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/fields.yml b/metricbeat/module/elasticsearch/index_summary/_meta/fields.yml index 17df1df7aebf..07a4bee3de50 100644 --- a/metricbeat/module/elasticsearch/index_summary/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/index_summary/_meta/fields.yml @@ -29,6 +29,54 @@ format: bytes description: > Total number of memory used by the segments in bytes. + - name: indexing + type: group + fields: + - name: index.count + description: TODO + type: long + - name: index.time.ms + description: TODO + type: long + - name: is_throttled + type: boolean + description: TODO + - name: throttle_time.ms + type: long + description: TODO + - name: search + type: group + fields: + - name: query + type: group + fields: + - name: count + type: long + description: TODO + - name: time.ms + type: long + description: TODO + - name: bulk + type: group + fields: + - name: operations.count + type: long + description: TODO + - name: size.bytes + type: long + description: TODO + - name: time + type: group + fields: + - name: count.ms + type: long + description: TODO + - name: avg.ms + type: long + description: TODO + - name: avg.bytes + type: long + description: TODO - name: total type: group fields: @@ -54,3 +102,51 @@ format: bytes description: > Total number of memory used by the segments in bytes. + - name: indexing + type: group + fields: + - name: index.count + description: TODO + type: long + - name: index.time.ms + description: TODO + type: long + - name: is_throttled + type: boolean + description: TODO + - name: throttle_time.ms + type: long + description: TODO + - name: search + type: group + fields: + - name: query + type: group + fields: + - name: count + type: long + description: TODO + - name: time.ms + type: long + description: TODO + - name: bulk + type: group + fields: + - name: operations.count + type: long + description: TODO + - name: size.bytes + type: long + description: TODO + - name: time + type: group + fields: + - name: count.ms + type: long + description: TODO + - name: avg.ms + type: long + description: TODO + - name: avg.bytes + type: long + description: TODO diff --git a/metricbeat/module/elasticsearch/index_summary/data.go b/metricbeat/module/elasticsearch/index_summary/data.go index fe75162bf737..17d228b8f80b 100644 --- a/metricbeat/module/elasticsearch/index_summary/data.go +++ b/metricbeat/module/elasticsearch/index_summary/data.go @@ -31,52 +31,73 @@ import ( var ( schema = s.Schema{ - "primaries": c.Dict("primaries", s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - "deleted": c.Int("deleted"), - }), - "store": c.Dict("store", s.Schema{ - "size": s.Object{ - "bytes": c.Int("size_in_bytes"), - }, - }), - "segments": c.Dict("segments", s.Schema{ - "count": c.Int("count"), - "memory": s.Object{ - "bytes": c.Int("memory_in_bytes"), - }, - }), - }), - "total": c.Dict("total", s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - "deleted": c.Int("deleted"), - }), - "store": c.Dict("store", s.Schema{ - "size": s.Object{ - "bytes": c.Int("size_in_bytes"), - }, - }), - "segments": c.Dict("segments", s.Schema{ - "count": c.Int("count"), - "memory": s.Object{ - "bytes": c.Int("memory_in_bytes"), - }, - }), - }), + "primaries": c.Dict("primaries", indexSummaryDict), + "total": c.Dict("total", indexSummaryDict), } ) -func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { - var event mb.Event - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", elasticsearch.ModuleName) +var indexSummaryDict = s.Schema{ + "docs": c.Dict("docs", s.Schema{ + "count": c.Int("count"), + "deleted": c.Int("deleted"), + }), + "store": c.Dict("store", s.Schema{ + "size": s.Object{ + "bytes": c.Int("size_in_bytes"), + }, + }), + "segments": c.Dict("segments", s.Schema{ + "count": c.Int("count"), + "memory": s.Object{ + "bytes": c.Int("memory_in_bytes"), + }, + }), + "indexing": indexingDict, + "bulk": bulkStatsDict, + "search": searchDict, +} - event.ModuleFields = common.MapStr{} - event.ModuleFields.Put("cluster.name", info.ClusterName) - event.ModuleFields.Put("cluster.id", info.ClusterID) +var indexingDict = c.Dict("indexing", s.Schema{ + "index": s.Object{ + "count": c.Int("index_total"), + "time": s.Object{ + "ms": c.Int("index_time_in_millis"), + }, + }, + "is_throttled": c.Bool("is_throttled"), + "throttle_time": s.Object{ + "ms": c.Int("throttle_time_in_millis"), + }, +}) + +var searchDict = c.Dict("search", s.Schema{ + "query": s.Object{ + "count": c.Int("query_total"), + "time": s.Object{ + "ms": c.Int("query_time_in_millis"), + }, + }, +}) + +var bulkStatsDict = c.Dict("bulk", s.Schema{ + "operations": s.Object{ + "count": c.Int("total_operations"), + }, + "time": s.Object{ + "count": s.Object{ + "ms": c.Int("total_time_in_millis"), + }, + "avg": s.Object{ + "ms": c.Int("avg_time_in_millis"), + "bytes": c.Int("avg_size_in_bytes"), + }, + }, + "size": s.Object{ + "bytes": c.Int("total_size_in_bytes"), + }, +}, c.DictOptional) +func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { var all struct { Data map[string]interface{} `json:"_all"` } @@ -91,6 +112,14 @@ func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) erro return errors.Wrap(err, "failure applying stats schema") } + var event mb.Event + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", elasticsearch.ModuleName) + + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("cluster.name", info.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) + event.MetricSetFields = fields r.Event(event) diff --git a/metricbeat/module/elasticsearch/index_summary/data_xpack.go b/metricbeat/module/elasticsearch/index_summary/data_xpack.go deleted file mode 100644 index 4e35744133d4..000000000000 --- a/metricbeat/module/elasticsearch/index_summary/data_xpack.go +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package index_summary - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" -) - -var ( - xpackSchema = s.Schema{ - "primaries": c.Dict("primaries", indexStatsSchema), - "total": c.Dict("total", indexStatsSchema), - } - - indexStatsSchema = s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - }), - "store": c.Dict("store", s.Schema{ - "size_in_bytes": c.Int("size_in_bytes"), - }), - "indexing": c.Dict("indexing", s.Schema{ - "index_total": c.Int("index_total"), - "index_time_in_millis": c.Int("index_time_in_millis"), - "is_throttled": c.Bool("is_throttled"), - "throttle_time_in_millis": c.Int("throttle_time_in_millis"), - }), - "bulk": elasticsearch.BulkStatsDict, - "search": c.Dict("search", s.Schema{ - "query_total": c.Int("query_total"), - "query_time_in_millis": c.Int("query_time_in_millis"), - }), - } -) - -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { - var all struct { - Data map[string]interface{} `json:"_all"` - } - - err := json.Unmarshal(content, &all) - if err != nil { - return errors.Wrap(err, "failure parsing Elasticsearch Stats API response") - } - - p := all.Data["primaries"] - primaries, ok := p.(map[string]interface{}) - if !ok { - return fmt.Errorf("primaries is not a map") - } - - if len(primaries) == 0 { - // There is no data in the cluster, hence no metrics to parse or report - return nil - } - - fields, err := xpackSchema.Apply(all.Data) - if err != nil { - return errors.Wrap(err, "failure applying stats schema") - } - - event := mb.Event{} - event.RootFields = common.MapStr{} - event.RootFields.Put("indices_stats._all", fields) - event.RootFields.Put("cluster_uuid", info.ClusterID) - event.RootFields.Put("timestamp", common.Time(time.Now())) - event.RootFields.Put("interval_ms", m.Module().Config().Period/time.Millisecond) - event.RootFields.Put("type", "indices_stats") - - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) - - r.Event(event) - return nil -} diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index dc3dbdd7207c..c163bef237ad 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -80,18 +80,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to get info from Elasticsearch") } - if m.XPack { - err = eventMappingXPack(r, m, *info, content) - if err != nil { - // Since this is an x-pack code path, we log the error but don't - // return it. Otherwise it would get reported into `metricbeat-*` - // indices. - m.Logger().Error(err) - return nil - } - } else { - return eventMapping(r, *info, content) - } - - return nil + return eventMapping(r, *info, content) }