diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index e736d6b691de..18328bc6bfa2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -229,6 +229,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Support GET requests in Jolokia module. {issue}8566[8566] {pull}9226[9226] - Add freebsd support for the uptime metricset. {pull}9413[9413] - Add `host.os.name` field to add_host_metadata processor. {issue}8948[8948] {pull}9405[9405] +- Add `key` metricset to the Redis module. {issue}9582[9582] {pull}9657[9657] - Add more TCP statuses to `socket_summary` metricset. {pull}9430[9430] ==== Deprecated diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e6713ffd9d60..26fcc366d326 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -20340,6 +20340,53 @@ type: long Count of slow operations +-- + +[float] +== key fields + +`key` contains information about keys. + + + +*`redis.key.name`*:: ++ +-- +type: keyword + +Key name. + + +-- + +*`redis.key.type`*:: ++ +-- +type: keyword + +Key type as shown by `TYPE` command. + + +-- + +*`redis.key.length`*:: ++ +-- +type: long + +Length of the key (Number of elements for lists, length for strings, cardinality for sets). + + +-- + +*`redis.key.expire.ttl`*:: ++ +-- +type: long + +Seconds to expire. + + -- [float] diff --git a/metricbeat/docs/modules/redis.asciidoc b/metricbeat/docs/modules/redis.asciidoc index 0aabeb510cd1..3b62b4584203 100644 --- a/metricbeat/docs/modules/redis.asciidoc +++ b/metricbeat/docs/modules/redis.asciidoc @@ -81,9 +81,13 @@ The following metricsets are available: * <> +* <> + * <> include::redis/info.asciidoc[] +include::redis/key.asciidoc[] + include::redis/keyspace.asciidoc[] diff --git a/metricbeat/docs/modules/redis/key.asciidoc b/metricbeat/docs/modules/redis/key.asciidoc new file mode 100644 index 000000000000..37f524c50e8b --- /dev/null +++ b/metricbeat/docs/modules/redis/key.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-redis-key]] +=== Redis key metricset + +include::../../../module/redis/key/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/redis/key/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 0d52dcf865a1..28d26a645e3a 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -119,7 +119,8 @@ This file is generated! See scripts/docs_collector.py |<> beta[] |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> +.3+| .3+| |<> +|<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .14+| .14+| |<> diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index d241c79389bc..e2afb2060f6a 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -142,6 +142,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/rabbitmq/queue" _ "github.com/elastic/beats/metricbeat/module/redis" _ "github.com/elastic/beats/metricbeat/module/redis/info" + _ "github.com/elastic/beats/metricbeat/module/redis/key" _ "github.com/elastic/beats/metricbeat/module/redis/keyspace" _ "github.com/elastic/beats/metricbeat/module/system" _ "github.com/elastic/beats/metricbeat/module/system/core" diff --git a/metricbeat/module/redis/fields.go b/metricbeat/module/redis/fields.go index 6f132e793588..6a9a151e5d68 100644 --- a/metricbeat/module/redis/fields.go +++ b/metricbeat/module/redis/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzkXM9z27byv+ev2PH3UHcmZvo9tAcf3kzcNH2Zpk3GTg7vRIPgUkIFAiwAylH/+jcASIqiCJL6Qdmdp4vHFIX97GJ/AgvcwAo3t6AwZfoVgGGG4y1c3dv/r14BpKipYoVhUtzCv14BALjvIEejGNVAJedIDaaQKZn7L6NXAAo5Eo23sCCvADKGPNW37vc3IEiOW5r2YzaFfVXJsqie9BC2n0f3q0egUhjChAazRGAikyon9l0gIgVtiGHaWHi7oOynDaUNxw7SPOxDNIDKIbMDTAem0JRKYArJxr36+OGP95/sz/OciDRqDb0ryfrTZaPNCuUMhdE734U4GuFqO+F+UMeCjjrv9IHZASSFcEqy90YNi0ux6PlyBJn9/FHmCSqQWY2wIsak0HCN3ygvUyYWO4+dVmhO1qi/7/KyRW0xoTaxLE1RmpgzbQ7HXyikxGB6C1c/RT9GP1wdx+VHjwU8FrBYgOTS8lUq5dju4V5hwQn1SpaTbzUnSZllqAY433t3hnk7hqMw4oQt3FwxUYF+tpm680jAIQEvviOmqmFkfKbar84wUUcyJIX38PBj9MMAAwmXdHURx6ChQOFcgfXGnrBzDIRzuL77+PnT59dwd7/98/Hz14d/t6AHfG2pzZ7cT/a1btB2/DjU5aIgCR+QayIlRyKOE+0HkTJrKzbKEePiVwe4rgGMia8ozyu6nz9/9THqQHmVGtNIb7o/2yLSlHBM44xL0hcGJkjtYaMN5g4hlUKX+Tb6e+wa1RpV2FZqjDFdMp4q7Ju9c4H9qlGdCrXUAw5pVnkmhK6sCokUCiUpao0DwaMB+6yCHcbcaz055lJtzmtAfszj8jwnyDXhJZ7dn3tHeAvJxmCfkQJ8kYZwEI3Xd28C4Vy6sG7FvFMIBOArHfYBAfBj2A4KVh6286t+QrYcEGt3KGqFkQUqYmwI094Srkm0iggo1Cx1uToa0OxvHAi/juUCyeoZeP6MZFWrW9sYpswSL8kzIP6qMa0RV5PwsSSAYsEERvVbw8hTYojGgyuJM6D/skSnDsBEpWUyczxUkKDf2cBuunmcfZ8B/u9e7pzlzAwmxFEhOaNdt7iFuMLNk1R9qdEEFL+smctwwRMBI+20wtMSRa0ZDqFNgRQSugymQG3UmSKLHIXxqZ61ahmEf0LsubcDQ4LmyboRq46xxxwrrd0KRevZRLChGZ1bHe6cBh/KS5ApQg1bY5yi5S5iOlalEKwX/BkS6PecLID5LNo6cJZVAMADaMRr1ch/M4EFHyhkOOc6SfMr+2uoTFDr5t24LyWHwWQFBvKNHhq9pQ6M6SFMyiwmicd+3jahOmBCsKdxLwG117wccxzDXKcWLwD1fZ3lTBD1NPcKoy72AHjvd6w4TPIwpwoXFnLNhUvMQqM1yqH1ZWTbTP24WC2mlyLMBvbuWL2VXYFKM21Q0K6HONf6yKG1HZckvWQ8tImppWmTVAJpmReQMY42IEpxs5D9WP4Pvsh3EnK5RnisID/aJK3+J6rWpR5dikDSFKRZoqrZ87KBxE2Vr8KutSHKgGE5vgbjikw3ga/db2rLeA1RFH0/HhJVmhwcBqfUUkquWYp6d8spkaWB+3d3A+oEE8MsJ9rEmqwxoksiFqhjzfpHgylmNdFkWmu4nio4ql45iDZOLybithM4M9xfCkmXNwmxZaIlpw3JC4veYdUlpah1VnI3JxZUWF/aPCQLxwATcaHkQmHvCgVMsMQDWOlaJGkwj1jgHmw/A4aYchh2ODk9APaDo1MXtk7sDe5qvUT2S6YPtZ3DSCM9NXik5QDhyby9q0YZ4c6W9xqpFOlwuK4YrfZwXjivtcL18psBEf0J4KAAqCw2sRTxk2KmVlP290vIyHtXaizcGyluHNy67HF7bWmprGi2OnH3riujkFzCpZsMb6HOE6fefnrv49QpYSq87QXz+keLnsvFwmUvVc2+U5WO1FVeCZ/ZxVsmKiht+5ro72smNF1iWj7LLBAR4OGJcQ4JQoMNZJ1H7LsWpoHKvOBocH+hsY/jf0qwCMzvtHhRM/sPCxgBnsMxo8uv73iIXkhoeLBhoeKxzVqn/WMk8u8o7jPmZn2TM42Hl4G/q1ZG1hMzjYv/oRRkomq+aGML9liNcGXZqf3mC2DLslK3brk1FTfRh3Fji9wXwsqWBSmAE4PatakqUxYgVe1fpulfpjeCRlWj2MUWNxzVpj3tT5loZ3bbhpAPbz7BXyX27rt2wafIyeb47ZCp8dZTqaBTWQoTCj/bgFpwRvti/QlrmqEhx5YzleThTeyT9snuJW+cBhPaEJtoXlMibP55lRNtUF29tpp55VqQr0IthtDXQR37tuUg9LP0S9bEoEMsCM9zFcssO6K1od16+1P043HoXbxyTWnf6ca5tTQOAtiCLFnr43IRBbfpTpV3t3Zpg62I9+z+hmAGIubsPSVuPby3q6SPn0AA7TKTMaVNbEc7WqUmKYxX20o1xnFvfzzKwZJpwwf6CI/H/RDq4LEPJ8p9xI6jWYV+jJV28PlydV7dqJSiLGxS/7RkdLkD9MM7DUQhEEqx6Os+6EDmTKzClcoZwk6nOmFiBddl8SaVT+L7UXC28GAyrpYBYrIItx+N+JKBav6gGFRB6e73MJtmEFqt6ZhlxcGowtjEaHhx7Qy7mE1fvMNbORemwVJ31ZGvDl1QnYSYY+bd4EtoSbVgIMFMKmw4aq2UTWPopSuaUzKjiNDW39u0u6psCTz854+fB/aQgvy76Z7Xpe570NoPOOJNGjqCsVBMKmbCnZOnoayH30uOiQYClIiUpdZ4MqkgI4zL9YBhe8RMxwpJKgUPg56jMaESq2vxTG92yPeWPj7mnavqeXCjdU+swoSqZ41K99vMIRFoPI9YMBPrJfn/ixBKmRrQ2XNRSkrG05iFT1Sdi1Au01Mr03Ei8tQsZJwEUXQZJ2yg7fNsEiu5YQXHb0wsYlKw+bWO0vhSplSdBRrSvAlueZyOKsUltNvQIi6kOikSjlMpi0C3zxlpLP+ed3yuyphySQ8+mnMYGSpFxhZxxk5eDxsJgT0d4Cf2Mp5yF4E7Tq6QIlufdvq4P93pnEVrn4yuiYbPsOxC/PPkmxOOgOiJTlqYdBdW6Lg+s3gJpJ5kc0xyCk6BJnIn2I87sDIVJJonqVbVWfl6lSY80xaVvwDhIrCquxb2cQUB+hzdEIGy1JEsdFygivt3/c+5AN2dYShQVaXaRKz+VoRVUsx5pNuWNZVwv/PFAChbwmzRWmH/dvemT2IBGft7Ny4K3O/njiMPV2O2uM9KzmcqHbfGb4mAQktQ+xWgwIbIDrSCKMMIj+RJwXQSwHpBECqaFVhQ+FeJuicb7gWKao6rPHaRpijYBJxBwCvc6Ai/FUzNcn9H1+2vcAOOml/nwHXPHUddcGs2+61DFQ1HENISwUjIybf2WdC9EQZRF4RitByqm84Bu9WMzqVclUUlYl1vI+SECUj9YVcycBq0gZwzrWfemMwI45geCDhcWZWJLhN3nkEgnwP5r1wmOzpclMkbXSZQ0/QerL5aq0yaEcOKXaEuiDGoet6bEXVFcwLocFHj2jPiTKpVXM6TRux3PrqOkMwlQO2mx5xRJbudj+FFBrawATKm7kx3rCVd4Sw2uuukKzoWuXAro79/+PX+7ZdfoChVIfWUzXkXIGPvqHVsFKErTGNrOrOjd/ZZUXToHYpNAx6uSeGWrhOOIAV35+ltNuIeVDe0jXO4e3h7dt/pbkCw2V6rAa5AlUlV3WBRneV2O7Od89xVRjuRlQv4VJJIZeNXH1Oui2p7Q03giPphLK1wE88+Q17vlsTAE6oaON+0oA/s2O7jvcA0dBDrFSuKoyS/tXr5xOUicq1ZvesuPbBHIP9sx3JeicunrTPt8UHd7OBVl/ghN2vWgwzdrumOkNin9cvzXrG5t1I6tFA2ItXfKsTgTgezjGHoboVGQ9eL2JhusXXsrL5doyILBGP4CN2emHEs0Y7mVzlck0wO46hi2ZFQ/hsAAP///1Ge5A==" + return "eJzkXE9z27YSv+dT7Pgd6s7YTN+hPfjwZuIm6cs0bTJ2cuiJBsmlhAoEWACUo376NwBIiqIAkvpD252nQzKWSOxvF/sXWOAaVri5AYkZVa8ANNUMb+Dizvx98QogQ5VKWmoq+A385xUAgP0NCtSSpgpSwRimGjPIpSjcj9ErAIkMicIbWJBXADlFlqkb+/41cFLglqb56E1pHpWiKutvPITN58G+9QCp4JpQrkAvESjPhSyIeRYIz0BpoqnSBt4uKPPpQunCMYO0X/oQDaCyyMwA04FJ1JXkmEGysY8+fPj9/SfzelEQnkWdoXcl2Xz6bHRZSRlFrtXObyGORrjaTrgb1LKgot4zPjA7gATnVkn2nmhgMcEXnh9HkJnP71WRoASRNwhrYlRwBZf4LWVVRvli52urFYqRNarv+7xsURtMqHQsKl1WOmZU6cPxlxJTojG7gYufoh+jHy6O4/KjwwIOCxgsQAph+KqktGx7uJdYMpI6JSvIt4aTpMpzlAOc7z07w7wdw1EYcUIXdq4or0E/20zdOiRgkYAT3xFT1TIyPlPdR2eYqCMZEtx5ePgx+mGAgYSJdPUkjkFBidy6AuONHWHrGAhjcHn78fOnz1dwe7f97+Pnr/f/7UAP+NpK6T25n+xr7aDd+HGoy0VOEjYg10QIhoQfJ9oPPKPGVkyUI9rGrx5w1QAYE19ZnVd0P3/+6mLUgfKqFGaR2vRf2yJSKWGYxTkTxBcGJkjtfqM0FhZhKriqim30d9gVyjXKsK00GON0SVkm0Td75wL7VaE8FWqlBhzSrPJMSLoyKsQzKKVIUSkcCB4t2GcV7DBmr/UUWAi5Oa8BuTGPy/OsINeEVXh2f+4c4Q0kG40+IwX4IjRhwFuvb58EwpiwYd2IeacQCMCXKuwDAuDHsB0UrBxs61fdhGw5IMbukDcKI0qURJsQppwlXJJoFRGQqGhmc3XUoOjfOBB+LcslktUz8PwZyapRt64xTJklVpFnQPxVYdYgrifhY0UA+YJyjJqnhpFnRBOFB1cSZ0D/ZYlWHYDyWstEbnmoIYHf2cBuunmcfZ8B/m9O7owWVA8mxFEpGE37bnELcYWbRyF9qdEEFO/W1Ga44IiAFmZa4XGJvNEMi9CkQBJJugymQF3UuSSLArl2qZ6xahGEf0LsuTMDQ4L60bgRo46xwxxLpewKRee7iWBDMzq3OtxaDT6UlyBTJNV0jXGGhruIqlhWnFMv+DMk0O8ZWQB1WbRx4DSvAYAD0IrXqJH7ZQILLlCIcM51kubX9tdSmaDW7bOxLyWHwWQFBvINDw1vqQNjegiTMotJ4jGfN22oDpgQ7GncS0DtNK/AAscwN6nFC0B912Q5E0Q9zb3CqIs9AN77HSsOkzzMqcITC7nhwiZmodFa5VDqaWTbTv24WA2mlyLMFvbuWN7KrkSpqNLI076HONf6yKG1HRMke8p4aBJTQ9MkqQSyqighpwxNQBT8eiH8WP4FX8RbAYVYIzzUkB9Mktb8EdXrUg82RSBZBkIvUTbsOdlAYqfKVWGXShOpQdMCr0DbItNO4JV9p7GMK4ii6PvxkCiz5OAwOKWWkmJNM1S7W06JqDTcvb0dUCeYGGYZUTpWZI1RuiR8gSpW1D8aTDGriSbTWcN1VMFSdcpBlLZ6MRG3mcCZ4b4rRbq8TogpEw05pUlRGvQWq6rSFJXKK2bnxIAK60uXh2RhGaA8LqVYSPSuUMAESzyAlb5FkhbziAXuwXYzoImuhmGHk9MDYN9bOk1ha8Xe4q7XS4RfMj7UZg4jhempwSOrBghP5u1tPcoId6a8V5gKng2H65rReg/nhfPaKJyX3xwI9yeAgwJIRbmJBY8fJdWNmtK/X0JG7l2pMXCvBb+2cJuyx+61ZZU0otnqxO3bvoxCcgmXbiK8hTpPnHrz6b2LU6eEqfC2F8zrHw16JhYLm73UNftOVTpSVzklfGYXb5iooXTta6K/b5hQ6RKz6llmgfAAD4+UMUgQWmwgmjxi37VQBakoSoYa9xcafRz/U4JFYH6nxYuG2X9YwAjwHI4ZfX5dx0P0QkLDvQkLNY9d1nrtHyORf0dxnzE3803ONB5eBv6+WmnRTMw0Lv6PUpCJqvmijS3YYzXClWGn8ZsvgC3DStO6ZddU7EQfxo0pcl8IK1sWBAdGNCrbpip1VYKQjX+Zpn+52vA0qhvFnmxxw1Jt29P+FImyZrdtCPnw+hP8VaF337UPPkNGNsdvh0yNt45KDT0VFdeh8LMNqCWjqS/Wn7CmGRpybDlTChbexD5pn+xOsNZpUK40MYnmZUq4yT8vCqI0yosro5kXtgX5ItRiCL4O6ti1LQehn6VfsiEGPWJBeI6rWOT5Ea0N3dbbn6Ifj0Nv45VtSvtOtc6to3EQwBZkyVgfE4souE13qrz7tUsXbE3cs/sbghmImLP3lNj1cG9XiY+fQADtM5NTqXRsRjtapSYpjFPbWjXGcW9fHuVgSZVmA32Ex+O+D3XwmC8nyn3EjqNZhX6MlfbwuXJ1Xt2olaIqTVL/uKTpcgfoh7cKiEQgaYqlr/ugB5lRvgpXKmcIO73qhPIVXFbl60w88u9HwZnCg4q4XgaIySLcfjTiSwaq+YNiUA2lv99DTZpB0npNRy9rDkYVxiRGw4trZ9jFbPviLd7auVAFhrqtjlx1aIPqJMQMc+cGX0JLqgEDCeZCYstRZ6VsGkMvXdGskmlJuDL+3qTddWVL4P6P338e2EMK8m+ne16Xuu9BGz9gibdp6AjGUlIhqQ53Tp6Gshl+LzkmCgikhGc0M8aTCwk5oUysBwzbIaYqlkgywVkY9ByNCbVYbYtndr1D3lv6uJh3rqrn3o7WP7EKE6qeNUrlt5lDItB4HrGgOlZL8u8nIZRROaCz56KUVJRlMQ2fqDoXoUJkp1am40TEqVnIOAki02Wc0IG2z7NJrGKalgy/Ub6ISUnn17o0jZ/KlOqzQEOaN8Etj9ORFX8K7dZpGZdCnhQJx6lUZaDb54w0ln/POz6TVZwykR58NOcwMqngOV3EOT15PWwkBHo6wE/sZTzlLgJ7nFxiinR92uljf7rTO4vWPRndEA2fYdmF+OfJNyccAdERnbQwaS+sUHFzZvEpkDqS7THJKTg56sieYD/uwMpUkKgfhVzVZ+WbVZrwTBtU7gKEJ4FV37WwjysI0OXomnAUlYpEqeISZezf9T/nAnR/hqFEWZdqE7G6WxFWSTnnkW5T1tTC/c4VAyBNCbNFa4T96+1rn8QCMnb3bjwpcLefO448XI2Z4j6vGJupdNwavyECEg1B5VaAAhsiO9BKIjUlLBInBdNJAJsFQahp1mBB4l8VKk827AWKco6rPHaRZsjpBJxBwCvcqAi/lVTOcn9H3+2vcAOWmlvnwLXnjqM+uDWd/dahmoYlCFmFoAUU5Fv3LOjeCIOoS5JitByqm84Bu9OMzoRYVWUtYtVsIxSEcsjcYVcycBq0hVxQpWbemMwJZZgdCDhcWVWJqhJ7noEjmwP5L0wkOzpcVslrVSXQ0HQerLlaq0raEcOKXaMuidYoPc/NiLqmOQF0uKix7RlxLuQqruZJI/Y7H21HSG4ToG7TY0FTKfqdj+FFBrowATJO7ZnuWIl0hbPY6K6TrukY5NyujP724Ze7N1/eQVnJUqgpm/M2QMbOUatYS5KuMIuN6cyO3tpnTdGityg2LXi4JKVduk4YguDMnqc32Yj9or6hbZzD3cPbs/tOewOCyfY6DXAlylzI+gaL+iy33ZntneeuM9qJrDyBTyWJkCZ++ZiyXVTbG2oCR9QPY2mFm3j2GXJ6tyQaHlE2wNmmA31gx3Yf7xNMQw+xWtGyPEryW6sXj0wsItua5V138cAegfyzGct6JSYet87U44M62cGrPt1DLtVc4aZzp+b+oRGb5nVeOfyyTPOvVzz+Ra8RCf2KGzviyMVNhsJ5iZrX7QVBS/Fobwh6+PLH53eeC0W9eBjyhV6eSUs+2sGa8GvS9stOusywsJfxmWjAqNLqqqZuv1FaUr5QV5ASmVFOGNUb9wNqNXYXnwtxkdb9MvRYTu7rvWEtmrF9Cm7T31O13A4ydH2sVfdaoPbhee+Q3dsKOE0/LWKwx99pTjF0eUjrgteL+HzT+GaNkiwQtGYjdD1J0bFEe669LlLaammKJh8L5X8BAAD//9IfWds=" } diff --git a/metricbeat/module/redis/key/_meta/data.json b/metricbeat/module/redis/key/_meta/data.json new file mode 100644 index 000000000000..55e1c81c45c1 --- /dev/null +++ b/metricbeat/module/redis/key/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "redis.key", + "duration": 115000, + "module": "redis" + }, + "metricset": { + "name": "key" + }, + "redis": { + "key": { + "expire": { + "ttl": 360 + }, + "length": 3, + "name": "foo", + "type": "string" + }, + "keyspace": { + "id": "db0" + } + }, + "service": { + "address": "192.168.32.2:6379", + "type": "redis" + } +} \ No newline at end of file diff --git a/metricbeat/module/redis/key/_meta/docs.asciidoc b/metricbeat/module/redis/key/_meta/docs.asciidoc new file mode 100644 index 000000000000..b81ba13c1660 --- /dev/null +++ b/metricbeat/module/redis/key/_meta/docs.asciidoc @@ -0,0 +1,25 @@ +The Redis `key` metricset collects information about Redis keys. + +For each key matching one of the configured patterns, an event is sent to +Elasticsearch with information about this key, what includes the type, its +length when available, and its ttl. + +Patterns are configured as a list containing these fields: +* `pattern` (required): pattern for key names, as accepted by the Redis + `KEYS` or `SCAN` commands. +* `limit` (optional): safeguard when using patterns with wildcards to avoid + collecting too many keys (Default: 0, no limit) +* `keyspace` (optional): Identifier of the database to use to look for the keys + (Default: 0) + +For example the following configuration will collect information about all keys +whose name starts with `pipeline-*`, with a limit of 20 keys. + +[source,yaml] +------------------------------------------------------------------------------ +- module: redis + metricsets: ['key'] + key.patterns: + - name: 'pipeline-*' + limit: 20 +------------------------------------------------------------------------------ diff --git a/metricbeat/module/redis/key/_meta/fields.yml b/metricbeat/module/redis/key/_meta/fields.yml new file mode 100644 index 000000000000..df87127e3f5f --- /dev/null +++ b/metricbeat/module/redis/key/_meta/fields.yml @@ -0,0 +1,25 @@ +- name: key + type: group + description: > + `key` contains information about keys. + release: ga + fields: + - name: name + type: keyword + description: > + Key name. + + - name: type + type: keyword + description: > + Key type as shown by `TYPE` command. + + - name: length + type: long + description: > + Length of the key (Number of elements for lists, length for strings, cardinality for sets). + + - name: expire.ttl + type: long + description: > + Seconds to expire. diff --git a/metricbeat/module/redis/key/data.go b/metricbeat/module/redis/key/data.go new file mode 100644 index 000000000000..41a8071bac0f --- /dev/null +++ b/metricbeat/module/redis/key/data.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package key + +import ( + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +func eventMapping(r mb.ReporterV2, keyspace uint, info map[string]interface{}) { + r.Event(mb.Event{ + MetricSetFields: info, + ModuleFields: common.MapStr{ + "keyspace": common.MapStr{ + "id": fmt.Sprintf("db%d", keyspace), + }, + }, + }) +} diff --git a/metricbeat/module/redis/key/key.go b/metricbeat/module/redis/key/key.go new file mode 100644 index 000000000000..86943729dcac --- /dev/null +++ b/metricbeat/module/redis/key/key.go @@ -0,0 +1,115 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package key + +import ( + "time" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/redis" + + rd "github.com/garyburd/redigo/redis" +) + +var ( + debugf = logp.MakeDebug("redis-key") +) + +func init() { + mb.Registry.MustAddMetricSet("redis", "key", New, + mb.WithHostParser(parse.PassThruHostParser), + ) +} + +// MetricSet for fetching Redis server information and statistics. +type MetricSet struct { + mb.BaseMetricSet + pool *rd.Pool + patterns []KeyPattern +} + +// KeyPattern contains the information required to query keys +type KeyPattern struct { + Keyspace uint `config:"keyspace"` + Pattern string `config:"pattern" validate:"required"` + Limit uint `config:"limit"` +} + +// New creates new instance of MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + // Unpack additional configuration options. + config := struct { + IdleTimeout time.Duration `config:"idle_timeout"` + Network string `config:"network"` + MaxConn int `config:"maxconn" validate:"min=1"` + Password string `config:"password"` + Patterns []KeyPattern `config:"key.patterns" validate:"nonzero,required"` + }{ + Network: "tcp", + MaxConn: 10, + Password: "", + } + err := base.Module().UnpackConfig(&config) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + pool: redis.CreatePool(base.Host(), config.Password, config.Network, + config.MaxConn, config.IdleTimeout, base.Module().Config().Timeout), + patterns: config.Patterns, + }, nil +} + +// Fetch fetches information from Redis keys +func (m *MetricSet) Fetch(r mb.ReporterV2) { + conn := m.pool.Get() + for _, p := range m.patterns { + if err := redis.Select(conn, p.Keyspace); err != nil { + logp.Err("Failed to select keyspace %d: %s", p.Keyspace, err) + continue + } + + keys, err := redis.FetchKeys(conn, p.Pattern, p.Limit) + if err != nil { + logp.Err("Failed to fetch list of keys in keyspace %d with pattern '%s': %s", p.Keyspace, p.Pattern, err) + continue + } + if p.Limit > 0 && len(keys) > int(p.Limit) { + debugf("Collecting stats for %d keys, but there are more available for pattern '%s' in keyspace %d", p.Limit) + keys = keys[:p.Limit] + } + + for _, key := range keys { + keyInfo, err := redis.FetchKeyInfo(conn, key) + if err != nil { + logp.Err("Failed to fetch key info for key %s in keyspace %d", key, p.Keyspace) + continue + } + eventMapping(r, p.Keyspace, keyInfo) + } + } +} + +// Close connections +func (m *MetricSet) Close() error { + return m.pool.Close() +} diff --git a/metricbeat/module/redis/key/key_integration_test.go b/metricbeat/module/redis/key/key_integration_test.go new file mode 100644 index 000000000000..76bfb89ce565 --- /dev/null +++ b/metricbeat/module/redis/key/key_integration_test.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package key + +import ( + "testing" + + rd "github.com/garyburd/redigo/redis" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/redis" +) + +var host = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort() + +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "redis") + + addEntry(t) + + ms := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, err := mbtest.ReportingFetchV2(ms) + if err != nil { + t.Fatal("fetch", err) + } + + t.Logf("%s/%s events: %+v", ms.Module().Name(), ms.Name(), events) + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + compose.EnsureUp(t, "redis") + + addEntry(t) + + ms := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(ms, t, "") + if err != nil { + t.Fatal("write", err) + } +} + +// addEntry adds an entry to redis +func addEntry(t *testing.T) { + // Insert at least one event to make sure db exists + c, err := rd.Dial("tcp", host) + if err != nil { + t.Fatal("connect", err) + } + defer c.Close() + _, err = c.Do("SET", "foo", "bar", "EX", "360") + if err != nil { + t.Fatal("SET", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "redis", + "metricsets": []string{"key"}, + "hosts": []string{host}, + "key.patterns": []map[string]interface{}{ + { + "pattern": "foo", + }, + }, + } +} diff --git a/metricbeat/module/redis/redis.go b/metricbeat/module/redis/redis.go index 0b1e1a5df0a9..06cb76193ec5 100644 --- a/metricbeat/module/redis/redis.go +++ b/metricbeat/module/redis/redis.go @@ -29,6 +29,16 @@ import ( rd "github.com/garyburd/redigo/redis" ) +// Redis types +const ( + TypeNone = "none" + TypeString = "string" + TypeList = "list" + TypeSet = "set" + TypeSortedSet = "zset" + TypeHash = "hash" +) + // ParseRedisInfo parses the string returned by the INFO command // Every line is split up into key and value func ParseRedisInfo(info string) map[string]string { @@ -76,6 +86,93 @@ func FetchSlowLogLength(c rd.Conn) (int64, error) { return count, nil } +// FetchKeyInfo collects info about a key +func FetchKeyInfo(c rd.Conn, key string) (map[string]interface{}, error) { + keyType, err := rd.String(c.Do("TYPE", key)) + if err != nil { + return nil, err + } + if keyType == TypeNone { + // Ignore it, it has been removed + return nil, nil + } + + keyTTL, err := rd.Int64(c.Do("TTL", key)) + if err != nil { + return nil, err + } + + info := map[string]interface{}{ + "name": key, + "type": keyType, + "expire": map[string]interface{}{ + "ttl": keyTTL, + }, + } + + lenCommand := "" + + switch keyType { + case TypeString: + lenCommand = "STRLEN" + case TypeList: + lenCommand = "LLEN" + case TypeSet: + lenCommand = "SCARD" + case TypeSortedSet: + lenCommand = "ZCARD" + case TypeHash: + lenCommand = "HLEN" + default: + logp.Debug("redis", "Not supported length for type %s", keyType) + } + + if lenCommand != "" { + length, err := rd.Int64(c.Do(lenCommand, key)) + if err != nil { + return nil, err + } + info["length"] = length + } + + return info, nil +} + +// FetchKeys gets a list of keys based on a pattern using SCAN, `limit` is a +// safeguard to limit the number of commands executed and the number of keys +// returned, if more than `limit` keys are being collected the method stops +// and returns the keys already collected. Setting `limit` to ' disables this +// limit. +func FetchKeys(c rd.Conn, pattern string, limit uint) ([]string, error) { + cursor := 0 + var keys []string + for { + resp, err := rd.Values(c.Do("SCAN", cursor, "MATCH", pattern)) + if err != nil { + return nil, err + } + + var scanKeys []string + _, err = rd.Scan(resp, &cursor, &scanKeys) + if err != nil { + return nil, err + } + + keys = append(keys, scanKeys...) + + if cursor == 0 || (limit > 0 && len(keys) > int(limit)) { + break + } + } + return keys, nil +} + +// Select selects the keyspace to use for this connection +func Select(c rd.Conn, keyspace uint) error { + _, err := c.Do("SELECT", keyspace) + return err +} + // CreatePool creates a redis connection pool func CreatePool( host, password, network string, diff --git a/metricbeat/module/redis/redis_integration_test.go b/metricbeat/module/redis/redis_integration_test.go index 5a4d09af543b..dca51fb8f415 100644 --- a/metricbeat/module/redis/redis_integration_test.go +++ b/metricbeat/module/redis/redis_integration_test.go @@ -20,5 +20,162 @@ package redis import ( + "testing" + + rd "github.com/garyburd/redigo/redis" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" _ "github.com/elastic/beats/metricbeat/mb/testing" ) + +var host = GetRedisEnvHost() + ":" + GetRedisEnvPort() + +func TestFetchKeys(t *testing.T) { + compose.EnsureUp(t, "redis") + + conn, err := rd.Dial("tcp", host) + if err != nil { + t.Fatal("connect", err) + } + defer conn.Close() + + conn.Do("FLUSHALL") + conn.Do("SET", "foo", "bar") + conn.Do("LPUSH", "foo-list", "42") + + k, err := FetchKeys(conn, "notexist", 0) + assert.NoError(t, err) + assert.Empty(t, k) + + k, err = FetchKeys(conn, "foo", 0) + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"foo"}, k) + + k, err = FetchKeys(conn, "foo*", 0) + assert.NoError(t, err) + assert.ElementsMatch(t, []string{"foo", "foo-list"}, k) +} + +func TestFetchKeyInfo(t *testing.T) { + compose.EnsureUp(t, "redis") + + conn, err := rd.Dial("tcp", host) + if err != nil { + t.Fatal("connect", err) + } + defer conn.Close() + + conn.Do("FLUSHALL") + + cases := []struct { + Title string + Key string + Command string + Value []interface{} + Expire uint + Expected map[string]interface{} + }{ + { + Title: "plain string", + Key: "string-key", + Command: "SET", + Value: []interface{}{"foo"}, + Expected: map[string]interface{}{ + "name": "string-key", + "type": "string", + "length": int64(3), + "expire": map[string]interface{}{ + "ttl": int64(-1), + }, + }, + }, + { + Title: "plain string with TTL", + Key: "string-key", + Command: "SET", + Value: []interface{}{"foo"}, + Expire: 60, + Expected: map[string]interface{}{ + "name": "string-key", + "type": "string", + "length": int64(3), + "expire": map[string]interface{}{ + "ttl": int64(60), + }, + }, + }, + { + Title: "list", + Key: "list-key", + Command: "LPUSH", + Value: []interface{}{"foo", "bar"}, + Expected: map[string]interface{}{ + "name": "list-key", + "type": "list", + "length": int64(2), + "expire": map[string]interface{}{ + "ttl": int64(-1), + }, + }, + }, + { + Title: "set", + Key: "set-key", + Command: "SADD", + Value: []interface{}{"foo", "bar"}, + Expected: map[string]interface{}{ + "name": "set-key", + "type": "set", + "length": int64(2), + "expire": map[string]interface{}{ + "ttl": int64(-1), + }, + }, + }, + { + Title: "sorted set", + Key: "sorted-set-key", + Command: "ZADD", + Value: []interface{}{1, "foo", 2, "bar"}, + Expected: map[string]interface{}{ + "name": "sorted-set-key", + "type": "zset", + "length": int64(2), + "expire": map[string]interface{}{ + "ttl": int64(-1), + }, + }, + }, + { + Title: "hash", + Key: "hash-key", + Command: "HSET", + Value: []interface{}{"foo", "bar"}, + Expected: map[string]interface{}{ + "name": "hash-key", + "type": "hash", + "length": int64(1), + "expire": map[string]interface{}{ + "ttl": int64(-1), + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.Title, func(t *testing.T) { + args := append([]interface{}{c.Key}, c.Value...) + conn.Do(c.Command, args...) + defer conn.Do("DEL", c.Key) + if c.Expire > 0 { + conn.Do("EXPIRE", c.Key, c.Expire) + } + + info, err := FetchKeyInfo(conn, c.Key) + require.NoError(t, err) + require.Equal(t, c.Expected, info) + }) + } +} diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py index d57a88465ba0..4a3fafffdfd7 100644 --- a/metricbeat/tests/system/test_redis.py +++ b/metricbeat/tests/system/test_redis.py @@ -63,6 +63,7 @@ def test_keyspace(self): host=self.compose_hosts()[0], port=os.getenv('REDIS_PORT', '6379'), db=0) + r.flushall() r.set('foo', 'bar') self.render_config_template(modules=[{ @@ -85,6 +86,43 @@ def test_keyspace(self): self.assertItemsEqual(self.de_dot(REDIS_KEYSPACE_FIELDS), redis_info.keys()) self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_key(self): + """ + Test redis key metricset + """ + + # At least one event must be inserted so db stats exist + r = redis.StrictRedis( + host=self.compose_hosts()[0], + port=os.getenv('REDIS_PORT', '6379'), + db=0) + r.flushall() + r.rpush('list-key', 'one', 'two', 'three') + + self.render_config_template(modules=[{ + "name": "redis", + "metricsets": ["key"], + "hosts": self.get_hosts(), + "period": "5s", + "additional_content": """ + key.patterns: + - pattern: list-key +""" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(REDIS_FIELDS), evt.keys()) + self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @attr('integration') def test_module_processors(self):