Collect version metadata for Fluentd

fluentd/datadog_checks/fluentd/data/conf.yaml.example

@@ -1,4 +1,10 @@
 init_config:
+    ## @param fluentd - string - optional - default: fluentd
+    ## Command or path to fluentd (e.g. `/usr/local/bin/fluentd` or `docker exec container fluentd`).
+    ## Can be overwritten on an instance.
+    #
+    # fluentd: fluentd
+
     ## @param proxy - object - optional
     ## Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list
     ## to specify hosts that must bypass proxies.
@@ -33,6 +39,11 @@ instances:
     #
   - monitor_agent_url: http://example.com:24220/api/plugins.json
 
+    ## @param fluentd - string - optional - default: fluentd
+    ## Command or path to fluentd (e.g. `/usr/local/bin/fluentd` or `docker exec container fluentd`).
+    #
+    # fluentd: fluentd
+
     ## @param plugin_ids - list of strings - optional
     ## Enter your Plugin IDs to monitor a specific scope of plugins.
     #

fluentd/datadog_checks/fluentd/fluentd.py

@@ -3,10 +3,14 @@
 # All rights reserved
 # Licensed under Simplified BSD License (see LICENSE)
 
+import re
+import shlex
+
 # 3rd party
 from six.moves.urllib.parse import urlparse
 
 # project
+from datadog_checks.base.utils.subprocess_output import get_subprocess_output
 from datadog_checks.checks import AgentCheck
 
 
@@ -15,6 +19,7 @@ class Fluentd(AgentCheck):
     SERVICE_CHECK_NAME = 'fluentd.is_ok'
     GAUGES = ['retry_count', 'buffer_total_queued_size', 'buffer_queue_length']
     _AVAILABLE_TAGS = frozenset(['plugin_id', 'type'])
+    VERSION_PATTERN = r'.* (?P<version>[0-9\.]+)$'
 
     def __init__(self, name, init_config, instances):
         super(Fluentd, self).__init__(name, init_config, instances)
@@ -29,6 +34,8 @@ def __init__(self, name, init_config, instances):
             )
             self.http.options['timeout'] = (timeout, timeout)
 
+        self._fluentd_command = shlex.split(self.instance.get('fluentd', init_config.get('fluentd', 'fluentd')))
+
     """Tracks basic fluentd metrics via the monitor_agent plugin
     * number of retry_count
     * number of buffer_queue_length
@@ -81,9 +88,34 @@ def check(self, instance):
                     # Filter unspecified plugins to keep backward compatibility.
                     if len(plugin_ids) == 0 or p.get('plugin_id') in plugin_ids:
                         self.gauge('fluentd.%s' % (m), metric, [tag] + custom_tags)
+
+            self._collect_metadata()
         except Exception as e:
             msg = "No stats could be retrieved from %s : %s" % (url, str(e))
             self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, message=msg)
             raise
         else:
             self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=service_check_tags)
+
+    def _collect_metadata(self):
+        raw_version = self._get_raw_version()
+
+        if raw_version:
+            self.set_metadata('version', raw_version)
+
+    def _get_raw_version(self):
+        command = self._fluentd_command + ['--version']
+
+        try:
+            out, _, _ = get_subprocess_output(command, self.log, False)
+        except OSError as exc:
+            self.log.warning("Error collecting fluentd version: %s", exc)
+            return None
+
+        match = re.match(self.VERSION_PATTERN, out)
+
+        if match is None:
+            self.log.warning("fluentd version not found in stdout: `%s`", out)
+            return None
+
+        return match.group('version')

fluentd/tests/common.py

@@ -9,6 +9,10 @@
 HERE = os.path.dirname(os.path.abspath(__file__))
 ROOT = os.path.dirname(os.path.dirname(HERE))
 
+FLUENTD_VERSION = os.environ.get('FLUENTD_VERSION')
+FLUENTD_IMAGE_TAG = os.environ.get('FLUENTD_IMAGE_TAG')
+FLUENTD_CONTAINER_NAME = 'dd-test-fluentd'
+
 HOST = get_docker_hostname()
 PORT = 24220
 BAD_PORT = 24222

fluentd/tests/compose/docker-compose.yaml

@@ -2,7 +2,8 @@ version: '3'
 services:
 
   fluentd:
-    image: fluent/fluentd:${FLUENTD_VERSION}
+    image: fluent/fluentd:${FLUENTD_IMAGE_TAG}
+    container_name: ${FLUENTD_CONTAINER_NAME}
     ports:
       - 24220:24220
     volumes:

fluentd/tests/conftest.py

@@ -8,7 +8,7 @@
 
 from datadog_checks.dev import docker_run
 
-from .common import DEFAULT_INSTANCE, HERE, URL
+from .common import DEFAULT_INSTANCE, FLUENTD_CONTAINER_NAME, FLUENTD_IMAGE_TAG, HERE, URL
 
 
 @pytest.fixture(scope="session")
@@ -19,10 +19,13 @@ def dd_environment():
     If there's any problem executing docker-compose, let the exception bubble
     up.
     """
+    if not FLUENTD_IMAGE_TAG:
+        pytest.skip('FLUENTD_IMAGE_TAG is required')
 
     env = {
         'TD_AGENT_CONF_PATH': os.path.join(HERE, 'compose', 'td-agent.conf'),
-        'FLUENTD_VERSION': os.environ.get('FLUENTD_VERSION') or 'v0.12.23',
+        'FLUENTD_IMAGE_TAG': FLUENTD_IMAGE_TAG,
+        'FLUENTD_CONTAINER_NAME': FLUENTD_CONTAINER_NAME,
     }
 
     with docker_run(

fluentd/tests/mock/fluentd_version.py

@@ -0,0 +1,5 @@
+import sys
+
+if __name__ == "__main__":
+    mock_output = sys.argv[1]
+    print('fluentd {}'.format(mock_output))

fluentd/tests/test_metadata.py

@@ -0,0 +1,59 @@
+# (C) Datadog, Inc. 2019
+# All rights reserved
+# Licensed under Simplified BSD License (see LICENSE)
+import os
+
+import pytest
+
+from datadog_checks.fluentd import Fluentd
+
+from .common import CHECK_NAME, FLUENTD_CONTAINER_NAME, FLUENTD_VERSION, HERE
+
+CHECK_ID = 'test:123'
+VERSION_MOCK_SCRIPT = os.path.join(HERE, 'mock', 'fluentd_version.py')
+
+
+@pytest.mark.skipif(not FLUENTD_VERSION, reason="FLUENTD_VERSION is required")
+@pytest.mark.usefixtures("dd_environment")
+def test_collect_metadata_instance(aggregator, datadog_agent, instance):
+    instance['fluentd'] = 'docker exec {} fluentd'.format(FLUENTD_CONTAINER_NAME)
+
+    check = Fluentd(CHECK_NAME, {}, [instance])
+    check.check_id = CHECK_ID
+    check.check(instance)
+
+    major, minor, patch = FLUENTD_VERSION.split('.')
+    version_metadata = {
+        'version.raw': FLUENTD_VERSION,
+        'version.scheme': 'semver',
+        'version.major': major,
+        'version.minor': minor,
+        'version.patch': patch,
+    }
+
+    datadog_agent.assert_metadata(CHECK_ID, version_metadata)
+    datadog_agent.assert_metadata_count(5)
+
+
+@pytest.mark.usefixtures("dd_environment")
+def test_collect_metadata_missing_version(aggregator, datadog_agent, instance):
+    instance["fluentd"] = "python {} 'fluentd not.a.version'".format(VERSION_MOCK_SCRIPT)
+
+    check = Fluentd(CHECK_NAME, {}, [instance])
+    check.check_id = CHECK_ID
+    check.check(instance)
+
+    datadog_agent.assert_metadata(CHECK_ID, {})
+    datadog_agent.assert_metadata_count(0)
+
+
+@pytest.mark.usefixtures("dd_environment")
+def test_collect_metadata_invalid_binary(datadog_agent, instance):
+    instance['fluentd'] = '/bin/does_not_exist'
+
+    check = Fluentd(CHECK_NAME, {}, [instance])
+    check.check_id = CHECK_ID
+    check.check(instance)
+
+    datadog_agent.assert_metadata(CHECK_ID, {})
+    datadog_agent.assert_metadata_count(0)

fluentd/tox.ini

@@ -17,8 +17,10 @@ passenv =
     COMPOSE*
     DOCKER*
 setenv =
-    0.12.23: FLUENTD_VERSION=v0.12.23
-    1.4: FLUENTD_VERSION=v1.4
+    0.12.23: FLUENTD_VERSION=0.12.23
+    0.12.23: FLUENTD_IMAGE_TAG=v0.12.23
+    1.4: FLUENTD_VERSION=1.4.2
+    1.4: FLUENTD_IMAGE_TAG=v1.4-2
 commands =
     pip install -r requirements.in
     pytest -v {posargs}