add ability to export/collect scrape_jobs

manifests/apache_exporter.pp

@@ -100,6 +100,9 @@
   String[1] $config_mode                                             = $prometheus::config_mode,
   String[1] $arch                                                    = $prometheus::real_arch,
   Stdlib::Absolutepath $bin_dir                                      = $prometheus::bin_dir,
+  Boolean $export_scrape_job                                         = false,
+  Stdlib::Port $scrape_port                                          = 9117,
+  String[1] $scrape_job_name                                         = 'apache',
 ) inherits prometheus {
 
   #Please provide the download_url for versions < 0.9.0
@@ -133,5 +136,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/beanstalkd_exporter.pp

@@ -108,6 +108,9 @@
   Variant[Undef,String] $download_url = undef,
   String $arch                        = $prometheus::real_arch,
   String $bin_dir                     = $prometheus::bin_dir,
+  Boolean $export_scrape_job          = false,
+  Stdlib::Port $scrape_port           = 8080,
+  String[1] $scrape_job_name          = 'beanstalkd',
 ) inherits prometheus {
 
   #Please provide the download_url for versions < 0.9.0
@@ -153,5 +156,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/blackbox_exporter.pp

@@ -116,6 +116,10 @@
   String $arch                   = $prometheus::real_arch,
   String $bin_dir                = $prometheus::bin_dir,
   Hash $modules                  = {},
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9115,
+  String[1] $scrape_job_name     = 'blackbox',
+
 ) inherits prometheus {
 
   # Prometheus added a 'v' on the release name at 0.1.0 of blackbox
@@ -164,5 +168,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/collectd_exporter.pp

@@ -88,6 +88,9 @@
   Optional[String[1]] $download_url = undef,
   String[1] $arch                   = $prometheus::real_arch,
   String[1] $bin_dir                = $prometheus::bin_dir,
+  Boolean $export_scrape_job        = false,
+  Stdlib::Port $scrape_port         = 9103,
+  String[1] $scrape_job_name        = 'collectd',
 ) inherits prometheus {
 
   $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}")
@@ -118,5 +121,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/config.pp

@@ -121,6 +121,38 @@
     }
   }
 
+  # TODO: promtool currently does not support checking the syntax of file_sd_config "includes".
+  # Ideally we'd check them the same way the other config files are checked.
+  file { "${prometheus::config_dir}/file_sd_config.d":
+    ensure  => directory,
+    group   => $prometheus::server::group,
+    purge   => true,
+    recurse => true,
+  }
+
+  $prometheus::server::collect_scrape_jobs.each |Hash $job_definition| {
+    if !has_key($job_definition, 'job_name') {
+      fail('collected scrape job has no job_name!')
+    }
+
+    $job_name = $job_definition['job_name']
+
+    Prometheus::Scrape_job <<| job_name == $job_name |>> {
+      collect_dir => "${prometheus::config_dir}/file_sd_config.d",
+      notify      => Class['::prometheus::service_reload'],
+    }
+  }
+  # assemble the scrape jobs in a single list that gets appended to
+  # $scrape_configs in the template
+  $collected_scrape_jobs = $prometheus::server::collect_scrape_jobs.map |$job_definition| {
+    $job_name = $job_definition['job_name']
+    merge($job_definition, {
+      file_sd_configs => [{
+        files => [ "${prometheus::config_dir}/file_sd_config.d/${job_name}_*.yaml" ]
+      }]
+    })
+  }
+
   if versioncmp($prometheus::server::version, '2.0.0') >= 0 {
     $cfg_verify_cmd = 'check config'
   } else {

manifests/consul_exporter.pp

@@ -112,6 +112,9 @@
   Optional[String] $download_url = undef,
   String $arch                   = $prometheus::real_arch,
   String $bin_dir                = $prometheus::bin_dir,
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9107,
+  String[1] $scrape_job_name     = 'consul',
 ) inherits prometheus {
 
   # Prometheus added a 'v' on the realease name at 0.3.0
@@ -164,5 +167,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/daemon.pp

@@ -85,6 +85,10 @@
   Hash[String, Scalar] $env_vars       = {},
   Optional[String] $env_file_path      = $prometheus::env_file_path,
   Optional[String[1]] $extract_command = $prometheus::extract_command,
+  Boolean $export_scrape_job           = false,
+  Stdlib::Host $scrape_host            = $facts['fqdn'],
+  Optional[Stdlib::Port] $scrape_port  = undef,
+  String[1] $scrape_job_name           = $name,
 ) {
 
   case $install_method {
@@ -255,4 +259,16 @@
       provider => $real_provider,
     }
   }
+
+  if $export_scrape_job {
+    if $scrape_port == undef {
+      fail('must set $scrape_port on exported daemon')
+    }
+
+    @@prometheus::scrape_job { "${scrape_host}:${scrape_port}":
+      job_name => $scrape_job_name,
+      targets  => ["${scrape_host}:${scrape_port}"],
+      labels   => { 'alias' => $scrape_host },
+    }
+  }
 }

manifests/elasticsearch_exporter.pp

@@ -103,6 +103,9 @@
   Optional[String] $download_url = undef,
   String $arch                   = $prometheus::real_arch,
   String $bin_dir                = $prometheus::bin_dir,
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9114,
+  String[1] $scrape_job_name     = 'elasticsearch',
 ) inherits prometheus {
 
   #Please provide the download_url for versions < 0.9.0
@@ -137,5 +140,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/graphite_exporter.pp

@@ -88,6 +88,9 @@
   Optional[String] $download_url = undef,
   String $arch                   = $prometheus::real_arch,
   String $bin_dir                = $prometheus::bin_dir,
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9108,
+  String[1] $scrape_job_name     = 'graphite',
 ) inherits prometheus {
 
   $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}")
@@ -118,5 +121,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/haproxy_exporter.pp

@@ -96,6 +96,9 @@
   Optional[String] $download_url = undef,
   String $arch                   = $prometheus::real_arch,
   Stdlib::Absolutepath $bin_dir  = $prometheus::bin_dir,
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9101,
+  String[1] $scrape_job_name     = 'haproxy',
 ) inherits prometheus {
 
   $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}")
@@ -128,6 +131,9 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 
 }

manifests/init.pp

@@ -133,6 +133,14 @@
 #  [*extract_command*]
 #  Custom command passed to the archive resource to extract the downloaded archive.
 #
+#  [*collect_scrape_jobs*]
+#  Array of scrape_configs. Format, e.g.:
+#  - job_name: some_exporter
+#    scheme: https
+#  The jobs defined here will be used to collect resources exported via prometheus::daemon,
+#  creating the appropriate prometheus scrape configs for each endpoint. All scrape_config
+#  options can be passed as hash elements. Only the job_name is mandatory.
+#
 # Actions:
 #
 # Requires: see Modulefile
@@ -185,6 +193,7 @@
   Hash $config_defaults = {},
   String $os            = downcase($facts['kernel']),
   Optional[Variant[Stdlib::HTTPUrl, Stdlib::Unixpath, String[1]]] $external_url = undef,
+  Optional[Array[Hash[String[1], Any]]] $collect_scrape_jobs = [],
 ) {
 
   case $arch {

manifests/mesos_exporter.pp

@@ -100,6 +100,9 @@
   Optional[String] $download_url = undef,
   String $arch                   = $prometheus::real_arch,
   String $bin_dir                = $prometheus::bin_dir,
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9105,
+  String[1] $scrape_job_name     = 'mesos',
 ) inherits prometheus {
 
   $real_download_url    = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}")
@@ -132,5 +135,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/mongodb_exporter.pp

@@ -96,6 +96,9 @@
   Optional[String] $download_url = undef,
   String $arch                   = $prometheus::real_arch,
   String $bin_dir                = $prometheus::bin_dir,
+  Boolean $export_scrape_job     = false,
+  Stdlib::Port $scrape_port      = 9216,
+  String[1] $scrape_job_name     = 'mongodb',
 ) inherits prometheus {
 
   #Please provide the download_url for versions < 0.9.0
@@ -130,5 +133,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/mysqld_exporter.pp

@@ -120,6 +120,9 @@
   Optional[Stdlib::Absolutepath] $cnf_socket                         = undef,
   String $arch                                                       = $prometheus::real_arch,
   Stdlib::Absolutepath $bin_dir                                      = $prometheus::bin_dir,
+  Boolean $export_scrape_job                                         = false,
+  Stdlib::Port $scrape_port                                          = 9104,
+  String[1] $scrape_job_name                                         = 'mysql',
 ) inherits prometheus {
 
   #Please provide the download_url for versions < 0.9.0
@@ -165,5 +168,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/nginx_vts_exporter.pp

@@ -96,6 +96,9 @@
   Optional[String] $download_url      = undef,
   String           $arch              = $prometheus::real_arch,
   String           $bin_dir           = $prometheus::bin_dir,
+  Boolean $export_scrape_job          = false,
+  Stdlib::Port $scrape_port           = 9913,
+  String[1] $scrape_job_name          = 'nginx_vts',
 ) inherits prometheus {
 
   $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}")
@@ -128,6 +131,9 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 
 }

manifests/node_exporter.pp

@@ -112,6 +112,9 @@
   Optional[Array[String]] $collectors = undef,
   Array[String] $collectors_enable    = [],
   Array[String] $collectors_disable   = [],
+  Boolean $export_scrape_job          = false,
+  Stdlib::Port $scrape_port           = 9100,
+  String[1] $scrape_job_name          = 'node',
 ) inherits prometheus {
 
   # Prometheus added a 'v' on the realease name at 0.13.0
@@ -166,5 +169,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/postgres_exporter.pp

@@ -123,6 +123,9 @@
   Optional[String] $postgres_user               = undef,
   String[1] $arch                               = $prometheus::real_arch,
   String[1] $bin_dir                            = $prometheus::bin_dir,
+  Boolean $export_scrape_job                    = false,
+  Stdlib::Port $scrape_port                     = 9187,
+  String[1] $scrape_job_name                    = 'postgres',
 ) inherits prometheus {
 
   $release = "v${version}"
@@ -213,5 +216,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }

manifests/process_exporter.pp

@@ -95,6 +95,9 @@
   Optional[Variant[Stdlib::HTTPSUrl, Stdlib::HTTPUrl]] $download_url = undef,
   String $arch                                                       = $prometheus::real_arch,
   Stdlib::Absolutepath $bin_dir                                      = $prometheus::bin_dir,
+  Boolean $export_scrape_job                                         = false,
+  Stdlib::Port $scrape_port                                          = 9256,
+  String[1] $scrape_job_name                                         = 'process',
 ) inherits prometheus {
 
   $filename = "${package_name}-${version}.${os}-${arch}.${download_extension}"
@@ -137,6 +140,8 @@
     service_ensure     => $service_ensure,
     service_enable     => $service_enable,
     manage_service     => $manage_service,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
-
 }

manifests/rabbitmq_exporter.pp

@@ -128,6 +128,9 @@
   Boolean $service_enable             = true,
   String $service_ensure              = 'running',
   Hash[String,String] $extra_env_vars = {},
+  Boolean $export_scrape_job          = false,
+  Stdlib::Port $scrape_port           = 9090,
+  String[1] $scrape_job_name          = 'rabbitmq',
 ) inherits prometheus {
 
   $real_download_url    = pick($download_url, "${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}")
@@ -171,5 +174,8 @@
     service_enable     => $service_enable,
     manage_service     => $manage_service,
     env_vars           => $real_env_vars,
+    export_scrape_job  => $export_scrape_job,
+    scrape_port        => $scrape_port,
+    scrape_job_name    => $scrape_job_name,
   }
 }