diff --git a/manifests/apache_exporter.pp b/manifests/apache_exporter.pp index 61de1b4ab..040734421 100644 --- a/manifests/apache_exporter.pp +++ b/manifests/apache_exporter.pp @@ -100,6 +100,9 @@ String[1] $config_mode = $prometheus::config_mode, String[1] $arch = $prometheus::real_arch, Stdlib::Absolutepath $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9117, + String[1] $scrape_job_name = 'apache', ) inherits prometheus { #Please provide the download_url for versions < 0.9.0 @@ -133,5 +136,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/beanstalkd_exporter.pp b/manifests/beanstalkd_exporter.pp index 9352fe408..bf25c06c8 100644 --- a/manifests/beanstalkd_exporter.pp +++ b/manifests/beanstalkd_exporter.pp @@ -108,6 +108,9 @@ Variant[Undef,String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 8080, + String[1] $scrape_job_name = 'beanstalkd', ) inherits prometheus { #Please provide the download_url for versions < 0.9.0 @@ -153,5 +156,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/blackbox_exporter.pp b/manifests/blackbox_exporter.pp index 8786e16cb..61ef24029 100644 --- a/manifests/blackbox_exporter.pp +++ b/manifests/blackbox_exporter.pp @@ -116,6 +116,10 @@ String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, Hash $modules = {}, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9115, + String[1] $scrape_job_name = 'blackbox', + ) inherits prometheus { # Prometheus added a 'v' on the release name at 0.1.0 of blackbox @@ -164,5 +168,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/collectd_exporter.pp b/manifests/collectd_exporter.pp index b333bdd70..75c3823b1 100644 --- a/manifests/collectd_exporter.pp +++ b/manifests/collectd_exporter.pp @@ -88,6 +88,9 @@ Optional[String[1]] $download_url = undef, String[1] $arch = $prometheus::real_arch, String[1] $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9103, + String[1] $scrape_job_name = 'collectd', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -118,5 +121,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/config.pp b/manifests/config.pp index 2b245e222..944fb4596 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -121,6 +121,38 @@ } } + # TODO: promtool currently does not support checking the syntax of file_sd_config "includes". + # Ideally we'd check them the same way the other config files are checked. + file { "${prometheus::config_dir}/file_sd_config.d": + ensure => directory, + group => $prometheus::server::group, + purge => true, + recurse => true, + } + + $prometheus::server::collect_scrape_jobs.each |Hash $job_definition| { + if !has_key($job_definition, 'job_name') { + fail('collected scrape job has no job_name!') + } + + $job_name = $job_definition['job_name'] + + Prometheus::Scrape_job <<| job_name == $job_name |>> { + collect_dir => "${prometheus::config_dir}/file_sd_config.d", + notify => Class['::prometheus::service_reload'], + } + } + # assemble the scrape jobs in a single list that gets appended to + # $scrape_configs in the template + $collected_scrape_jobs = $prometheus::server::collect_scrape_jobs.map |$job_definition| { + $job_name = $job_definition['job_name'] + merge($job_definition, { + file_sd_configs => [{ + files => [ "${prometheus::config_dir}/file_sd_config.d/${job_name}_*.yaml" ] + }] + }) + } + if versioncmp($prometheus::server::version, '2.0.0') >= 0 { $cfg_verify_cmd = 'check config' } else { diff --git a/manifests/consul_exporter.pp b/manifests/consul_exporter.pp index 9126d23b4..569539c57 100644 --- a/manifests/consul_exporter.pp +++ b/manifests/consul_exporter.pp @@ -112,6 +112,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9107, + String[1] $scrape_job_name = 'consul', ) inherits prometheus { # Prometheus added a 'v' on the realease name at 0.3.0 @@ -164,5 +167,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/daemon.pp b/manifests/daemon.pp index 56ddce9f7..f04f5facd 100644 --- a/manifests/daemon.pp +++ b/manifests/daemon.pp @@ -85,6 +85,10 @@ Hash[String, Scalar] $env_vars = {}, Optional[String] $env_file_path = $prometheus::env_file_path, Optional[String[1]] $extract_command = $prometheus::extract_command, + Boolean $export_scrape_job = false, + Stdlib::Host $scrape_host = $facts['fqdn'], + Optional[Stdlib::Port] $scrape_port = undef, + String[1] $scrape_job_name = $name, ) { case $install_method { @@ -255,4 +259,16 @@ provider => $real_provider, } } + + if $export_scrape_job { + if $scrape_port == undef { + fail('must set $scrape_port on exported daemon') + } + + @@prometheus::scrape_job { "${scrape_host}:${scrape_port}": + job_name => $scrape_job_name, + targets => ["${scrape_host}:${scrape_port}"], + labels => { 'alias' => $scrape_host }, + } + } } diff --git a/manifests/elasticsearch_exporter.pp b/manifests/elasticsearch_exporter.pp index 102262bb6..2665a5c0b 100644 --- a/manifests/elasticsearch_exporter.pp +++ b/manifests/elasticsearch_exporter.pp @@ -103,6 +103,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9114, + String[1] $scrape_job_name = 'elasticsearch', ) inherits prometheus { #Please provide the download_url for versions < 0.9.0 @@ -137,5 +140,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/graphite_exporter.pp b/manifests/graphite_exporter.pp index 4715271b8..42fd74a5e 100644 --- a/manifests/graphite_exporter.pp +++ b/manifests/graphite_exporter.pp @@ -88,6 +88,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9108, + String[1] $scrape_job_name = 'graphite', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -118,5 +121,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/haproxy_exporter.pp b/manifests/haproxy_exporter.pp index f56ed290a..e4f0a66d5 100644 --- a/manifests/haproxy_exporter.pp +++ b/manifests/haproxy_exporter.pp @@ -96,6 +96,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, Stdlib::Absolutepath $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9101, + String[1] $scrape_job_name = 'haproxy', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -128,6 +131,9 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/init.pp b/manifests/init.pp index 9e7c178cd..dfedaeacd 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -133,6 +133,14 @@ # [*extract_command*] # Custom command passed to the archive resource to extract the downloaded archive. # +# [*collect_scrape_jobs*] +# Array of scrape_configs. Format, e.g.: +# - job_name: some_exporter +# scheme: https +# The jobs defined here will be used to collect resources exported via prometheus::daemon, +# creating the appropriate prometheus scrape configs for each endpoint. All scrape_config +# options can be passed as hash elements. Only the job_name is mandatory. +# # Actions: # # Requires: see Modulefile @@ -185,6 +193,7 @@ Hash $config_defaults = {}, String $os = downcase($facts['kernel']), Optional[Variant[Stdlib::HTTPUrl, Stdlib::Unixpath, String[1]]] $external_url = undef, + Optional[Array[Hash[String[1], Any]]] $collect_scrape_jobs = [], ) { case $arch { diff --git a/manifests/mesos_exporter.pp b/manifests/mesos_exporter.pp index 19d72f894..73386dc8d 100644 --- a/manifests/mesos_exporter.pp +++ b/manifests/mesos_exporter.pp @@ -100,6 +100,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9105, + String[1] $scrape_job_name = 'mesos', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -132,5 +135,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/mongodb_exporter.pp b/manifests/mongodb_exporter.pp index da8c3a505..6582a0be9 100644 --- a/manifests/mongodb_exporter.pp +++ b/manifests/mongodb_exporter.pp @@ -96,6 +96,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9216, + String[1] $scrape_job_name = 'mongodb', ) inherits prometheus { #Please provide the download_url for versions < 0.9.0 @@ -130,5 +133,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/mysqld_exporter.pp b/manifests/mysqld_exporter.pp index e1d1814e0..348c48174 100644 --- a/manifests/mysqld_exporter.pp +++ b/manifests/mysqld_exporter.pp @@ -120,6 +120,9 @@ Optional[Stdlib::Absolutepath] $cnf_socket = undef, String $arch = $prometheus::real_arch, Stdlib::Absolutepath $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9104, + String[1] $scrape_job_name = 'mysql', ) inherits prometheus { #Please provide the download_url for versions < 0.9.0 @@ -165,5 +168,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/nginx_vts_exporter.pp b/manifests/nginx_vts_exporter.pp index 7677ab81c..2dcd6b50c 100644 --- a/manifests/nginx_vts_exporter.pp +++ b/manifests/nginx_vts_exporter.pp @@ -96,6 +96,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9913, + String[1] $scrape_job_name = 'nginx_vts', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -128,6 +131,9 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/node_exporter.pp b/manifests/node_exporter.pp index 0f3174c83..5acf1289a 100644 --- a/manifests/node_exporter.pp +++ b/manifests/node_exporter.pp @@ -112,6 +112,9 @@ Optional[Array[String]] $collectors = undef, Array[String] $collectors_enable = [], Array[String] $collectors_disable = [], + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9100, + String[1] $scrape_job_name = 'node', ) inherits prometheus { # Prometheus added a 'v' on the realease name at 0.13.0 @@ -166,5 +169,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/postgres_exporter.pp b/manifests/postgres_exporter.pp index 5be42c308..2ce105b78 100644 --- a/manifests/postgres_exporter.pp +++ b/manifests/postgres_exporter.pp @@ -123,6 +123,9 @@ Optional[String] $postgres_user = undef, String[1] $arch = $prometheus::real_arch, String[1] $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9187, + String[1] $scrape_job_name = 'postgres', ) inherits prometheus { $release = "v${version}" @@ -213,5 +216,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/process_exporter.pp b/manifests/process_exporter.pp index 4b8203528..d4fda62a2 100644 --- a/manifests/process_exporter.pp +++ b/manifests/process_exporter.pp @@ -95,6 +95,9 @@ Optional[Variant[Stdlib::HTTPSUrl, Stdlib::HTTPUrl]] $download_url = undef, String $arch = $prometheus::real_arch, Stdlib::Absolutepath $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9256, + String[1] $scrape_job_name = 'process', ) inherits prometheus { $filename = "${package_name}-${version}.${os}-${arch}.${download_extension}" @@ -137,6 +140,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } - } diff --git a/manifests/rabbitmq_exporter.pp b/manifests/rabbitmq_exporter.pp index c54d9a1cb..173b8344d 100644 --- a/manifests/rabbitmq_exporter.pp +++ b/manifests/rabbitmq_exporter.pp @@ -128,6 +128,9 @@ Boolean $service_enable = true, String $service_ensure = 'running', Hash[String,String] $extra_env_vars = {}, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9090, + String[1] $scrape_job_name = 'rabbitmq', ) inherits prometheus { $real_download_url = pick($download_url, "${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -171,5 +174,8 @@ service_enable => $service_enable, manage_service => $manage_service, env_vars => $real_env_vars, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/redis_exporter.pp b/manifests/redis_exporter.pp index 6e94ddff1..edf955f6d 100644 --- a/manifests/redis_exporter.pp +++ b/manifests/redis_exporter.pp @@ -107,6 +107,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9121, + String[1] $scrape_job_name = 'redis', ) inherits prometheus { $release = "v${version}" @@ -174,5 +177,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/scrape_job.pp b/manifests/scrape_job.pp new file mode 100644 index 000000000..428a86727 --- /dev/null +++ b/manifests/scrape_job.pp @@ -0,0 +1,40 @@ +# Define: prometheus::scrape_job +# +# This define is used to export prometheus scrape settings from nodes to be scraped to the node +# running prometheus itself. +# This can be used to make prometheus find instances of your running service or application. +# +# [*job_name*] +# The name of the scrape job. This will be used when collecting resources on the prometheus node. +# Corresponds to the prometheus::collect_scrape_jobs parameter. +# +# [*targets*] +# Array of hosts and ports in the form "host:port" +# +# [*labels*] +# Labels added to the scraped metrics on the prometheus side, as label:values pairs +# +# [*collect_dir*] +# Directory used for collecting scrape definitions. +# NOTE: this is a prometheus setting and will be overridden during collection. +# +define prometheus::scrape_job ( + String[1] $job_name, + Array[String[1]] $targets, + Hash[String[1], String[1]] $labels = {}, + Stdlib::Absolutepath $collect_dir = undef, +) { + $config = to_yaml([ + { + targets => $targets, + labels => $labels, + }, + ]) + file { "${collect_dir}/${job_name}_${name}.yaml": + ensure => present, + owner => $prometheus::user, + group => $prometheus::group, + mode => $prometheus::config_mode, + content => "# this file is managed by puppet; changes will be overwritten\n${config}", + } +} diff --git a/manifests/server.pp b/manifests/server.pp index 87cd7d1cf..cc9c30d24 100644 --- a/manifests/server.pp +++ b/manifests/server.pp @@ -44,6 +44,7 @@ Boolean $manage_user = $prometheus::manage_user, Boolean $manage_config = $prometheus::manage_config, Optional[Variant[Stdlib::HTTPurl, Stdlib::Unixpath, String[1]]] $external_url = $prometheus::external_url, + Optional[Array[Hash[String[1], Any]]] $collect_scrape_jobs = $prometheus::collect_scrape_jobs, ) inherits prometheus { if( versioncmp($version, '1.0.0') == -1 ){ diff --git a/manifests/snmp_exporter.pp b/manifests/snmp_exporter.pp index e686ad334..e21d3c8e3 100644 --- a/manifests/snmp_exporter.pp +++ b/manifests/snmp_exporter.pp @@ -104,6 +104,9 @@ String $config_mode = $prometheus::config_mode, String $arch = $prometheus::real_arch, String $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9116, + String[1] $scrape_job_name = 'snmp', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/v${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -163,5 +166,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/statsd_exporter.pp b/manifests/statsd_exporter.pp index 3c1778280..48c4d64c7 100644 --- a/manifests/statsd_exporter.pp +++ b/manifests/statsd_exporter.pp @@ -106,6 +106,9 @@ Boolean $manage_user = true, String $extra_options = '', Optional[Variant[Stdlib::HTTPSUrl, Stdlib::HTTPUrl]] $download_url = undef, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9102, + String[1] $scrape_job_name = 'statsd', ) inherits prometheus { # Prometheus added a 'v' on the realease name at 0.4.0 and changed the configuration format to yaml in 0.5.0 @@ -159,5 +162,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/manifests/varnish_exporter.pp b/manifests/varnish_exporter.pp index dd655493f..567fe2d62 100644 --- a/manifests/varnish_exporter.pp +++ b/manifests/varnish_exporter.pp @@ -94,6 +94,9 @@ Optional[String] $download_url = undef, String $arch = $prometheus::real_arch, Stdlib::Absolutepath $bin_dir = $prometheus::bin_dir, + Boolean $export_scrape_job = false, + Stdlib::Port $scrape_port = 9131, + String[1] $scrape_job_name = 'varnish', ) inherits prometheus { $real_download_url = pick($download_url,"${download_url_base}/download/${version}/${package_name}-${version}.${os}-${arch}.${download_extension}") @@ -125,5 +128,8 @@ service_ensure => $service_ensure, service_enable => $service_enable, manage_service => $manage_service, + export_scrape_job => $export_scrape_job, + scrape_port => $scrape_port, + scrape_job_name => $scrape_job_name, } } diff --git a/spec/defines/daemon_spec.rb b/spec/defines/daemon_spec.rb index 88271b511..1527bf8db 100644 --- a/spec/defines/daemon_spec.rb +++ b/spec/defines/daemon_spec.rb @@ -24,7 +24,10 @@ group: 'smurf_group', env_vars: { SOMEVAR: 42 }, bin_dir: '/usr/local/bin', - install_method: 'url' + install_method: 'url', + export_scrape_job: true, + scrape_host: 'localhost', + scrape_port: 1234 } ].each do |parameters| context "with parameters #{parameters}" do @@ -229,6 +232,13 @@ 'enable' => true ) } + context 'exported resources' do + subject { exported_resources } + + it { + is_expected.to contain_prometheus__scrape_job('localhost:1234') + } + end end end end diff --git a/templates/prometheus.yaml.erb b/templates/prometheus.yaml.erb index 483c475ee..4e813077a 100644 --- a/templates/prometheus.yaml.erb +++ b/templates/prometheus.yaml.erb @@ -7,7 +7,7 @@ <% full_config = { 'global'=>global_config, 'rule_files'=>rule_files, - 'scrape_configs'=>scrape_configs, + 'scrape_configs'=>scrape_configs + @collected_scrape_jobs, 'alerting'=>{ 'alert_relabel_configs'=>scope.lookupvar('::prometheus::server::alert_relabel_config'), 'alertmanagers'=>scope.lookupvar('::prometheus::server::alertmanagers_config'),