From 0ae766b8fc851ca9881c61cc593e7650388f6f8c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 10 Jul 2019 09:40:52 -0600 Subject: [PATCH] Separate AWS module config file example into metricset specific (#12701) * Separate AWS module config file example into metricset specific --- metricbeat/docs/modules/aws.asciidoc | 58 +--------- x-pack/metricbeat/metricbeat.reference.yml | 9 +- .../module/aws/_meta/config.reference.yml | 52 +++++++++ x-pack/metricbeat/module/aws/_meta/config.yml | 39 +------ .../metricbeat/module/aws/_meta/docs.asciidoc | 49 -------- .../module/aws/cloudwatch/_meta/docs.asciidoc | 108 +++++++++++++++++- .../module/aws/ec2/_meta/docs.asciidoc | 14 +++ .../module/aws/rds/_meta/docs.asciidoc | 14 +++ .../aws/s3_daily_storage/_meta/docs.asciidoc | 16 +++ .../module/aws/s3_request/_meta/docs.asciidoc | 16 +++ .../module/aws/sqs/_meta/docs.asciidoc | 16 +++ x-pack/metricbeat/modules.d/aws.yml.disabled | 39 +------ 12 files changed, 243 insertions(+), 187 deletions(-) create mode 100644 x-pack/metricbeat/module/aws/_meta/config.reference.yml diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 7c63cc1bfe28..d1c81facca2d 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -50,55 +50,6 @@ The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] -The aws.yml used for collecting metrics to display on the predefined aws -dashboard is: -[source,yaml] ----- -- module: aws - period: 300s - metricsets: - - ec2 - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' -- module: aws - period: 300s - metricsets: - - sqs - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - -- module: aws - period: 86400s - metricsets: - - s3_request - - s3_daily_storage - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - # regions: - # - us-west-1 - # - us-east-1 -- module: aws - period: 300s - metricsets: - - cloudwatch - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - cloudwatch_metrics: - - namespace: AWS/EC2 - - namespace: AWS/EBS - - namespace: AWS/Lambda - - namespace: AWS/ECS - - namespace: AWS/ELB ----- - [float] == Metricsets @@ -184,8 +135,8 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 + regions: + - us-west-1 - module: aws period: 86400s metricsets: @@ -195,8 +146,6 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 - module: aws period: 300s metricsets: @@ -214,9 +163,6 @@ metricbeat.modules: - namespace: AWS/EBS - namespace: AWS/ELB tags.resource_type_filter: elasticloadbalancing - #regions: - # - us-east-1 - # - us-east-2 - module: aws period: 60s metricsets: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index a9222ac900d8..f62f751fb38d 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -173,8 +173,8 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 + regions: + - us-west-1 - module: aws period: 86400s metricsets: @@ -184,8 +184,6 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 - module: aws period: 300s metricsets: @@ -203,9 +201,6 @@ metricbeat.modules: - namespace: AWS/EBS - namespace: AWS/ELB tags.resource_type_filter: elasticloadbalancing - #regions: - # - us-east-1 - # - us-east-2 - module: aws period: 60s metricsets: diff --git a/x-pack/metricbeat/module/aws/_meta/config.reference.yml b/x-pack/metricbeat/module/aws/_meta/config.reference.yml new file mode 100644 index 000000000000..fabd35a194a2 --- /dev/null +++ b/x-pack/metricbeat/module/aws/_meta/config.reference.yml @@ -0,0 +1,52 @@ +- module: aws + period: 300s + metricsets: + - ec2 + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - sqs + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + regions: + - us-west-1 +- module: aws + period: 86400s + metricsets: + - s3_request + - s3_daily_storage + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS + - namespace: AWS/ELB + tags.resource_type_filter: elasticloadbalancing +- module: aws + period: 60s + metricsets: + - rds + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index a67d931ca8b1..7ab020b552dc 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -6,27 +6,8 @@ secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' -- module: aws - period: 300s - metricsets: - - sqs - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 -- module: aws - period: 86400s - metricsets: - - s3_request - - s3_daily_storage - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 + regions: + - us-east-1 - module: aws period: 300s metricsets: @@ -36,22 +17,6 @@ session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' cloudwatch_metrics: - - namespace: AWS/EC2 - metricname: CPUUtilization - dimensions: - - name: InstanceId - value: i-0686946e22cf9494a - namespace: AWS/EBS - namespace: AWS/ELB tags.resource_type_filter: elasticloadbalancing - #regions: - # - us-east-1 - # - us-east-2 -- module: aws - period: 60s - metricsets: - - rds - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index b0892d2288aa..21ea39cba998 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -43,55 +43,6 @@ The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] -The aws.yml used for collecting metrics to display on the predefined aws -dashboard is: -[source,yaml] ----- -- module: aws - period: 300s - metricsets: - - ec2 - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' -- module: aws - period: 300s - metricsets: - - sqs - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - -- module: aws - period: 86400s - metricsets: - - s3_request - - s3_daily_storage - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - # regions: - # - us-west-1 - # - us-east-1 -- module: aws - period: 300s - metricsets: - - cloudwatch - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - cloudwatch_metrics: - - namespace: AWS/EC2 - - namespace: AWS/EBS - - namespace: AWS/Lambda - - namespace: AWS/ECS - - namespace: AWS/ELB ----- - [float] == Metricsets diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc index a5010636315c..c5c1d0dbe683 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -13,7 +13,7 @@ tag:getResources ---- [float] -=== Module-specific configuration notes +=== Metricset-specific configuration notes * *namespace*: The namespace to filter against. For example, AWS/EC2, AWS/S3. * *metricname*: The name of the metric to filter against. For example, CPUUtilization for EC2 instance. * *dimensions*: The dimensions to filter against. For example, InstanceId=i-123. @@ -22,3 +22,109 @@ The format of each resource type is service[:resourceType]. For example, specifying a resource type of ec2 returns all Amazon EC2 resources (which includes EC2 instances). Specifying a resource type of ec2:instance returns only EC2 instances. + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EBS <1> + - namespace: AWS/ELB <2> + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/EC2 <3> + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a +---- + +<1> Users can configure the `cloudwatch` metricset to collect all metrics from one +specific namespace, such as `AWS/EBS`. + +<2> `cloudwatch` metricset also has the ability to collect tags from AWS resources. +If user specify `tags.resource_type_filter`, then tags will be collected and stored +as a part of the event. Please see https://docs.aws.amazon.com/resourcegroupstagging/latest/APIReference/API_GetResources.html[AWS API GetResources] +for more details about `tags.resource_type_filter`. + +<3> If users knows exactly what are the cloudwatch metrics they want to collect, +this configuration format can be used. `namespace` and `metricname` need to be +specified and `dimensions` can be used to filter cloudwatch metrics. Please see +https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/list-metrics.html[AWS List Metrics] +for more details. + +[float] +=== More examples +With the configuration below, users will be able to collect cloudwatch metrics +from EBS, ELB and EC2 without tag information. + +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EBS + - namespace: AWS/ELB + - namespace: AWS/EC2 +---- + +With the configuration below, users will be able to collect cloudwatch metrics +from EBS, ELB and EC2 with tags from these services. + +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EBS + tags.resource_type_filter: ebs + - namespace: AWS/ELB + tags.resource_type_filter: elasticloadbalancing + - namespace: AWS/EC2 + tags.resource_type_filter: ec2:instance +---- + +With the configuration below, users will be able to collect specific cloudwatch +metrics. For example CPUUtilization metric from EC2 instance i-123 and NetworkIn +metric from EC2 instance i-456. +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-123 + - namespace: AWS/EC2 + metricname: NetworkIn + dimensions: + - name: InstanceId + value: i-456 +---- diff --git a/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc index 4490ac688e49..f2715bd444fc 100644 --- a/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/ec2/_meta/docs.asciidoc @@ -46,3 +46,17 @@ cloudwatch:ListMetrics The aws ec2 metricset comes with a predefined dashboard. For example: image::./images/metricbeat-aws-ec2-overview.png[] + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - ec2 + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' +---- diff --git a/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc index 68857b443492..b70dacd66614 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc @@ -15,3 +15,17 @@ rds:DescribeDBInstances === Dashboard The aws rds metricset comes with a predefined dashboard. + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 60s + metricsets: + - rds + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' +---- diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc index c12d1b6a3432..7df5d646c298 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc @@ -17,3 +17,19 @@ cloudwatch:ListMetrics The aws s3_daily_storage metricset and s3_request metricset shares one predefined dashboard. For example: image::./images/metricbeat-aws-s3-overview.png[] + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 86400s + metricsets: + - s3_daily_storage + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + #regions: + # - us-west-1 +---- diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc index 82e399416141..898ad53a8f1d 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc @@ -17,3 +17,19 @@ cloudwatch:ListMetrics The aws s3_request metricset and s3_daily_storage metricset shares one predefined dashboard. For example: image::./images/metricbeat-aws-s3-overview.png[] + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 86400s + metricsets: + - s3_request + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + #regions: + # - us-west-1 +---- diff --git a/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc index 9b6f0374014d..976d669bda64 100644 --- a/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/sqs/_meta/docs.asciidoc @@ -18,3 +18,19 @@ sqs:ListQueues The aws sqs metricset comes with a predefined dashboard. For example: image::./images/metricbeat-aws-sqs-overview.png[] + +[float] +=== Configuration example +[source,yaml] +---- +- module: aws + period: 300s + metricsets: + - sqs + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + #regions: + # - us-west-1 +---- diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 21c9a7adf3e3..699f4d8942c3 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -9,27 +9,8 @@ secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' -- module: aws - period: 300s - metricsets: - - sqs - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 -- module: aws - period: 86400s - metricsets: - - s3_request - - s3_daily_storage - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - #regions: - # - us-west-1 + regions: + - us-east-1 - module: aws period: 300s metricsets: @@ -39,22 +20,6 @@ session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' cloudwatch_metrics: - - namespace: AWS/EC2 - metricname: CPUUtilization - dimensions: - - name: InstanceId - value: i-0686946e22cf9494a - namespace: AWS/EBS - namespace: AWS/ELB tags.resource_type_filter: elasticloadbalancing - #regions: - # - us-east-1 - # - us-east-2 -- module: aws - period: 60s - metricsets: - - rds - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}'