From 6f41cf80d72c4a385c7129a9125a0133f92d7baa Mon Sep 17 00:00:00 2001 From: "Mateusz \"mat\" Rumian" <58699800+mat-rumian@users.noreply.github.com> Date: Mon, 15 Mar 2021 11:01:03 +0100 Subject: [PATCH] AWS OTel Collector templates (#788) * Add AWS OTel Collector config templates --- examples/README.md | 27 +- .../non-kubernetes/aws-otel-config-file.yaml | 32 +++ .../aws-otel-ec2-deployment.yaml | 242 ++++++++++++++++++ .../aws-otel-ecs-ec2-deployment.yaml | 127 +++++++++ .../aws-otel-ecs-fargate-deployment.yaml | 138 ++++++++++ 5 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 examples/non-kubernetes/aws-otel-config-file.yaml create mode 100644 examples/non-kubernetes/aws-otel-ec2-deployment.yaml create mode 100644 examples/non-kubernetes/aws-otel-ecs-ec2-deployment.yaml create mode 100644 examples/non-kubernetes/aws-otel-ecs-fargate-deployment.yaml diff --git a/examples/README.md b/examples/README.md index 968efb02956d..10e7e1b8b461 100644 --- a/examples/README.md +++ b/examples/README.md @@ -42,4 +42,29 @@ the configuration also includes cascading filter config, which is described in more detail in [trace filtering documentation](https://help.sumologic.com/Traces/Getting_Started_with_Transaction_Tracing/What_if_I_don't_want_to_send_all_the_tracing_data_to_Sumo_Logic%3F). Please refer to [relevant documentation](https://help.sumologic.com/Traces/Getting_Started_with_Transaction_Tracing/Set_up_traces_collection_for_other_environments) -for more details. \ No newline at end of file +for more details. + +### AWS OTel Collector configuration file +[non-kubernetes/aws-otel-config.yaml](non-kubernetes/aws-otel-config.yaml) contains +an AWS OpenTelemetry Collector distrubtion YAML file which includes configuration +for OpenTelemetry Collector. Should be deployed on the AWS environments. + +### AWS OTel Collector for ECS in EC2 mode template +[non-kubernetes/aws-otel-ecs-ec2-deployment.yaml](non-kubernetes/aws-otel-ecs-ec2-deployment.yaml) contains +an AWS OpenTelemetry Collector distribution YAML file which includes +CloudFormation template. It should be deployed on the AWS ECS EC2 +environment. + +### AWS OTel Collector for ECS in Fargate mode template +[non-kubernetes/aws-otel-ecs-fargate-deployment.yaml](non-kubernetes/aws-otel-ecs-fargate-deployment.yaml) contains +an AWS OpenTelemetry Collector distribution YAML file which includes +CloudFormation template. It should be deployed on the AWS ECS Fargate +environment. + +### AWS OTel Collector for EC2 deployment template +[non-kubernetes/aws-otel-ec2-deployment.yaml](non-kubernetes/aws-otel-ec2-deployment.yaml) contains +an AWS OpenTelemetry Collector distribution YAML file which includes +CloudFormation template. It should be deployed on the AWS EC2. +======= +for more details. +>>>>>>> main diff --git a/examples/non-kubernetes/aws-otel-config-file.yaml b/examples/non-kubernetes/aws-otel-config-file.yaml new file mode 100644 index 000000000000..063c07d90a87 --- /dev/null +++ b/examples/non-kubernetes/aws-otel-config-file.yaml @@ -0,0 +1,32 @@ +extensions: + health_check: +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:55681 + awsxray: + endpoint: 0.0.0.0:2000 + transport: udp +processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s +exporters: + otlphttp: + endpoint: $SUMO_HTTP_TRACES_URL +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp,awsxray] + processors: [batch/traces] + exporters: [otlphttp] + metrics: + receivers: [otlp] + processors: [batch/metrics] + exporters: [otlphttp] diff --git a/examples/non-kubernetes/aws-otel-ec2-deployment.yaml b/examples/non-kubernetes/aws-otel-ec2-deployment.yaml new file mode 100644 index 000000000000..6cfef8fcec26 --- /dev/null +++ b/examples/non-kubernetes/aws-otel-ec2-deployment.yaml @@ -0,0 +1,242 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Template to install AWS OTel Collector on EC2 - Amazon Linux.' +Parameters: + SSHKeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance + Type: AWS::EC2::KeyPair::KeyName + ConstraintDescription: must be the name of an existing EC2 KeyPair. + InstanceType: + Description: EC2 instance type + Type: String + Default: m4.2xlarge + ConstraintDescription: must be a valid EC2 instance type. + InstanceAMI: + Description: Managed AMI ID for EC2 Instance + Type : String + IAMRole: + Description: EC2 attached IAM role + Type: String + Default: SumologicAWSOTelColRoleEC2 + ConstraintDescription: must be an existing IAM role which will be attached to EC2 instance. + IAMPolicy: + Description: IAM Role attached IAM Managed Policy + Type: String + Default: SumologicAWSOTelColPolicyEC2 + ConstraintDescription: Must be an existing IAM Managed Policy which will be attached to IAM Role. + IAMInstanceProfileName: + Description: IAM Role attached IAM Instance Profile + Type: String + Default: SumologicAWSOTelColRoleEC2 + ConstraintDescription: Must be an existing IAM Instance Profile which will be attached to IAM Role. + SumoHttpTracesURL: + Type: String + Description: Enther the Sumologic HTTP Traces Endpoint URL +Resources: + EC2Instance: + Type: AWS::EC2::Instance + Metadata: + AWS::CloudFormation::Init: + configSets: + default: + - 01_setupCfnHup + - 02_config-aws-otel-collector + - 03_restart-aws-otel-collector + UpdateEnvironment: + - 02_config-aws-otel-collector + - 03_restart-aws-otel-collector + # Definition of YAML configuration of aws-otel-collector, you can change the configuration below. + 02_config-aws-otel-collector: + files: + '/opt/aws/aws-otel-collector/etc/config.yaml': + content: !Sub + - | + extensions: + health_check: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:55681 + awsxray: + endpoint: 0.0.0.0:2000 + transport: udp + processors: + batch/traces: + timeout: 1s + send_batch_size: 50 + batch/metrics: + timeout: 60s + exporters: + otlphttp: + endpoint: ${sumo_http_traces_url} + service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp,awsxray] + processors: [batch/traces] + exporters: [otlphttp] + metrics: + receivers: [otlp] + processors: [batch/metrics] + exporters: [otlphttp] + - sumo_http_traces_url: !Ref SumoHttpTracesURL + # Invoke aws-otel-collector-ctl to restart aws-otel-collector. + 03_restart-aws-otel-collector: + commands: + 01_stop_service: + command: sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -a stop + 02_start_service: + command: sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -a start + # Cfn-hup setting, it is to monitor the change of metadata. + # When there is change in the contents of json file in the metadata section, cfn-hup will call cfn-init to restart aws-otel-collector. + 01_setupCfnHup: + files: + '/etc/cfn/cfn-hup.conf': + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + interval=1 + mode: '000400' + owner: root + group: root + '/etc/cfn/hooks.d/aws-otel-collector-auto-reloader.conf': + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.EC2Instance.Metadata.AWS::CloudFormation::Init.02_config-aws-otel-collector + action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets UpdateEnvironment + runas=root + mode: '000400' + owner: root + group: root + "/lib/systemd/system/cfn-hup.service": + content: !Sub | + [Unit] + Description=cfn-hup daemon + [Service] + Type=simple + ExecStart=/opt/aws/bin/cfn-hup + Restart=always + [Install] + WantedBy=multi-user.target + commands: + 01enable_cfn_hup: + command: !Sub | + systemctl enable cfn-hup.service + 02start_cfn_hup: + command: !Sub | + systemctl start cfn-hup.service + + Properties: + InstanceType: + Ref: InstanceType + IamInstanceProfile: + Ref: IAMRole + KeyName: + Ref: SSHKeyName + ImageId: + Ref: InstanceAMI + SecurityGroups: + - Ref: InstanceSecurityGroup + Tags: + - Key: Name + Value: sumologic-aws-otel-col-ec2 + UserData: + # This script below is to install aws-otel-collector, restart aws-otel-collector and tell the result to cloudformation. + Fn::Base64: !Sub + - | + #!/bin/bash + + # Download AWS OTel Collector RPM + sudo rpm -Uvh https://aws-otel-collector.s3.amazonaws.com/amazon_linux/amd64/latest/aws-otel-collector.rpm + + # Setup Sumologic HTTP Traces URL ENV + echo "export SUMO_HTTP_TRACES_URL=${sumo_http_traces_url}" > /etc/profile.d/setSumoVar.sh + + /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + - sumo_http_traces_url: !Ref SumoHttpTracesURL + DependsOn: + - EC2Role + - IAMInstanceProfile + - InstanceSecurityGroup + + IAMInstanceProfile: + Type: 'AWS::IAM::InstanceProfile' + Properties: + InstanceProfileName: !Ref IAMInstanceProfileName + Path: / + Roles: + - !Ref IAMRole + DependsOn: EC2Role + + EC2Role: + Type: 'AWS::IAM::Role' + Properties: + Description: Allows EC2 to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: 'sts:AssumeRole' + ManagedPolicyArns: + - !Sub 'arn:aws:iam::${AWS::AccountId}:policy/${IAMPolicy}' + RoleName: !Ref IAMRole + DependsOn: EC2Policy + + EC2Policy: + Type: 'AWS::IAM::ManagedPolicy' + Properties: + Description: Allows EC2 to call AWS services on your behalf. + Path: / + ManagedPolicyName: !Ref IAMPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - logs:PutLogEvents + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DescribeLogStreams + - logs:DescribeLogGroups + - xray:PutTraceSegments + - xray:PutTelemetryRecords + - xray:GetSamplingRules + - xray:GetSamplingTargets + - xray:GetSamplingStatisticSummaries + - ssm:GetParameters + Resource: '*' + + InstanceSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Enable SSH access via port 22 + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 4317 + ToPort: 4317 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 55680 + ToPort: 55680 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 55681 + ToPort: 55681 + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: 2000 + ToPort: 2000 + CidrIp: 0.0.0.0/0 diff --git a/examples/non-kubernetes/aws-otel-ecs-ec2-deployment.yaml b/examples/non-kubernetes/aws-otel-ecs-ec2-deployment.yaml new file mode 100644 index 000000000000..31e593fa76c9 --- /dev/null +++ b/examples/non-kubernetes/aws-otel-ecs-ec2-deployment.yaml @@ -0,0 +1,127 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: 'Template to install AWS OTel Collector on ECS in EC2 mode' +Parameters: + IAMTaskRole: + Description: Task attached IAM role + Type: String + Default: SumologicAWSOTelColTaskRoleECSEC2 + ConstraintDescription: must be an existing IAM role which will be attached to EC2 instance. + IAMExecutionRole: + Description: Task Execution attached IAM role + Type: String + Default: SumologicAWSOTelColExecutionRoleECSEC2 + ConstraintDescription: must be an existing IAM role which will be attached to EC2 instance. + IAMPolicy: + Description: IAM Role attached IAM Policy + Type: String + Default: SumologicAWSOTelColPolicyECSEC2 + ConstraintDescription: Must be an existing IAM Managed Policy which will be attached to IAM Role. + ClusterName: + Type: String + Description: Enter the name of your ECS cluster from which you want to collect telemetry data + SumoHttpTracesURL: + Type: String + Description: Enther the Sumologic HTTP Traces Endpoint URL + SumoAWSOTelColConfig: + Type: AWS::SSM::Parameter::Value + Default: sumologic-otel-col-config + Description: AWS SSM Parameter which contains OTel Collector config file +Resources: + ECSTaskDefinition: + Type: 'AWS::ECS::TaskDefinition' + Properties: + Family: sumologic-aws-otel-collector-ec2 + TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/${IAMTaskRole}' + ExecutionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/${IAMExecutionRole}' + ContainerDefinitions: + - logConfiguration: + logDriver: awslogs + options: + awslogs-create-group: 'True' + awslogs-group: /ecs/aws-otel-collector + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ecs + portMappings: + - hostPort: 2000 + protocol: udp + containerPort: 2000 + - hostPort: 4317 + protocol: tcp + containerPort: 4317 + - hostPort: 55681 + protocol: tcp + containerPort: 55681 + environment: + - name: SUMO_HTTP_TRACES_URL + value: !Ref SumoHttpTracesURL + - name: AOT_CONFIG_CONTENT + value: !Ref SumoAWSOTelColConfig + image: amazon/aws-otel-collector:latest + name: sumologic-aws-otel-collector + RequiresCompatibilities: + - EC2 + Cpu: 1024 + Memory: 2048 + DependsOn: + - ECSTaskRole + - ECSExecutionRole + ECSReplicaService: + Type: 'AWS::ECS::Service' + Properties: + TaskDefinition: !Ref ECSTaskDefinition + Cluster: !Ref ClusterName + LaunchType: EC2 + SchedulingStrategy: REPLICA + DesiredCount: 1 + ServiceName: sumologic-aws-otel-col-svc-ecs-ec2 + ECSTaskRole: + Type: 'AWS::IAM::Role' + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: !Ref IAMPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:PutLogEvents' + - 'logs:CreateLogGroup' + - 'logs:CreateLogStream' + - 'logs:DescribeLogStreams' + - 'logs:DescribeLogGroups' + - 'xray:PutTraceSegments' + - 'xray:PutTelemetryRecords' + - 'xray:GetSamplingRules' + - 'xray:GetSamplingTargets' + - 'xray:GetSamplingStatisticSummaries' + - 'ssm:GetParameters' + Resource: '*' + RoleName: !Ref IAMTaskRole + ECSExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + Description: >- + Allows ECS container agent makes calls to the Amazon ECS API on your + behalf. + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: 'sts:AssumeRole' + ManagedPolicyArns: + - 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy' + - 'arn:aws:iam::aws:policy/CloudWatchLogsFullAccess' + - 'arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess' + RoleName: !Ref IAMExecutionRole diff --git a/examples/non-kubernetes/aws-otel-ecs-fargate-deployment.yaml b/examples/non-kubernetes/aws-otel-ecs-fargate-deployment.yaml new file mode 100644 index 000000000000..2d9359d0b79a --- /dev/null +++ b/examples/non-kubernetes/aws-otel-ecs-fargate-deployment.yaml @@ -0,0 +1,138 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: 'Template to install AWS OTel Collector on ECS in Fargate mode' +Parameters: + IAMTaskRole: + Description: Task attached IAM role + Type: String + Default: SumologicAWSOTelColTaskRoleECSFargate + ConstraintDescription: must be an existing IAM role which will be attached to EC2 instance. + IAMExecutionRole: + Description: Task Execution attached IAM role + Type: String + Default: SumologicAWSOTelColExecutionRoleECSFargate + ConstraintDescription: must be an existing IAM role which will be attached to EC2 instance. + IAMPolicy: + Description: IAM Role attached IAM Policy + Type: String + Default: SumologicAWSOTelColPolicyECSFargate + ConstraintDescription: Must be an existing IAM Managed Policy which will be attached to IAM Role. + ClusterName: + Type: String + Description: Enter the name of your ECS cluster from which you want to collect telemetry data + SecurityGroups: + Type: CommaDelimitedList + Description: The list of SecurityGroupIds in your Virtual Private Cloud (VPC) + Subnets: + Type: CommaDelimitedList + Description: The list of Subnets in your Virtual Private Cloud (VPC) + SumoHttpTracesURL: + Type: String + Description: Enther the Sumologic HTTP Traces Endpoint URL + SumoAWSOTelColConfig: + Type: AWS::SSM::Parameter::Value + Default: sumologic-otel-col-config + Description: AWS SSM Parameter which contains OTel Collector config file +Resources: + ECSTaskDefinition: + Type: 'AWS::ECS::TaskDefinition' + Properties: + Family: sumologic-aws-otel-collector-fargate + TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/${IAMTaskRole}' + ExecutionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/${IAMExecutionRole}' + NetworkMode: awsvpc + ContainerDefinitions: + - LogConfiguration: + LogDriver: awslogs + Options: + awslogs-create-group: 'True' + awslogs-group: /ecs/aws-otel-collector + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ecs + portMappings: + - hostPort: 2000 + protocol: udp + containerPort: 2000 + - hostPort: 4317 + protocol: tcp + containerPort: 4317 + - hostPort: 55681 + protocol: tcp + containerPort: 55681 + environment: + - name: SUMO_HTTP_TRACES_URL + value: !Ref SumoHttpTracesURL + - name: AOT_CONFIG_CONTENT + value: !Ref SumoAWSOTelColConfig + image: amazon/aws-otel-collector:latest + name: sumologic-aws-otel-collector + RequiresCompatibilities: + - FARGATE + Cpu: 1024 + Memory: 2048 + DependsOn: + - ECSTaskRole + - ECSExecutionRole + ECSReplicaService: + Type: 'AWS::ECS::Service' + Properties: + TaskDefinition: !Ref ECSTaskDefinition + Cluster: !Ref ClusterName + LaunchType: FARGATE + SchedulingStrategy: REPLICA + DesiredCount: 1 + ServiceName: sumologic-aws-otel-col-svc-ecs-fargate + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + SecurityGroups: !Ref SecurityGroups + Subnets: !Ref Subnets + ECSTaskRole: + Type: 'AWS::IAM::Role' + Properties: + Description: Allows ECS tasks to call AWS services on your behalf. + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: !Ref IAMPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:PutLogEvents' + - 'logs:CreateLogGroup' + - 'logs:CreateLogStream' + - 'logs:DescribeLogStreams' + - 'logs:DescribeLogGroups' + - 'xray:PutTraceSegments' + - 'xray:PutTelemetryRecords' + - 'xray:GetSamplingRules' + - 'xray:GetSamplingTargets' + - 'xray:GetSamplingStatisticSummaries' + - 'ssm:GetParameters' + Resource: '*' + RoleName: !Ref IAMTaskRole + ECSExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + Description: >- + Allows ECS container agent makes calls to the Amazon ECS API on your + behalf. + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: 'sts:AssumeRole' + ManagedPolicyArns: + - 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy' + - 'arn:aws:iam::aws:policy/CloudWatchLogsFullAccess' + - 'arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess' + RoleName: !Ref IAMExecutionRole