Configurations of OpenTelemetryCollector and AMP Rules for AmpAddOn

.github/workflows/linkcheck.json

@@ -20,6 +20,7 @@
       { "pattern": "https://sqs" },
       { "pattern": "www.rsa-2048.example.com" },
       { "pattern": "rsa-2048.example.com" },
-      { "pattern": "https://ingress-red-saas.instana.io/" }
+      { "pattern": "https://ingress-red-saas.instana.io/" },
+      { "pattern": "http://console-eks.yourdomain.com/" }
     ]
 }

docs/addons/amp-addon.md

@@ -15,7 +15,7 @@ For more information on the add-on, please review the [user guide](https://docs.
 
 This add-on can used with four different patterns :
 
-Pattern # 1 : Simple and Easy - Using all default property values. This pattern creates a new AMP workspace with default property values such as `workspaceName`, `namespace` with no tags on the AMP workspace and deploys an ADOT collector in the `default` namespace with `deployment` as the mode to remote write metrics to AMP workspace. 
+Pattern #1: Simple and Easy - Using all default property values. This pattern creates a new AMP workspace with default property values such as `workspaceName`, `namespace` with no tags on the AMP workspace and deploys an ADOT collector in the `default` namespace with `deployment` as the mode to remote write metrics to AMP workspace. 
 
 ```typescript
 import * as cdk from 'aws-cdk-lib';
@@ -30,7 +30,7 @@ const blueprint = blueprints.EksBlueprint.builder()
   .build(app, 'my-stack-name');
 ```
 
-Pattern # 2 : Overriding property values for Name and Tags for a custom AMP Workspace name and tags. This pattern creates a new AMP workspace with property values passed on such as `workspaceName`, `tags` and deploys an ADOT collector on the namespace specified in `namespace` with name in `name` and `deployment` as the mode to remote write metrics to AMP workspace.
+Pattern #2: Overriding property values for Name and Tags for a custom AMP Workspace name and tags. This pattern creates a new AMP workspace with property values passed on such as `workspaceName`, `tags` and deploys an ADOT collector on the namespace specified in `namespace` with name in `name` and `deployment` as the mode to remote write metrics to AMP workspace.
 
 ```typescript
 import * as cdk from 'aws-cdk-lib';
@@ -66,7 +66,7 @@ const blueprint = blueprints.EksBlueprint.builder()
   }))
   .build(app, 'my-stack-name');
 ```
-Pattern # 3 : Passing on AMP Remote Endpoint of an existing AMP workspace to be used to remote write metrics. This pattern does not create an AMP workspace. Deploys an ADOT collector on the namespace specified in `namespace` with name in `name` and `deployment` as the mode to remote write metrics to AMP workspace of the URL passed as input. This pattern ignores any other property values passed if `ampPrometheusEndpoint` is present.
+Pattern #3: Passing on AMP Remote Endpoint of an existing AMP workspace to be used to remote write metrics. This pattern does not create an AMP workspace. Deploys an ADOT collector on the namespace specified in `namespace` with name in `name` and `deployment` as the mode to remote write metrics to AMP workspace of the URL passed as input. This pattern ignores any other property values passed if `ampPrometheusEndpoint` is present.
 
 ```typescript
 import * as cdk from 'aws-cdk-lib';
@@ -88,7 +88,7 @@ const blueprint = blueprints.EksBlueprint.builder()
   .build(app, 'my-stack-name');
 ```
 
-Pattern # 4 : Overriding property values for different deployment Modes. This pattern creates a new AMP workspace with property values passed on such as `workspaceName`, `tags` and deploys an ADOT collector on the namespace specified in `namespace` with name in `name` and `daemonset` as the mode to remote write metrics to AMP workspace. Deployment modes can be overridden to any of these values - `deployment`, `daemonset`, `statefulset`, `sidecar`.
+Pattern #4: Overriding property values for different deployment Modes. This pattern creates a new AMP workspace with property values passed on such as `workspaceName`, `tags` and deploys an ADOT collector on the namespace specified in `namespace` with name in `name` and `daemonset` as the mode to remote write metrics to AMP workspace. Deployment modes can be overridden to any of these values - `deployment`, `daemonset`, `statefulset`, `sidecar`.
 
 ```typescript
 import * as cdk from 'aws-cdk-lib';
@@ -128,6 +128,101 @@ const blueprint = blueprints.EksBlueprint.builder()
   .build(app, 'my-stack-name');
 ```
 
+Pattern #5: Configuring rules into AMP. This pattern creates recording rules and/or alerting rules, based on the YAML files provided. A workspace ARN is also required as input.
+
+```typescript
+import * as cdk from 'aws-cdk-lib';
+import { CfnWorkspace } from 'aws-cdk-lib/aws-aps';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+
+const app = new cdk.App();
+const ampWorkspaceName = "blueprints-amp-workspace";
+const ampWorkspace = blueprints.getNamedResource(ampWorkspaceName);
+const attrPrometheusEndpoint = ampWorkspace.attrPrometheusEndpoint;
+const ampWorkspaceArn = ampWorkspace.attrArn;
+
+const addOn = new blueprints.addons.AmpAddOn({
+    ampPrometheusEndpoint: attrPrometheusEndpoint,
+    ampRules: {
+        ampWorkspaceArn: ampWorkspaceArn,
+        ruleFilePaths: [
+            __dirname + '/../common/resources/amp-config/alerting-rules.yml',
+            __dirname + '/../common/resources/amp-config/recording-rules.yml'
+        ]
+    }
+})
+
+const blueprint = blueprints.EksBlueprint.builder()
+  .addOns(addOn)
+  .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName))
+  .build(app, 'my-stack-name');
+```
+
+The recording rules and alerting rules files should be placed in the same repository that is using this add-on. They should also follow the same format as rules files in standalone Prometheus, like explained in [the AMP User Guide](https://docs.aws.amazon.com/prometheus/latest/userguide/AMP-ruler-rulesfile.html):
+
+```yaml
+groups:
+  - name: test
+    rules:
+    - record: metric:recording_rule
+      expr: avg(rate(container_cpu_usage_seconds_total[5m]))
+  - name: alert-test
+    rules:
+    - alert: metric:alerting_rule
+      expr: avg(rate(container_cpu_usage_seconds_total[5m])) > 0
+      for: 2m
+```
+
+An example of rules configuration can be found in the data section of the [EKS monitoring rules file](https://github.com/aws-observability/terraform-aws-observability-accelerator/blob/main/modules/eks-monitoring/rules.tf) in the aws-observability repository.
+
+Pattern #6: Configuring the [AWS Distro for OpenTelemetry Collector](https://aws-otel.github.io/docs/getting-started/collector). This pattern enables you to configure the Collector by providing a manifest describing an OpenTelemetryCollector resource:
+
+```yaml
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: sidecar-for-my-app
+spec:
+  mode: sidecar
+  config: |
+    ...
+    {{javaScrapeSampleLimit}}
+    ...
+    {{javaPrometheusMetricsEndpoint}}
+```
+
+This pattern is useful when you need to customize the configuration of the OpenTelemetryCollector, e.g. to add specific scraping targets and parameters in the Prometheus receiver configuration, for a job capturing metrics of a Java workload.
+
+The optional `openTelemetryCollector.manifestParameterMap` parameter allows you to define a map where the keys can be used (in double curly braces) within the OpenTelemetryCollector manifest as e.g. {{javaScrapeSampleLimit}}, to be replaced by the corresponding values.
+
+```typescript
+import * as cdk from 'aws-cdk-lib';
+import { CfnWorkspace } from 'aws-cdk-lib/aws-aps';
+import * as blueprints from '@aws-quickstart/eks-blueprints';
+
+const app = new cdk.App();
+const ampWorkspaceName = "blueprints-amp-workspace";
+const attrPrometheusEndpoint = blueprints.getNamedResource(ampWorkspaceName).attrPrometheusEndpoint;
+
+const addOn = new blueprints.addons.AmpAddOn({
+    ampPrometheusEndpoint: attrPrometheusEndpoint,
+    openTelemetryCollector: {
+        manifestPath: __dirname + '/../common/resources/otel-collector-config.yml',
+        manifestParameterMap: {
+            javaScrapeSampleLimit: 1000,
+            javaPrometheusMetricsEndpoint: "/metrics"
+        }
+    }
+});
+
+const blueprint = blueprints.EksBlueprint.builder()
+  .addOns(addOn)
+  .resourceProvider(ampWorkspaceName, new blueprints.CreateAmpProvider(ampWorkspaceName, ampWorkspaceName))
+  .build(app, 'my-stack-name');
+```
+
+If you need to configure rules, please do not use the rule_files field like in standalone Prometheus, but rather use the ampRules parameter.
+
 ## Validation
 
 To validate that AMP add-on is installed properly, ensure that the required kubernetes resources are running in the cluster:

lib/addons/amp/index.ts

@@ -3,6 +3,8 @@ import { dependable, loadYaml, readYamlDocument } from "../../utils";
 import { AdotCollectorAddOn } from "../adot";
 import { Construct } from 'constructs';
 import { KubectlProvider, ManifestDeployment } from "../helm-addon/kubectl-provider";
+import { CfnRuleGroupsNamespace } from "aws-cdk-lib/aws-aps";
+import { ICluster } from "aws-cdk-lib/aws-eks";
 
 /**
  * This AMP add-on installs an ADOT Collector for Amazon Managed Service for Prometheus 
@@ -11,6 +13,33 @@ import { KubectlProvider, ManifestDeployment } from "../helm-addon/kubectl-provi
  * to AMP remote write endpoint of the created or passed AMP workspace.
  */
 
+export interface AmpRules {
+    /** 
+     * AMP workspace ARN.
+     */
+    ampWorkspaceArn: string;
+
+    /** 
+     * Paths of the files listing the AMP rules.
+     */
+    ruleFilePaths: string[];
+}
+
+export interface OpenTelemetryCollector {
+     /**
+     * An alternative OpenTelemetryCollector if you need further cusomisation.
+     * If not provided, the default will be used.
+     */
+     manifestPath: string;
+     /**
+     * This parameter is optional and holds an object of type Values, with keys and values.
+     * The same key literals will need to be used within the manifest between double curly braces {{}} and the add-on will replace them with the related values.
+     * The following keys are already in use by the add-on and will be replaced in your manifest with configured values, if you want to use them:
+     * remoteWriteEndpoint, awsRegion, deploymentMode, namespace, clusterName. To use any of those you can just include e.g. {{remoteWriteEndpoint}} inside the manifest.
+     */
+     manifestParameterMap?: Values;
+}
+
 /**
  * Configuration options for add-on.
  */
@@ -24,17 +53,28 @@ export interface AmpAddOnProps {
      * Modes supported : `deployment`, `daemonset`, `statefulSet`, and `sidecar`
      * @default deployment
      */
-     deploymentMode?: DeploymentMode;
+    deploymentMode?: DeploymentMode;
     /**
      * Namespace to deploy the ADOT Collector for AMP.
      * @default default
      */
-     namespace?: string;
+    namespace?: string;
     /**
      * Name for deployment of the ADOT Collector for AMP.
      * @default 'adot-collector-amp'
      */
-     name?: string;
+    name?: string;
+    /** 
+     * AMP rules providing AMP workspace ARN and paths to files encoding recording and/or alerting rules following the same format as a rules file in standalone Prometheus.
+     * This parameter is optional and if not provided, no rules will be applied.
+     */
+    ampRules?: AmpRules;
+     /**
+     * An alternative OpenTelemetryCollector if you need further customisation.
+     * If you need to configure rules, please do not use the rule_files field like in standalone Prometheus, but rather use the ampRules parameter.
+     * If not provided, the default will be used.
+     */
+    openTelemetryCollector?: OpenTelemetryCollector;
 }
 
 export const enum DeploymentMode {
@@ -77,14 +117,23 @@ export class AmpAddOn implements ClusterAddOn {
             doc = readYamlDocument(__dirname + '/collector-config-amp.ytpl');
         }
 
-        const manifest = doc.split("---").map(e => loadYaml(e));
+        const manifest = doc.split("---").map(e => {
+            let object = loadYaml(e);
+
+            if (this.ampAddOnProps.openTelemetryCollector?.manifestPath !== undefined && object.kind === "OpenTelemetryCollector"){
+                object = readYamlDocument(this.ampAddOnProps.openTelemetryCollector.manifestPath!);
+                object = loadYaml(object);
+            }
+            return object;
+        });
         const attrPrometheusEndpoint = this.ampAddOnProps.ampPrometheusEndpoint + 'api/v1/remote_write';
         const values: Values = {
             remoteWriteEndpoint: attrPrometheusEndpoint,
             awsRegion: cluster.stack.region,
             deploymentMode: this.ampAddOnProps.deploymentMode,
             namespace: this.ampAddOnProps.namespace,
-            clusterName: cluster.clusterName
+            clusterName: cluster.clusterName,
+            ...this.ampAddOnProps.openTelemetryCollector?.manifestParameterMap
          };
 
          const manifestDeployment: ManifestDeployment = {
@@ -93,10 +142,37 @@ export class AmpAddOn implements ClusterAddOn {
             manifest,
             values
         };
-
         const kubectlProvider = new KubectlProvider(clusterInfo);
         const statement = kubectlProvider.addManifest(manifestDeployment);
 
+        const ampRules = this.ampAddOnProps.ampRules;
+        if (ampRules !== undefined){
+            const ruleGroupsNamespaces = this.configureRules(cluster, ampRules.ruleFilePaths, ampRules.ampWorkspaceArn);
+            statement.node.addDependency(ruleGroupsNamespaces.at(-1)!);
+        }
+
         return Promise.resolve(statement);
     }
+
+    private configureRules(cluster: ICluster, ruleFilePaths: string[], ampWorkspaceArn: string): CfnRuleGroupsNamespace[] {
+        const ruleGroupsNamespaces: CfnRuleGroupsNamespace[] = [];
+
+        if (ruleFilePaths.length == 0) {
+            throw new Error("No paths defined for AMP rules");
+        }
+
+        ruleFilePaths.map((ruleFilePath, index) => {
+            const ruleGroupsNamespace = new CfnRuleGroupsNamespace(cluster, "AmpRulesConfigurator-" + index, {
+                data: readYamlDocument(ruleFilePath),
+                name: "AmpRulesConfigurator-" + index,
+                workspace: ampWorkspaceArn
+            });
+            if (index > 0){
+                ruleGroupsNamespace.node.addDependency(ruleGroupsNamespaces.at(-1)!);
+            }
+            ruleGroupsNamespaces.push(ruleGroupsNamespace);
+        });
+
+        return ruleGroupsNamespaces;
+    }
 }

test/amp.test.ts

@@ -0,0 +1,54 @@
+import * as cdk from 'aws-cdk-lib';
+import * as blueprints from '../lib';
+
+describe('Unit tests for AMP addon', () => {
+
+    test("Stack creation succeeds", () => {
+        const app = new cdk.App();
+
+        const blueprint = blueprints.EksBlueprint.builder();
+
+        blueprint.account("123567891").region('us-west-1')
+        .addOns(new blueprints.addons.AwsLoadBalancerControllerAddOn())
+        .addOns(new blueprints.addons.CertManagerAddOn())
+        .addOns(new blueprints.addons.AdotCollectorAddOn())
+        .addOns(
+            new blueprints.addons.AmpAddOn({
+                ampPrometheusEndpoint: "test",
+                ampRules: {
+                    ampWorkspaceArn: "test",
+                    ruleFilePaths: [
+                        __dirname + "/resources/recording-rules-test.yml",
+                    ]
+                }
+            })
+        )
+        .build(app, 'amp-addon-stack-succeeds');
+
+        expect(blueprint).toBeDefined();
+    });
+
+    test("Stack creation fails due to ruleFilePaths.length == 0", () => {
+        const app = new cdk.App();
+
+        const blueprint = blueprints.EksBlueprint.builder();
+
+        blueprint.account("123567891").region('us-west-1')
+        .addOns(new blueprints.addons.AwsLoadBalancerControllerAddOn())
+        .addOns(new blueprints.addons.CertManagerAddOn())
+        .addOns(new blueprints.addons.AdotCollectorAddOn())
+        .addOns(
+            new blueprints.addons.AmpAddOn({
+                ampPrometheusEndpoint: "test",
+                ampRules: {
+                    ampWorkspaceArn: "test",
+                    ruleFilePaths: []
+                }
+            })
+        );
+
+        expect(()=> {
+            blueprint.build(app, "amp-missing-rules");
+        }).toThrow("No paths defined for AMP rules");    
+    });
+});

test/resources/recording-rules-test.yml

@@ -0,0 +1,5 @@
+groups:
+  - name: infra-rules-01
+    rules:
+      - record: "node_namespace_pod:kube_pod_info:"
+        expr: topk by(cluster, namespace, pod) (1, max by(cluster, node, namespace, pod) (label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")))