diff --git a/managed/devops/yb-otel-collector.yml b/managed/devops/yb-otel-collector.yml index 5bb0a6ff929..bb6e2061e41 100644 --- a/managed/devops/yb-otel-collector.yml +++ b/managed/devops/yb-otel-collector.yml @@ -8,7 +8,7 @@ - name: Manage OpenTelemetry Collector hosts: "{{ yb_prebuilt_ami_host | default(instance_search_pattern) }}" - become: "{{ 'yes' if use_sudo|default(false) else '' }}" + become: "{{ 'yes' if use_sudo|default(false) else 'no' }}" become_method: "{{ 'sudo' if use_sudo|default(false) else '' }}" roles: - "manage_otel_collector" diff --git a/managed/src/main/java/com/yugabyte/yw/common/audit/otel/AuditLogRegexGenerator.java b/managed/src/main/java/com/yugabyte/yw/common/audit/otel/AuditLogRegexGenerator.java index 41fb0dff9e7..cbbb0ea5092 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/audit/otel/AuditLogRegexGenerator.java +++ b/managed/src/main/java/com/yugabyte/yw/common/audit/otel/AuditLogRegexGenerator.java @@ -141,5 +141,9 @@ public enum LogPrefixTokens { public String getAttributeName() { return name().toLowerCase(); } + + public String getYugabyteAttributeName() { + return "yugabyte." + name().toLowerCase(); + } } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigFormat.java b/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigFormat.java index 3dd2b1bd453..c92ea71b247 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigFormat.java +++ b/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigFormat.java @@ -104,6 +104,7 @@ public static class AttributeAction { private String key; private String value; private String action; + private String from_attribute; } @Data diff --git a/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGenerator.java b/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGenerator.java index 8b54339982d..3609355774b 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGenerator.java +++ b/managed/src/main/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGenerator.java @@ -8,8 +8,11 @@ import com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat.MultilineConfig; import com.yugabyte.yw.common.yaml.SkipNullRepresenter; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; +import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.Provider; import com.yugabyte.yw.models.TelemetryProvider; +import com.yugabyte.yw.models.Universe; +import com.yugabyte.yw.models.helpers.NodeDetails; import com.yugabyte.yw.models.helpers.TelemetryProviderService; import com.yugabyte.yw.models.helpers.audit.AuditLogConfig; import com.yugabyte.yw.models.helpers.audit.UniverseLogsExporterConfig; @@ -24,8 +27,11 @@ import java.util.*; import javax.inject.Inject; import javax.inject.Singleton; +import lombok.AllArgsConstructor; +import lombok.Data; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.yaml.snakeyaml.Yaml; import play.Environment; @@ -71,6 +77,8 @@ void generateConfigFile( String logLinePrefix, Path path, int otelColMetricsPort) { + Customer customer = Customer.getOrBadRequest(provider.getCustomerUUID()); + Universe universe = Universe.getOrBadRequest(nodeParams.getUniverseUUID()); try (BufferedWriter writer = new BufferedWriter(new FileWriter(path.toFile()))) { Yaml yaml = new Yaml(new SkipNullRepresenter()); OtelCollectorConfigFormat collectorConfigFormat = new OtelCollectorConfigFormat(); @@ -118,7 +126,13 @@ void generateConfigFile( .forEach( config -> appendExporter( - collectorConfigFormat, config, currentProcessors, nodeParams.nodeName)); + customer, + universe, + collectorConfigFormat, + config, + currentProcessors, + nodeParams.nodeName, + logLinePrefix)); } yaml.dump(collectorConfigFormat, writer); @@ -240,15 +254,36 @@ private OtelCollectorConfigFormat.FileLogReceiver createFileLogReceiver( receiver.setStart_at("beginning"); receiver.setStorage("file_storage/queue"); receiver.setOperators(operators); - receiver.setAttributes(ImmutableMap.of("audit_log_type", logType)); + receiver.setAttributes(ImmutableMap.of("yugabyte.audit_log_type", logType)); return receiver; } + @Data + @AllArgsConstructor + public static class RenamePair { + private String before; + private String after; + + private List getRenameAttributeActions() { + List renameActionsList = new ArrayList<>(); + // Copy the attribute from existing attribute and delete the original one. + renameActionsList.add( + new OtelCollectorConfigFormat.AttributeAction(this.after, null, "upsert", this.before)); + renameActionsList.add( + new OtelCollectorConfigFormat.AttributeAction(this.before, null, "delete", null)); + return renameActionsList; + } + } + private void appendExporter( + Customer customer, + Universe universe, OtelCollectorConfigFormat collectorConfig, UniverseLogsExporterConfig logsExporterConfig, List currentProcessors, - String nodeName) { + String nodeName, + String logLinePrefix) { + NodeDetails nodeDetails = universe.getNode(nodeName); TelemetryProvider telemetryProvider = telemetryProviderService.getOrBadRequest(logsExporterConfig.getExporterUuid()); Map exporters = collectorConfig.getExporters(); @@ -271,10 +306,10 @@ private void appendExporter( // Add Datadog specific labels. attributeActions.add( - new OtelCollectorConfigFormat.AttributeAction("ddsource", "yugabyte", "upsert")); + new OtelCollectorConfigFormat.AttributeAction("ddsource", "yugabyte", "upsert", null)); attributeActions.add( new OtelCollectorConfigFormat.AttributeAction( - "service", "yb-otel-collector", "upsert")); + "service", "yb-otel-collector", "upsert", null)); break; case SPLUNK: SplunkConfig splunkConfig = (SplunkConfig) telemetryProvider.getConfig(); @@ -327,7 +362,67 @@ private void appendExporter( } // Add some common collector labels. - attributeActions.add(new OtelCollectorConfigFormat.AttributeAction("host", nodeName, "upsert")); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction("host", nodeName, "upsert", null)); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction( + "yugabyte.cloud", + StringUtils.defaultString(nodeDetails.cloudInfo.cloud, ""), + "upsert", + null)); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction( + "yugabyte.universe_uuid", universe.getUniverseUUID().toString(), "upsert", null)); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction( + "yugabyte.node_type", + universe.getCluster(nodeDetails.placementUuid).clusterType.toString(), + "upsert", + null)); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction( + "yugabyte.region", + StringUtils.defaultString(nodeDetails.cloudInfo.region, ""), + "upsert", + null)); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction( + "yugabyte.zone", + StringUtils.defaultString(nodeDetails.cloudInfo.az, ""), + "upsert", + null)); + attributeActions.add( + new OtelCollectorConfigFormat.AttributeAction( + "yugabyte.purpose", + telemetryProvider.getConfig().getType().toString() + "_LOG_EXPORT", + "upsert", + null)); + + // Rename the attributes to organise under the key yugabyte. + List renamePairs = new ArrayList(); + renamePairs.add(new RenamePair("log.file.name", "yugabyte.log.file.name")); + renamePairs.add(new RenamePair("log_level", "yugabyte.log_level")); + renamePairs.add(new RenamePair("audit_type", "yugabyte.audit_type")); + renamePairs.add(new RenamePair("statement_id", "yugabyte.statement_id")); + renamePairs.add(new RenamePair("substatement_id", "yugabyte.substatement_id")); + renamePairs.add(new RenamePair("class", "yugabyte.class")); + renamePairs.add(new RenamePair("command", "yugabyte.command")); + renamePairs.add(new RenamePair("object_type", "yugabyte.object_type")); + renamePairs.add(new RenamePair("object_name", "yugabyte.object_name")); + renamePairs.add(new RenamePair("statement", "yugabyte.statement")); + renamePairs.forEach(rp -> attributeActions.addAll(rp.getRenameAttributeActions())); + + // Rename the log prefix extracted attributes to come under the key yugabyte. + AuditLogRegexGenerator.LogRegexResult regexResult = + auditLogRegexGenerator.generateAuditLogRegex(logLinePrefix, /*onlyPrefix*/ true); + regexResult + .getTokens() + .forEach( + token -> { + RenamePair rp = + new RenamePair(token.getAttributeName(), token.getYugabyteAttributeName()); + attributeActions.addAll(rp.getRenameAttributeActions()); + }); // Override or add tags from the exporter config. if (MapUtils.isNotEmpty(telemetryProvider.getTags())) { @@ -336,7 +431,7 @@ private void appendExporter( .map( e -> new OtelCollectorConfigFormat.AttributeAction( - e.getKey(), e.getValue(), "upsert")) + e.getKey(), e.getValue(), "upsert", null)) .toList()); } @@ -347,7 +442,7 @@ private void appendExporter( .map( e -> new OtelCollectorConfigFormat.AttributeAction( - e.getKey(), e.getValue(), "upsert")) + e.getKey(), e.getValue(), "upsert", null)) .toList()); } diff --git a/managed/src/test/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGeneratorTest.java b/managed/src/test/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGeneratorTest.java index 687b3a7b044..ad2f83ee890 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGeneratorTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/audit/otel/OtelCollectorConfigGeneratorTest.java @@ -58,7 +58,9 @@ public void setUp() { doNothing().when(mockTelemetryProviderService).validateBean(any()); customer = ModelFactory.testCustomer(); provider = ModelFactory.awsProvider(customer); - universe = ModelFactory.createUniverse(customer.getId()); + universe = + ModelFactory.createUniverse( + "test-universe", UUID.fromString("00000000-0000-0000-0000-000000000000")); universe = ModelFactory.addNodesToUniverse(universe.getUniverseUUID(), 1); // update the node name universe = diff --git a/managed/src/test/resources/audit/dd_config.yml b/managed/src/test/resources/audit/dd_config.yml index 7c8ca30ad10..61c67279994 100644 --- a/managed/src/test/resources/audit/dd_config.yml +++ b/managed/src/test/resources/audit/dd_config.yml @@ -15,11 +15,39 @@ processors: - {action: upsert, key: ddsource, value: yugabyte} - {action: upsert, key: service, value: yb-otel-collector} - {action: upsert, key: host, value: test-node} + - {action: upsert, key: yugabyte.cloud, value: ''} + - {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000} + - {action: upsert, key: yugabyte.node_type, value: PRIMARY} + - {action: upsert, key: yugabyte.region, value: ''} + - {action: upsert, key: yugabyte.zone, value: ''} + - {action: upsert, key: yugabyte.purpose, value: DATA_DOG_LOG_EXPORT} + - {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name} + - {action: delete, key: log.file.name} + - {action: upsert, from_attribute: log_level, key: yugabyte.log_level} + - {action: delete, key: log_level} + - {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type} + - {action: delete, key: audit_type} + - {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id} + - {action: delete, key: statement_id} + - {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id} + - {action: delete, key: substatement_id} + - {action: upsert, from_attribute: class, key: yugabyte.class} + - {action: delete, key: class} + - {action: upsert, from_attribute: command, key: yugabyte.command} + - {action: delete, key: command} + - {action: upsert, from_attribute: object_type, key: yugabyte.object_type} + - {action: delete, key: object_type} + - {action: upsert, from_attribute: object_name, key: yugabyte.object_name} + - {action: delete, key: object_name} + - {action: upsert, from_attribute: statement, key: yugabyte.statement} + - {action: delete, key: statement} + - {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms} + - {action: delete, key: timestamp_without_ms} - {action: upsert, key: tag, value: value} - {action: upsert, key: additionalTag, value: otherValue} receivers: filelog/ysql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver - attributes: {audit_log_type: ysql} + attributes: {yugabyte.audit_log_type: ysql} exclude: [/home/yugabyte/tserver/logs/*.gz] include: [/home/yugabyte/tserver/logs/postgresql-*.log] multiline: {line_start_pattern: '([A-Z]\d{4})|((?P\d{4}-\d{2}-\d{2} diff --git a/managed/src/test/resources/audit/multi_config.yml b/managed/src/test/resources/audit/multi_config.yml index ba7e26d916e..ee260ce030b 100644 --- a/managed/src/test/resources/audit/multi_config.yml +++ b/managed/src/test/resources/audit/multi_config.yml @@ -20,16 +20,72 @@ processors: attributes/00000000-0000-0000-0000-000000000000: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor actions: - {action: upsert, key: host, value: test-node} + - {action: upsert, key: yugabyte.cloud, value: ''} + - {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000} + - {action: upsert, key: yugabyte.node_type, value: PRIMARY} + - {action: upsert, key: yugabyte.region, value: ''} + - {action: upsert, key: yugabyte.zone, value: ''} + - {action: upsert, key: yugabyte.purpose, value: AWS_CLOUDWATCH_LOG_EXPORT} + - {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name} + - {action: delete, key: log.file.name} + - {action: upsert, from_attribute: log_level, key: yugabyte.log_level} + - {action: delete, key: log_level} + - {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type} + - {action: delete, key: audit_type} + - {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id} + - {action: delete, key: statement_id} + - {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id} + - {action: delete, key: substatement_id} + - {action: upsert, from_attribute: class, key: yugabyte.class} + - {action: delete, key: class} + - {action: upsert, from_attribute: command, key: yugabyte.command} + - {action: delete, key: command} + - {action: upsert, from_attribute: object_type, key: yugabyte.object_type} + - {action: delete, key: object_type} + - {action: upsert, from_attribute: object_name, key: yugabyte.object_name} + - {action: delete, key: object_name} + - {action: upsert, from_attribute: statement, key: yugabyte.statement} + - {action: delete, key: statement} + - {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms} + - {action: delete, key: timestamp_without_ms} - {action: upsert, key: tag, value: value} - {action: upsert, key: additionalTag, value: otherValue} attributes/11111111-1111-1111-1111-111111111111: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor actions: - {action: upsert, key: host, value: test-node} + - {action: upsert, key: yugabyte.cloud, value: ''} + - {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000} + - {action: upsert, key: yugabyte.node_type, value: PRIMARY} + - {action: upsert, key: yugabyte.region, value: ''} + - {action: upsert, key: yugabyte.zone, value: ''} + - {action: upsert, key: yugabyte.purpose, value: GCP_CLOUD_MONITORING_LOG_EXPORT} + - {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name} + - {action: delete, key: log.file.name} + - {action: upsert, from_attribute: log_level, key: yugabyte.log_level} + - {action: delete, key: log_level} + - {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type} + - {action: delete, key: audit_type} + - {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id} + - {action: delete, key: statement_id} + - {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id} + - {action: delete, key: substatement_id} + - {action: upsert, from_attribute: class, key: yugabyte.class} + - {action: delete, key: class} + - {action: upsert, from_attribute: command, key: yugabyte.command} + - {action: delete, key: command} + - {action: upsert, from_attribute: object_type, key: yugabyte.object_type} + - {action: delete, key: object_type} + - {action: upsert, from_attribute: object_name, key: yugabyte.object_name} + - {action: delete, key: object_name} + - {action: upsert, from_attribute: statement, key: yugabyte.statement} + - {action: delete, key: statement} + - {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms} + - {action: delete, key: timestamp_without_ms} - {action: upsert, key: tag, value: value1} - {action: upsert, key: additionalTag, value: yetAnotherValue} receivers: filelog/ysql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver - attributes: {audit_log_type: ysql} + attributes: {yugabyte.audit_log_type: ysql} exclude: [/home/yugabyte/tserver/logs/*.gz] include: [/home/yugabyte/tserver/logs/postgresql-*.log] multiline: {line_start_pattern: '([A-Z]\d{4})|((?P\d{4}-\d{2}-\d{2} @@ -46,7 +102,7 @@ receivers: start_at: beginning storage: file_storage/queue filelog/ycql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver - attributes: {audit_log_type: ycql} + attributes: {yugabyte.audit_log_type: ycql} exclude: [/home/yugabyte/tserver/logs/*.gz] include: [/home/yugabyte/tserver/logs/yb-tserver.*.WARNING.*] multiline: {line_start_pattern: '([A-Z]\d{4})'} diff --git a/managed/src/test/resources/audit/splunk_config.yml b/managed/src/test/resources/audit/splunk_config.yml index 4c1fea208c4..612eb23f1ef 100644 --- a/managed/src/test/resources/audit/splunk_config.yml +++ b/managed/src/test/resources/audit/splunk_config.yml @@ -18,11 +18,39 @@ processors: attributes/00000000-0000-0000-0000-000000000000: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor actions: - {action: upsert, key: host, value: test-node} + - {action: upsert, key: yugabyte.cloud, value: ''} + - {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000} + - {action: upsert, key: yugabyte.node_type, value: PRIMARY} + - {action: upsert, key: yugabyte.region, value: ''} + - {action: upsert, key: yugabyte.zone, value: ''} + - {action: upsert, key: yugabyte.purpose, value: SPLUNK_LOG_EXPORT} + - {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name} + - {action: delete, key: log.file.name} + - {action: upsert, from_attribute: log_level, key: yugabyte.log_level} + - {action: delete, key: log_level} + - {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type} + - {action: delete, key: audit_type} + - {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id} + - {action: delete, key: statement_id} + - {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id} + - {action: delete, key: substatement_id} + - {action: upsert, from_attribute: class, key: yugabyte.class} + - {action: delete, key: class} + - {action: upsert, from_attribute: command, key: yugabyte.command} + - {action: delete, key: command} + - {action: upsert, from_attribute: object_type, key: yugabyte.object_type} + - {action: delete, key: object_type} + - {action: upsert, from_attribute: object_name, key: yugabyte.object_name} + - {action: delete, key: object_name} + - {action: upsert, from_attribute: statement, key: yugabyte.statement} + - {action: delete, key: statement} + - {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms} + - {action: delete, key: timestamp_without_ms} - {action: upsert, key: tag, value: value} - {action: upsert, key: additionalTag, value: otherValue} receivers: filelog/ycql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver - attributes: {audit_log_type: ycql} + attributes: {yugabyte.audit_log_type: ycql} exclude: [/home/yugabyte/tserver/logs/*.gz] include: [/home/yugabyte/tserver/logs/yb-tserver.*.WARNING.*] multiline: {line_start_pattern: '([A-Z]\d{4})'}