Skip to content

Commit

Permalink
[PLAT-16112][PLAT-16269] Update log attributes getting sent in Databa…
Browse files Browse the repository at this point in the history
…se Audit Logging

Summary:
This diff fixes 2 tickets:
1. [PLAT-16112] Update log attributes that are set from YBA in DBAL. Puts most of the logs under the key `yugabyte` for better organisation.
2. [PLAT-16269] [Master] Enabling audit log on a universe is failing due to error 'the field ''become'' has an invalid value

Test Plan:
Manually tested and ensured the log attributes are correct.
Log Before:
```
{
	"id": "...",
	"content": {
		"timestamp": "2024-12-07T19:37:06Z",
		"tags": [
			"source:yugabyte",
			"otel_source:datadog_exporter",
			"datadog.submission_auth:api_key"
		],
		"host": "yb-admin-sahith-uni-1-n1",
		"service": "yb-otel-collector",
		"message": "2024-12-07 19:37:06.945 UTC [69811] LOG:  AUDIT: SESSION,1,1,MISC,UNKNOWN,COMPOSITE TYPE,public.write_read_test,<previously logged>,<previously logged>",
		"attributes": {
			"process_id": "69811",
			"log": {
				"file": {
					"name": "postgresql-2024-12-07_193605.log"
				}
			},
			"object_type": "COMPOSITE TYPE",
			"log_level": "LOG",
			"statement_id": "1",
			"command": "UNKNOWN",
			"timestamp_with_ms": "2024-12-07 19:37:06.945 UTC",
			"yugabyte": {
				"audit_log_type": "ysql"
			},
			"hostname": "yb-admin-sahith-uni-1-n1",
			"@timestamp": "2024-12-07T19:37:06Z",
			"service": "yb-otel-collector",
			"object_name": "public.write_read_test",
			"host": "yb-admin-sahith-uni-1-n1",
			"otel": {
				"timestamp": "1733600226945000000"
			},
			"statement": "<previously logged>,<previously logged>",
			"class": "MISC",
			"substatement_id": "1",
			"audit_type": "SESSION",
			"status": ""
		}
	}
}
```

Log After:
```
{
	"id": "...",
	"content": {
		"timestamp": "2024-12-12T09:20:26Z",
		"tags": [
			"source:yugabyte",
			"otel_source:datadog_exporter",
			"datadog.submission_auth:api_key"
		],
		"host": "yb-admin-sahith-uni-1-n1",
		"service": "yb-otel-collector",
		"message": "2024-12-12 09:20:26.733 UTC [3634113] LOG:  AUDIT: SESSION,1,1,MISC,UNKNOWN,COMPOSITE TYPE,public.write_read_test,<previously logged>,<previously logged>",
		"attributes": {
			"yugabyte": {
				"process_id": "3634113",
				"log": {
					"file": {
						"name": "postgresql-2024-12-12_082604.log"
					}
				},
				"object_type": "COMPOSITE TYPE",
				"purpose": "DATA_DOG_LOG_EXPORT",
				"log_level": "LOG",
				"audit_log_type": "ysql",
				"universe_uuid": "75dca85b-05d9-4150-9234-e055d421b52c",
				"statement_id": "1",
				"command": "UNKNOWN",
				"cloud": "gcp",
				"timestamp_with_ms": "2024-12-12 09:20:26.733 UTC",
				"node_type": "PRIMARY",
				"zone": "us-west1-a",
				"object_name": "public.write_read_test",
				"statement": "<previously logged>,<previously logged>",
				"region": "us-west1",
				"class": "MISC",
				"substatement_id": "1",
				"audit_type": "SESSION"
			},
			"hostname": "yb-admin-sahith-uni-1-n1",
			"@timestamp": "2024-12-12T09:20:26Z",
			"service": "yb-otel-collector",
			"host": "yb-admin-sahith-uni-1-n1",
			"otel": {
				"timestamp": "1733995226733000000"
			},
			"status": ""
		}
	}
}
```

Also ran UTs and itests.

Reviewers: amalyshev

Reviewed By: amalyshev

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D40532
  • Loading branch information
Sahith02 committed Dec 13, 2024
1 parent c47afcd commit 93d699f
Show file tree
Hide file tree
Showing 8 changed files with 228 additions and 14 deletions.
2 changes: 1 addition & 1 deletion managed/devops/yb-otel-collector.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

- name: Manage OpenTelemetry Collector
hosts: "{{ yb_prebuilt_ami_host | default(instance_search_pattern) }}"
become: "{{ 'yes' if use_sudo|default(false) else '' }}"
become: "{{ 'yes' if use_sudo|default(false) else 'no' }}"
become_method: "{{ 'sudo' if use_sudo|default(false) else '' }}"
roles:
- "manage_otel_collector"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,5 +141,9 @@ public enum LogPrefixTokens {
public String getAttributeName() {
return name().toLowerCase();
}

public String getYugabyteAttributeName() {
return "yugabyte." + name().toLowerCase();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ public static class AttributeAction {
private String key;
private String value;
private String action;
private String from_attribute;
}

@Data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@
import com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat.MultilineConfig;
import com.yugabyte.yw.common.yaml.SkipNullRepresenter;
import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
import com.yugabyte.yw.models.Customer;
import com.yugabyte.yw.models.Provider;
import com.yugabyte.yw.models.TelemetryProvider;
import com.yugabyte.yw.models.Universe;
import com.yugabyte.yw.models.helpers.NodeDetails;
import com.yugabyte.yw.models.helpers.TelemetryProviderService;
import com.yugabyte.yw.models.helpers.audit.AuditLogConfig;
import com.yugabyte.yw.models.helpers.audit.UniverseLogsExporterConfig;
Expand All @@ -24,8 +27,11 @@
import java.util.*;
import javax.inject.Inject;
import javax.inject.Singleton;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.yaml.snakeyaml.Yaml;
import play.Environment;

Expand Down Expand Up @@ -71,6 +77,8 @@ void generateConfigFile(
String logLinePrefix,
Path path,
int otelColMetricsPort) {
Customer customer = Customer.getOrBadRequest(provider.getCustomerUUID());
Universe universe = Universe.getOrBadRequest(nodeParams.getUniverseUUID());
try (BufferedWriter writer = new BufferedWriter(new FileWriter(path.toFile()))) {
Yaml yaml = new Yaml(new SkipNullRepresenter());
OtelCollectorConfigFormat collectorConfigFormat = new OtelCollectorConfigFormat();
Expand Down Expand Up @@ -118,7 +126,13 @@ void generateConfigFile(
.forEach(
config ->
appendExporter(
collectorConfigFormat, config, currentProcessors, nodeParams.nodeName));
customer,
universe,
collectorConfigFormat,
config,
currentProcessors,
nodeParams.nodeName,
logLinePrefix));
}

yaml.dump(collectorConfigFormat, writer);
Expand Down Expand Up @@ -240,15 +254,36 @@ private OtelCollectorConfigFormat.FileLogReceiver createFileLogReceiver(
receiver.setStart_at("beginning");
receiver.setStorage("file_storage/queue");
receiver.setOperators(operators);
receiver.setAttributes(ImmutableMap.of("audit_log_type", logType));
receiver.setAttributes(ImmutableMap.of("yugabyte.audit_log_type", logType));
return receiver;
}

@Data
@AllArgsConstructor
public static class RenamePair {
private String before;
private String after;

private List<OtelCollectorConfigFormat.AttributeAction> getRenameAttributeActions() {
List<OtelCollectorConfigFormat.AttributeAction> renameActionsList = new ArrayList<>();
// Copy the attribute from existing attribute and delete the original one.
renameActionsList.add(
new OtelCollectorConfigFormat.AttributeAction(this.after, null, "upsert", this.before));
renameActionsList.add(
new OtelCollectorConfigFormat.AttributeAction(this.before, null, "delete", null));
return renameActionsList;
}
}

private void appendExporter(
Customer customer,
Universe universe,
OtelCollectorConfigFormat collectorConfig,
UniverseLogsExporterConfig logsExporterConfig,
List<String> currentProcessors,
String nodeName) {
String nodeName,
String logLinePrefix) {
NodeDetails nodeDetails = universe.getNode(nodeName);
TelemetryProvider telemetryProvider =
telemetryProviderService.getOrBadRequest(logsExporterConfig.getExporterUuid());
Map<String, OtelCollectorConfigFormat.Exporter> exporters = collectorConfig.getExporters();
Expand All @@ -271,10 +306,10 @@ private void appendExporter(

// Add Datadog specific labels.
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction("ddsource", "yugabyte", "upsert"));
new OtelCollectorConfigFormat.AttributeAction("ddsource", "yugabyte", "upsert", null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"service", "yb-otel-collector", "upsert"));
"service", "yb-otel-collector", "upsert", null));
break;
case SPLUNK:
SplunkConfig splunkConfig = (SplunkConfig) telemetryProvider.getConfig();
Expand Down Expand Up @@ -327,7 +362,67 @@ private void appendExporter(
}

// Add some common collector labels.
attributeActions.add(new OtelCollectorConfigFormat.AttributeAction("host", nodeName, "upsert"));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction("host", nodeName, "upsert", null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"yugabyte.cloud",
StringUtils.defaultString(nodeDetails.cloudInfo.cloud, ""),
"upsert",
null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"yugabyte.universe_uuid", universe.getUniverseUUID().toString(), "upsert", null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"yugabyte.node_type",
universe.getCluster(nodeDetails.placementUuid).clusterType.toString(),
"upsert",
null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"yugabyte.region",
StringUtils.defaultString(nodeDetails.cloudInfo.region, ""),
"upsert",
null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"yugabyte.zone",
StringUtils.defaultString(nodeDetails.cloudInfo.az, ""),
"upsert",
null));
attributeActions.add(
new OtelCollectorConfigFormat.AttributeAction(
"yugabyte.purpose",
telemetryProvider.getConfig().getType().toString() + "_LOG_EXPORT",
"upsert",
null));

// Rename the attributes to organise under the key yugabyte.
List<RenamePair> renamePairs = new ArrayList<RenamePair>();
renamePairs.add(new RenamePair("log.file.name", "yugabyte.log.file.name"));
renamePairs.add(new RenamePair("log_level", "yugabyte.log_level"));
renamePairs.add(new RenamePair("audit_type", "yugabyte.audit_type"));
renamePairs.add(new RenamePair("statement_id", "yugabyte.statement_id"));
renamePairs.add(new RenamePair("substatement_id", "yugabyte.substatement_id"));
renamePairs.add(new RenamePair("class", "yugabyte.class"));
renamePairs.add(new RenamePair("command", "yugabyte.command"));
renamePairs.add(new RenamePair("object_type", "yugabyte.object_type"));
renamePairs.add(new RenamePair("object_name", "yugabyte.object_name"));
renamePairs.add(new RenamePair("statement", "yugabyte.statement"));
renamePairs.forEach(rp -> attributeActions.addAll(rp.getRenameAttributeActions()));

// Rename the log prefix extracted attributes to come under the key yugabyte.
AuditLogRegexGenerator.LogRegexResult regexResult =
auditLogRegexGenerator.generateAuditLogRegex(logLinePrefix, /*onlyPrefix*/ true);
regexResult
.getTokens()
.forEach(
token -> {
RenamePair rp =
new RenamePair(token.getAttributeName(), token.getYugabyteAttributeName());
attributeActions.addAll(rp.getRenameAttributeActions());
});

// Override or add tags from the exporter config.
if (MapUtils.isNotEmpty(telemetryProvider.getTags())) {
Expand All @@ -336,7 +431,7 @@ private void appendExporter(
.map(
e ->
new OtelCollectorConfigFormat.AttributeAction(
e.getKey(), e.getValue(), "upsert"))
e.getKey(), e.getValue(), "upsert", null))
.toList());
}

Expand All @@ -347,7 +442,7 @@ private void appendExporter(
.map(
e ->
new OtelCollectorConfigFormat.AttributeAction(
e.getKey(), e.getValue(), "upsert"))
e.getKey(), e.getValue(), "upsert", null))
.toList());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ public void setUp() {
doNothing().when(mockTelemetryProviderService).validateBean(any());
customer = ModelFactory.testCustomer();
provider = ModelFactory.awsProvider(customer);
universe = ModelFactory.createUniverse(customer.getId());
universe =
ModelFactory.createUniverse(
"test-universe", UUID.fromString("00000000-0000-0000-0000-000000000000"));
universe = ModelFactory.addNodesToUniverse(universe.getUniverseUUID(), 1);
// update the node name
universe =
Expand Down
30 changes: 29 additions & 1 deletion managed/src/test/resources/audit/dd_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,39 @@ processors:
- {action: upsert, key: ddsource, value: yugabyte}
- {action: upsert, key: service, value: yb-otel-collector}
- {action: upsert, key: host, value: test-node}
- {action: upsert, key: yugabyte.cloud, value: ''}
- {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000}
- {action: upsert, key: yugabyte.node_type, value: PRIMARY}
- {action: upsert, key: yugabyte.region, value: ''}
- {action: upsert, key: yugabyte.zone, value: ''}
- {action: upsert, key: yugabyte.purpose, value: DATA_DOG_LOG_EXPORT}
- {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name}
- {action: delete, key: log.file.name}
- {action: upsert, from_attribute: log_level, key: yugabyte.log_level}
- {action: delete, key: log_level}
- {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type}
- {action: delete, key: audit_type}
- {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id}
- {action: delete, key: statement_id}
- {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id}
- {action: delete, key: substatement_id}
- {action: upsert, from_attribute: class, key: yugabyte.class}
- {action: delete, key: class}
- {action: upsert, from_attribute: command, key: yugabyte.command}
- {action: delete, key: command}
- {action: upsert, from_attribute: object_type, key: yugabyte.object_type}
- {action: delete, key: object_type}
- {action: upsert, from_attribute: object_name, key: yugabyte.object_name}
- {action: delete, key: object_name}
- {action: upsert, from_attribute: statement, key: yugabyte.statement}
- {action: delete, key: statement}
- {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms}
- {action: delete, key: timestamp_without_ms}
- {action: upsert, key: tag, value: value}
- {action: upsert, key: additionalTag, value: otherValue}
receivers:
filelog/ysql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver
attributes: {audit_log_type: ysql}
attributes: {yugabyte.audit_log_type: ysql}
exclude: [/home/yugabyte/tserver/logs/*.gz]
include: [/home/yugabyte/tserver/logs/postgresql-*.log]
multiline: {line_start_pattern: '([A-Z]\d{4})|((?P<timestamp_without_ms>\d{4}-\d{2}-\d{2}
Expand Down
60 changes: 58 additions & 2 deletions managed/src/test/resources/audit/multi_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,72 @@ processors:
attributes/00000000-0000-0000-0000-000000000000: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor
actions:
- {action: upsert, key: host, value: test-node}
- {action: upsert, key: yugabyte.cloud, value: ''}
- {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000}
- {action: upsert, key: yugabyte.node_type, value: PRIMARY}
- {action: upsert, key: yugabyte.region, value: ''}
- {action: upsert, key: yugabyte.zone, value: ''}
- {action: upsert, key: yugabyte.purpose, value: AWS_CLOUDWATCH_LOG_EXPORT}
- {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name}
- {action: delete, key: log.file.name}
- {action: upsert, from_attribute: log_level, key: yugabyte.log_level}
- {action: delete, key: log_level}
- {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type}
- {action: delete, key: audit_type}
- {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id}
- {action: delete, key: statement_id}
- {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id}
- {action: delete, key: substatement_id}
- {action: upsert, from_attribute: class, key: yugabyte.class}
- {action: delete, key: class}
- {action: upsert, from_attribute: command, key: yugabyte.command}
- {action: delete, key: command}
- {action: upsert, from_attribute: object_type, key: yugabyte.object_type}
- {action: delete, key: object_type}
- {action: upsert, from_attribute: object_name, key: yugabyte.object_name}
- {action: delete, key: object_name}
- {action: upsert, from_attribute: statement, key: yugabyte.statement}
- {action: delete, key: statement}
- {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms}
- {action: delete, key: timestamp_without_ms}
- {action: upsert, key: tag, value: value}
- {action: upsert, key: additionalTag, value: otherValue}
attributes/11111111-1111-1111-1111-111111111111: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor
actions:
- {action: upsert, key: host, value: test-node}
- {action: upsert, key: yugabyte.cloud, value: ''}
- {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000}
- {action: upsert, key: yugabyte.node_type, value: PRIMARY}
- {action: upsert, key: yugabyte.region, value: ''}
- {action: upsert, key: yugabyte.zone, value: ''}
- {action: upsert, key: yugabyte.purpose, value: GCP_CLOUD_MONITORING_LOG_EXPORT}
- {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name}
- {action: delete, key: log.file.name}
- {action: upsert, from_attribute: log_level, key: yugabyte.log_level}
- {action: delete, key: log_level}
- {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type}
- {action: delete, key: audit_type}
- {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id}
- {action: delete, key: statement_id}
- {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id}
- {action: delete, key: substatement_id}
- {action: upsert, from_attribute: class, key: yugabyte.class}
- {action: delete, key: class}
- {action: upsert, from_attribute: command, key: yugabyte.command}
- {action: delete, key: command}
- {action: upsert, from_attribute: object_type, key: yugabyte.object_type}
- {action: delete, key: object_type}
- {action: upsert, from_attribute: object_name, key: yugabyte.object_name}
- {action: delete, key: object_name}
- {action: upsert, from_attribute: statement, key: yugabyte.statement}
- {action: delete, key: statement}
- {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms}
- {action: delete, key: timestamp_without_ms}
- {action: upsert, key: tag, value: value1}
- {action: upsert, key: additionalTag, value: yetAnotherValue}
receivers:
filelog/ysql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver
attributes: {audit_log_type: ysql}
attributes: {yugabyte.audit_log_type: ysql}
exclude: [/home/yugabyte/tserver/logs/*.gz]
include: [/home/yugabyte/tserver/logs/postgresql-*.log]
multiline: {line_start_pattern: '([A-Z]\d{4})|((?P<timestamp_without_ms>\d{4}-\d{2}-\d{2}
Expand All @@ -46,7 +102,7 @@ receivers:
start_at: beginning
storage: file_storage/queue
filelog/ycql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver
attributes: {audit_log_type: ycql}
attributes: {yugabyte.audit_log_type: ycql}
exclude: [/home/yugabyte/tserver/logs/*.gz]
include: [/home/yugabyte/tserver/logs/yb-tserver.*.WARNING.*]
multiline: {line_start_pattern: '([A-Z]\d{4})'}
Expand Down
30 changes: 29 additions & 1 deletion managed/src/test/resources/audit/splunk_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,39 @@ processors:
attributes/00000000-0000-0000-0000-000000000000: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor
actions:
- {action: upsert, key: host, value: test-node}
- {action: upsert, key: yugabyte.cloud, value: ''}
- {action: upsert, key: yugabyte.universe_uuid, value: 00000000-0000-0000-0000-000000000000}
- {action: upsert, key: yugabyte.node_type, value: PRIMARY}
- {action: upsert, key: yugabyte.region, value: ''}
- {action: upsert, key: yugabyte.zone, value: ''}
- {action: upsert, key: yugabyte.purpose, value: SPLUNK_LOG_EXPORT}
- {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name}
- {action: delete, key: log.file.name}
- {action: upsert, from_attribute: log_level, key: yugabyte.log_level}
- {action: delete, key: log_level}
- {action: upsert, from_attribute: audit_type, key: yugabyte.audit_type}
- {action: delete, key: audit_type}
- {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id}
- {action: delete, key: statement_id}
- {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id}
- {action: delete, key: substatement_id}
- {action: upsert, from_attribute: class, key: yugabyte.class}
- {action: delete, key: class}
- {action: upsert, from_attribute: command, key: yugabyte.command}
- {action: delete, key: command}
- {action: upsert, from_attribute: object_type, key: yugabyte.object_type}
- {action: delete, key: object_type}
- {action: upsert, from_attribute: object_name, key: yugabyte.object_name}
- {action: delete, key: object_name}
- {action: upsert, from_attribute: statement, key: yugabyte.statement}
- {action: delete, key: statement}
- {action: upsert, from_attribute: timestamp_without_ms, key: yugabyte.timestamp_without_ms}
- {action: delete, key: timestamp_without_ms}
- {action: upsert, key: tag, value: value}
- {action: upsert, key: additionalTag, value: otherValue}
receivers:
filelog/ycql: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver
attributes: {audit_log_type: ycql}
attributes: {yugabyte.audit_log_type: ycql}
exclude: [/home/yugabyte/tserver/logs/*.gz]
include: [/home/yugabyte/tserver/logs/yb-tserver.*.WARNING.*]
multiline: {line_start_pattern: '([A-Z]\d{4})'}
Expand Down

0 comments on commit 93d699f

Please sign in to comment.