feat(new transform): Inital kubernetes_pod_metadata transform

.github/labels.yml

@@ -318,6 +318,9 @@
 - name: "transform: json_parser"
   description: Anything `json_parser` transform related
   color: 54a0ff
+- name: "transform: kubernetes_pod_metadata"
+  description: Anything `kubernetes_pod_metadata` transform related
+  color: 54a0ff
 - name: "transform: log_to_metric"
   description: Anything `log_to_metric` transform related
   color: 54a0ff

.github/semantic.yml

@@ -96,6 +96,7 @@ scopes:
   - geoip transform
   - grok_parser transform
   - json_parser transform
+  - kubernetes_pod_metadata transform
   - log_to_metric transform
   - logfmt_parser transform
   - lua transform

.meta/.schemas/guides.json

@@ -52,7 +52,7 @@
       "description": "Array of tags",
       "items": {
         "type": "string",
-        "enum": ["type: guide","type: post","type: tutorial","domain: buffers","domain: cli","domain: config","domain: dependencies","domain: networking","domain: observability","domain: operations","domain: platforms","domain: security","domain: sinks","domain: sources","domain: tests","domain: topology","domain: transforms","domain: ux","domain: website","platform: docker","source: docker","source: file","source: http","source: journald","source: kafka","source: logplex","source: prometheus","source: socket","source: splunk_hec","source: statsd","source: stdin","source: syslog","source: vector","source: add_fields","source: add_tags","source: ansi_stripper","source: aws_ec2_metadata","source: coercer","source: concat","source: dedupe","source: filter","source: geoip","source: grok_parser","source: json_parser","source: log_to_metric","source: logfmt_parser","source: lua","source: merge","source: regex_parser","source: remove_fields","source: remove_tags","source: rename_fields","source: sampler","source: split","source: swimlanes","source: tag_cardinality_limit","source: tokenizer","sink: aws_cloudwatch_logs","sink: aws_cloudwatch_metrics","sink: aws_kinesis_firehose","sink: aws_kinesis_streams","sink: aws_s3","sink: blackhole","sink: clickhouse","sink: console","sink: datadog_logs","sink: datadog_metrics","sink: elasticsearch","sink: file","sink: gcp_cloud_storage","sink: gcp_pubsub","sink: gcp_stackdriver_logs","sink: honeycomb","sink: http","sink: humio_logs","sink: influxdb_metrics","sink: kafka","sink: logdna","sink: loki","sink: new_relic_logs","sink: papertrail","sink: prometheus","sink: pulsar","sink: sematext_logs","sink: socket","sink: splunk_hec","sink: statsd","sink: vector"]
+        "enum": ["type: guide","type: post","type: tutorial","domain: buffers","domain: cli","domain: config","domain: dependencies","domain: networking","domain: observability","domain: operations","domain: platforms","domain: security","domain: sinks","domain: sources","domain: tests","domain: topology","domain: transforms","domain: ux","domain: website","platform: docker","source: docker","source: file","source: http","source: journald","source: kafka","source: logplex","source: prometheus","source: socket","source: splunk_hec","source: statsd","source: stdin","source: syslog","source: vector","source: add_fields","source: add_tags","source: ansi_stripper","source: aws_ec2_metadata","source: coercer","source: concat","source: dedupe","source: filter","source: geoip","source: grok_parser","source: json_parser","source: kubernetes_pod_metadata","source: log_to_metric","source: logfmt_parser","source: lua","source: merge","source: regex_parser","source: remove_fields","source: remove_tags","source: rename_fields","source: sampler","source: split","source: swimlanes","source: tag_cardinality_limit","source: tokenizer","sink: aws_cloudwatch_logs","sink: aws_cloudwatch_metrics","sink: aws_kinesis_firehose","sink: aws_kinesis_streams","sink: aws_s3","sink: blackhole","sink: clickhouse","sink: console","sink: datadog_logs","sink: datadog_metrics","sink: elasticsearch","sink: file","sink: gcp_cloud_storage","sink: gcp_pubsub","sink: gcp_stackdriver_logs","sink: honeycomb","sink: http","sink: humio_logs","sink: influxdb_metrics","sink: kafka","sink: logdna","sink: loki","sink: new_relic_logs","sink: papertrail","sink: prometheus","sink: pulsar","sink: sematext_logs","sink: socket","sink: splunk_hec","sink: statsd","sink: vector"]
       }
     },
     "title": {

.meta/.schemas/meta.json

@@ -59,7 +59,7 @@
       "type": "object",
       "description": "A Vector field, such as a component option, environment variable, log schema field, etc.",
       "additionalProperties": false,
-      "required": ["description", "type"],
+      "required": ["description"],
       "properties": {
         "category": {
           "type": "string",
@@ -189,6 +189,10 @@
                 "type": "string",
                 "description": "The output exmaple body. Markdown syntax is supported."
               },
+              "global_log_schema_key": {
+                "type": "string",
+                "descripttion": "If these fields are inserted under a global log schema key, what is the name of that key?"
+              },
               "label": {
                 "type": "string",
                 "description": "The output label. This will be used in the tab name if multiple output examples are provided."
@@ -201,6 +205,31 @@
         }
       }
     },
+    "permissions": {
+      "type": "object",
+      "description": "An object of permissions keyed by the permission's name.",
+      "patternProperties": {
+        "^[a-z0-9_]+$": {
+          "$ref": "#/definitions/permission"
+        }
+      }
+    },
+    "permission": {
+      "type": "object",
+      "description": "A Vector permission, such as a AWS IAM, GCP IAM, or Kubernetes RBAC permission.",
+      "additionalProperties": false,
+      "required": ["description", "name"],
+      "properties": {
+        "description": {
+          "type": "string",
+          "description": "The permission description."
+        },
+        "name": {
+          "type": "string",
+          "description": "The permission name."
+        }
+      }
+    },
     "requirements": {
       "type": "object",
       "additionalProperties": false,
@@ -233,6 +262,10 @@
           "type": "string",
           "description": "The minimum Kafka version required."
         },
+        "kubernetes": {
+          "type": "string",
+          "description": "The minimum Kubernetes version required."
+        },
         "network_port": {
           "type": "string",
           "description": "The network port that this component exposes."
@@ -346,6 +379,10 @@
           "$ref": "#/definitions/output",
           "description": "Output examples of this sink."
         },
+        "permissions": {
+          "$ref": "#/definitions/permissions",
+          "description": "The permissions required for this sink to work."
+        },
         "requirements": {
           "$ref": "#/definitions/requirements",
           "description": "Component requirements"
@@ -457,6 +494,10 @@
           "$ref": "#/definitions/fields",
           "description": "Options provided by this source, keyed by the option name."
         },
+        "permissions": {
+          "$ref": "#/definitions/permissions",
+          "description": "The permissions required for this source to work."
+        },
         "output": {
           "type": "object",
           "description": "Output examples for this source.",
@@ -595,6 +636,10 @@
           "$ref": "#/definitions/event_types",
           "description": "The types of events this transform outputs."
         },
+        "permissions": {
+          "$ref": "#/definitions/permissions",
+          "description": "The permissions required for this transform to work."
+        },
         "requirements": {
           "$ref": "#/definitions/requirements",
           "description": "Component requirements"

.meta/links.toml

@@ -132,9 +132,15 @@ kafka = "https://kafka.apache.org/"
 kafka_partitioning_docs = "https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Partitioningandbootstrapping"
 kafka_protocol = "https://kafka.apache.org/protocol"
 kubernetes = "https://kubernetes.io/"
-kubernetes_limit_resources = "https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/"
+kubernetes_accessing_api_from_pod = "https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod"
+kubernetes_api = "https://kubernetes.io/docs/concepts/overview/kubernetes-api/"
+kubernetes_api_server = "https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/"
 kubernetes_daemonset = "https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/"
 kubernetes_example_daemonset = "https://github.com/timberio/vector/blob/master/config/kubernetes/vector-daemonset.yaml"
+kubernetes_limit_resources = "https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/"
+kubernetes_rbac = "https://kubernetes.io/docs/reference/access-authn-authz/rbac/"
+kubernetes_request_verbs = "https://kubernetes.io/docs/reference/access-authn-authz/authorization/#determine-the-request-verb"
+kubernetes_watch_api = "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#watch-30"
 leveldb = "https://github.com/google/leveldb"
 leveldb-sys-2 = "https://crates.io/crates/leveldb-sys"
 leveldb-sys-3 = "https://github.com/timberio/leveldb-sys/tree/v3.0.0"
@@ -192,6 +198,7 @@ rpm = "https://rpm.org/"
 rust = "https://www.rust-lang.org/"
 rust_date_time = "https://docs.rs/chrono/0.4.0/chrono/struct.DateTime.html"
 rust_grok_library = "https://github.com/daschl/grok"
+rust_k8s_openapi_crate = "https://docs.rs/k8s-openapi/"
 rust_memory_safety = "https://hacks.mozilla.org/2019/01/fearless-security-memory-safety/"
 rust_regex_syntax = "https://docs.rs/regex/1.1.7/regex/#syntax"
 rust_sec = "https://rustsec.org/"

.meta/options.toml

@@ -50,3 +50,9 @@ description = """\
 The key used to represent when the log was generated. See the \
 [log data model page][docs.data-model.log#timestamp] for more info.\
 """
+
+[options.log_schema.children.kubernetes_key]
+type = "string"
+default = "kubernetes"
+examples = ["kubernetes", "kube", "k8s"]
+description = "The key under which Kubernetes related fields are nested."

.meta/transforms/kubernetes_pod_metadata.toml

@@ -0,0 +1,224 @@
+[transforms.kubernetes_pod_metadata]
+title = "Kubernetes Pod Metadata"
+allow_you_to_description = "enrich [Kubernetes][urls.kubernetes] logs with Pod metadata"
+beta = true
+common = true
+function_category = "enrich"
+input_types = ["log"]
+output_types = ["log"]
+requirements.kubernetes = ">= 1.15"
+requirements.kubernetes_rbac = true
+requirements.additional = """\
+Vector must be run inside a Kubernetes cluster for this transform to work. \
+See [Connecting To The Kubernetes API server section](#connecting-to-the-kubernetes-api-server) for more info.
+
+Vector must be given authorization to `watch` resource `pods`. \
+In the case of using [predefined Kubernetes config][https://github.com/timberio/vector/blob/master/config/kubernetes/vector-daemonset.yaml] \
+[RBAC Authorization][https://kubernetes.io/docs/reference/access-authn-authz/rbac/] must be enabled. Which is usually enabled by default.
+"""
+
+<%= render(
+  "_partials/fields/_component_options.toml",
+  type: "transform",
+  name: "kubernetes_pod_metadata"
+) %>
+
+#
+# env_vars
+#
+
+[transforms.kubernetes_pod_metadata.env_vars.VECTOR_NODE_NAME]
+type = "string"
+examples = ["minikube"]
+required = true
+description = """\
+The name of the node whose Pod's log should be enriched. If you're using the \
+[provided daemon set][urls.kubernetes_limit_resources] then this environment \
+variable is set for you.\
+"""
+
+#
+# options
+#
+
+[transforms.kubernetes_pod_metadata.options.fields]
+type = "[string]"
+common = true
+default = ["name","namespace","labels","annotations","node_name"]
+description = """\
+A list of fields to include in each event. By default, these fields are placed \
+under the global [`kubernetes_key` \
+option][docs.reference.global-options#kubernetes_key]. See \
+[Nesting](#nesting-metadata-fields) for more info. \
+"""
+
+[transforms.kubernetes_pod_metadata.options.cache_ttl]
+type = "int"
+common = false
+default = 3600
+unit = "seconds"
+description = """\
+How many seconds will the metadata be available after its Pod has been deleted.\
+"""
+
+[transforms.kubernetes_pod_metadata.options.node_name]
+type = "string"
+common = false
+default = "${VECTOR_NODE_NAME}"
+examples = ["${VECTOR_NODE_NAME}", "minikube"]
+description = """\
+The name of the node whose Pod's log should be enriched. This field defaults \
+to the `VECTOR_NODE_NAME` env var. If you're using the \
+[provided daemon set][urls.kubernetes_limit_resources] then this environment \
+variable is set for you. Setting this to a static value will take priority \
+over `VECTOR_NODE_NAME`.
+"""
+
+[transforms.kubernetes_pod_metadata.options.pod_uid]
+type = "string"
+common = false
+default = "object_uid"
+description = "Field containg Pod UID to which log belongs."
+
+[transforms.kubernetes_pod_metadata.options.max_retry_timeout]
+type = "int"
+common = false
+default = 1
+description = """\
+In the case of Kubernetes API communication failures Vector will indefinitely \
+retry with an exponenetial backoff. This option defines that maximum backoff \
+allowed.\
+"""
+
+#
+# permissions
+#
+
+[transforms.kubernetes_pod_metadata.permissions.watch]
+description = """\
+Vector requires the [`watch` request verb][urls.kubernetes_request_verbs] in \
+order to retrieve Kubernetes metadata.\
+"""
+
+#
+# output
+#
+
+[transforms.kubernetes_pod_metadata.output.log]
+global_log_schema_key = "kubernetes"
+
+[transforms.kubernetes_pod_metadata.output.log.fields.name]
+type = "string"
+examples = ["nginx"]
+description = "[Name][http://kubernetes.io/docs/user-guide/identifiers#names] of the Pod where the log originated."
+
+[transforms.kubernetes_pod_metadata.output.log.fields.namespace]
+type = "string"
+examples = ["default","telemetry","kube-system"]
+description = "[Namespace][http://kubernetes.io/docs/user-guide/namespaces] of the Pod where the log originated."
+
+[transforms.kubernetes_pod_metadata.output.log.fields.creation_timestamp]
+type = "timestamp"
+examples = ["2019-11-01T21:15:47+00:00"]
+description = "Timestamp representing when the Pod was created."
+
+[transforms.kubernetes_pod_metadata.output.log.fields.deletion_timestamp]
+type = "timestamp"
+examples = ["2019-11-01T21:15:47+00:00"]
+description = "Timestamp representing when the Pod was deleted."
+
+[transforms.kubernetes_pod_metadata.output.log.fields.labels]
+type = "table"
+description = """\
+[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) \
+of the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.labels.children."`[label-key]`"]
+type = "*"
+examples = [
+  {release = "stable"},
+  {tier = "frontend"}
+]
+description = """\
+Each individual label will be added as a key. If you need to remove or alter \
+these labels then we recommend the [`remove_felds` \
+transform][docs.transforms.remove_fields].
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.annotations]
+type = "table"
+description = """\
+[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) \
+of the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.annotations.children."`[annotation-key]`"]
+type = "*"
+examples = [
+  {"kubectl.kubernetes.io/last-applied-configuration" = "..."},
+]
+description = """\
+Each individual annotation will be added as a key. If you need to remove or \
+alter these labels then we recommend the [`remove_felds` \
+transform][docs.transforms.remove_fields].
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.node_name]
+type = "string"
+examples = ["minikube"]
+description = """\
+The name of the node where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.hostname]
+type = "string"
+examples = ["host"]
+description = """\
+The hostname of the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.priority]
+type = "int"
+examples = [0]
+description = """\
+The [priority](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/) \
+of the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.priority_class_name]
+type = "string"
+examples = ["system-node-critical","system-cluster-critical"]
+description = """\
+The [priority class](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass) \
+of the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.service_account_name]
+type = "string"
+examples = ["default"]
+description = """\
+The name of the [ServiceAccount][https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/] \
+used to run the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.subdomain]
+type = "string"
+examples = ["net"]
+description = """\
+The subdomain of the Pod where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.host_ip]
+type = "string"
+examples = ["192.168.99.106"]
+description = """\
+The IP address of the host that the Pod is assigned where the log originated.\
+"""
+
+[transforms.kubernetes_pod_metadata.output.log.fields.ip]
+type = "string"
+examples = ["172.17.0.5"]
+description = """\
+The IP address allocated to the Pod where the log originated.\
+"""

Cargo.toml

@@ -280,7 +280,7 @@ transforms-field_filter = []
 transforms-geoip = ["maxminddb"]
 transforms-grok_parser = ["grok"]
 transforms-json_parser = []
-transforms-kubernetes = ["k8s-openapi"]
+transforms-kubernetes = ["k8s-openapi","evmap","sources-kubernetes"]
 transforms-log_to_metric = []
 transforms-logfmt_parser = ["logfmt"]
 transforms-lua = ["rlua"]