feat(apache_metrics source): Initial implementation

.github/CODEOWNERS

@@ -78,6 +78,7 @@
 /src/sinks/tcp.rs @lukesteensen
 /src/sinks/util/auto_concurrency @bruceg
 
+/src/sources/apache_metrics/ @jszwedko
 /src/sources/docker.rs @fanatid
 /src/sources/generator.rs @bruceg
 /src/sources/journald.rs @bruceg

.meta/_partials/descriptions/_apache_metrics.toml

@@ -0,0 +1,3 @@
+description = """\
+[Apache HTTP Server][urls.apache] is a popular open-source HTTP server.
+"""

.meta/links.toml

@@ -17,6 +17,8 @@ program_transforms = "/components/?functions%5B%5D=program"
 
 [links.urls]
 add_company = "https://github.com/timberio/vector/blob/master/.meta/companies.toml"
+apache = "https://httpd.apache.org/"
+apache_mod_status = "http://httpd.apache.org/docs/current/mod/mod_status.html"
 arm = "https://en.wikipedia.org/wiki/ARM_architecture"
 aws_arm_g2_announcement = "https://aws.amazon.com/about-aws/whats-new/2019/12/announcing-new-amazon-ec2-m6g-c6g-and-r6g-instances-powered-by-next-generation-arm-based-aws-graviton2-processors/"
 aws_athena = "https://aws.amazon.com/athena/"

.meta/sources/apache_metrics.toml.erb

@@ -0,0 +1,130 @@
+[sources.apache_metrics]
+title = "Apache Metrics"
+noun = "Apache Metrics"
+beta = true
+common = false
+delivery_guarantee = "at_least_once"
+<%= render("_partials/descriptions/_apache_metrics.toml") %>
+features = [
+  "Scrape one or more Apache Server mod_status endpoints.",
+  "Automatically parse metrics into a lossless interoperable data model.",
+]
+function_category = "collect"
+output_types = ["metric"]
+requirements = {}
+strategies = ["daemon", "sidecar", "service"]
+through_description = "the [mod_status][urls.apache_mod_status] module"
+
+<%= render("_partials/fields/_component_options.toml", type: "source", name: "apache_metrics") %>
+
+[sources.apache_metrics.options.endpoints]
+type = "[string]"
+common = true
+required = true
+examples = [["http://localhost:8080/server-status/?auto"]]
+description = "mod_status endpoints to scrape metrics from."
+
+[sources.apache_metrics.options.scrape_interval_secs]
+type = "uint"
+common = true
+default = 15
+unit = "seconds"
+description = "The interval between scrapes, in seconds."
+
+[[sources.apache_metrics.examples]]
+label = "Extended"
+body = """\
+Given the following output from [mod_status][urls.apache_mod_status] with [ExtendedStatus On][urls.apache_extended_status]:
+
+```text title="Example mod_status payload"
+localhost
+ServerVersion: Apache/2.4.46 (Unix)
+ServerMPM: event
+Server Built: Aug  5 2020 23:20:17
+CurrentTime: Thursday, 03-Sep-2020 20:48:54 UTC
+RestartTime: Thursday, 03-Sep-2020 20:48:41 UTC
+ParentServerConfigGeneration: 1
+ParentServerMPMGeneration: 0
+ServerUptimeSeconds: 12
+ServerUptime: 12 seconds
+Load1: 0.75
+Load5: 0.59
+Load15: 0.76
+BusyWorkers: 1
+IdleWorkers: 74
+Processes: 3
+Stopping: 0
+BusyWorkers: 1
+IdleWorkers: 74
+ConnsTotal: 1
+ConnsAsyncWriting: 0
+ConnsAsyncKeepAlive: 0
+ConnsAsyncClosing: 0
+Scoreboard: ____S_____I______R____I_______KK___D__C__G_L____________W__________________.....................................................................................................................................................................................................................................................................................................................................
+```
+
+The following metrics will be generated.
+
+```text title="Example metrics"
+2020-09-09T19:26:05.473213405Z apache_access_total{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 30
+2020-09-09T19:26:05.473213405Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="closing"} = 0
+2020-09-09T19:26:05.473213405Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="keepalive"} = 0
+2020-09-09T19:26:05.473213405Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="total"} = 1
+2020-09-09T19:26:05.473213405Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="writing"} = 0
+2020-09-09T19:26:05.473213405Z apache_cpu_load{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 0.846154
+2020-09-09T19:26:05.473213405Z apache_cpu_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost",type="children_system"} = 0
+2020-09-09T19:26:05.473213405Z apache_cpu_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost",type="children_user"} = 0
+2020-09-09T19:26:05.473213405Z apache_cpu_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost",type="system"} = 0.02
+2020-09-09T19:26:05.473213405Z apache_cpu_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost",type="user"} = 0.2
+2020-09-09T19:26:05.473213405Z apache_duration_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 11
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="closing"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="dnslookup"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="finishing"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="idle_cleanup"} = 2
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="keepalive"} = 2
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="logging"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="open"} = 325
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="reading"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="sending"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="starting"} = 1
+2020-09-09T19:26:05.473213405Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="waiting"} = 64
+2020-09-09T19:26:05.473213405Z apache_sent_bytes_total{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 222208
+2020-09-09T19:26:05.473213405Z apache_uptime_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 26
+2020-09-09T19:26:05.473213405Z apache_workers{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="busy"} = 1
+2020-09-09T19:26:05.473213405Z apache_workers{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="idle"} = 74
+2020-09-09T19:26:05.474493420Z apache_up{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 0
+```\
+"""
+
+[[sources.apache_metrics.examples]]
+label = "Non-Extended"
+body = """\
+Given the following output from [mod_status][urls.apache_mod_status] with [ExtendedStatus Off][urls.apache_extended_status]:
+
+```text title="Example mod_status payload"
+```
+
+The following metrics will be generated.
+
+```text title="Example metrics"
+2020-09-09T19:24:20.475444391Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="closing"} = 0
+2020-09-09T19:24:20.475444391Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="keepalive"} = 0
+2020-09-09T19:24:20.475444391Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="total"} = 1
+2020-09-09T19:24:20.475444391Z apache_connections{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="writing"} = 0
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="closing"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="dnslookup"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="finishing"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="idle_cleanup"} = 2
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="keepalive"} = 2
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="logging"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="open"} = 325
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="reading"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="sending"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="starting"} = 1
+2020-09-09T19:24:20.475444391Z apache_scoreboard{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="waiting"} = 64
+2020-09-09T19:24:20.475444391Z apache_uptime_seconds_total{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 12
+2020-09-09T19:24:20.475444391Z apache_workers{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="busy"} = 1
+2020-09-09T19:24:20.475444391Z apache_workers{endpoint="http://localhost:8080/server-status?auto",host="localhost",state="idle"} = 74
+2020-09-09T19:24:20.476079976Z apache_up{endpoint="http://localhost:8080/server-status?auto",host="localhost"} = 0
+```\
+"""

Cargo.toml

@@ -250,6 +250,7 @@ kubernetes = ["k8s-openapi", "evmap"]
 
 # Sources
 sources = [
+  "sources-apache_metrics",
   "sources-docker",
   "sources-file",
   "sources-generator",
@@ -268,6 +269,7 @@ sources = [
   "sources-vector",
   "sources-kubernetes-logs",
 ]
+sources-apache_metrics = []
 sources-docker = ["bollard"]
 sources-file = ["bytesize"]
 sources-generator = []

src/internal_events/apache_metrics.rs

@@ -0,0 +1,115 @@
+use super::InternalEvent;
+use crate::sources::apache_metrics;
+use http::Uri;
+use metrics::{counter, timing};
+use std::time::Instant;
+
+#[derive(Debug)]
+pub struct ApacheMetricsEventReceived {
+    pub byte_size: usize,
+    pub count: usize,
+}
+
+impl InternalEvent for ApacheMetricsEventReceived {
+    fn emit_logs(&self) {
+        debug!(message = "Scraped events.", ?self.count);
+    }
+
+    fn emit_metrics(&self) {
+        counter!(
+            "events_processed", self.count as u64,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+        counter!(
+            "bytes_processed", self.byte_size as u64,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+    }
+}
+
+#[derive(Debug)]
+pub struct ApacheMetricsRequestCompleted {
+    pub start: Instant,
+    pub end: Instant,
+}
+
+impl InternalEvent for ApacheMetricsRequestCompleted {
+    fn emit_logs(&self) {
+        debug!(message = "Request completed.");
+    }
+
+    fn emit_metrics(&self) {
+        counter!("requests_completed", 1,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+        timing!("request_duration_nanoseconds", self.start, self.end,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+    }
+}
+
+#[derive(Debug)]
+pub struct ApacheMetricsParseError {
+    pub error: apache_metrics::ParseError,
+    pub url: Uri,
+}
+
+impl InternalEvent for ApacheMetricsParseError {
+    fn emit_logs(&self) {
+        error!(message = "Parsing error.", url = %self.url, error = %self.error);
+        debug!(
+            message = %format!("Parse error:\n\n{}\n\n", self.error),
+            url = %self.url,
+            rate_limit_secs = 10
+        );
+    }
+
+    fn emit_metrics(&self) {
+        counter!("parse_errors", 1,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+    }
+}
+
+#[derive(Debug)]
+pub struct ApacheMetricsErrorResponse {
+    pub code: hyper::StatusCode,
+    pub url: Uri,
+}
+
+impl InternalEvent for ApacheMetricsErrorResponse {
+    fn emit_logs(&self) {
+        error!(message = "HTTP error response.", url = %self.url, code = %self.code);
+    }
+
+    fn emit_metrics(&self) {
+        counter!("http_error_response", 1,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+    }
+}
+
+#[derive(Debug)]
+pub struct ApacheMetricsHttpError {
+    pub error: hyper::Error,
+    pub url: Uri,
+}
+
+impl InternalEvent for ApacheMetricsHttpError {
+    fn emit_logs(&self) {
+        error!(message = "HTTP request processing error.", url = %self.url, error = %self.error);
+    }
+
+    fn emit_metrics(&self) {
+        counter!("http_request_errors", 1,
+            "component_kind" => "source",
+            "component_type" => "apache_metrics",
+        );
+    }
+}

src/internal_events/mod.rs

@@ -3,6 +3,8 @@ use std::borrow::Cow;
 mod add_fields;
 mod add_tags;
 mod ansi_stripper;
+#[cfg(feature = "sources-apache_metrics")]
+mod apache_metrics;
 mod auto_concurrency;
 mod aws_kinesis_streams;
 mod blackhole;
@@ -77,6 +79,8 @@ pub mod kubernetes;
 pub use self::add_fields::*;
 pub use self::add_tags::*;
 pub use self::ansi_stripper::*;
+#[cfg(feature = "sources-apache_metrics")]
+pub use self::apache_metrics::*;
 pub use self::auto_concurrency::*;
 pub use self::aws_kinesis_streams::*;
 pub use self::blackhole::*;