enhancement(vdev): Load compose files and inject network block (vectordotdev#17025)

* enhancement(vdev): Load compose files and inject network block

* remove temp file after compose.stop() is called

* Make config service for serialization include more fields

* Add compose support for webdhfs

* remove renaming setting from serde annotations

* Use from_iter() in network injection

* utilize tempfile for creation of the rewritten compose file

* chore(vdev): Make network non optional in Compose struct

* Merge via cherry-pick network name PR, cargo fmt

* Remove unneeded comments

* Reintroduce path in Compose struct, clean up comments

* add cargo.toml

* Utilize tempfile_in() to fix relative file path resolution of compose files

* Restore proxy network on nginx integration test

* Make Compose run command command use the original file path if environment is active

* Conditional compile for Unix, prevent windows CI failure

* Condtional compilation fix

* Using Value for fields not used

* Remove conditional compilation flags to ensure temp_file created on Windows

* Allow deadcode for windows

* Remove unneeded to_string()

Author: jonathanpv

scripts/integration/amqp/compose.yaml

@@ -5,7 +5,3 @@ services:
     image: docker.io/rabbitmq:${CONFIG_VERSION}
     ports:
     - 5672:5672
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/aws/compose.yaml

@@ -14,7 +14,3 @@ services:
     volumes:
     - $DOCKER_SOCKET:/var/run/docker.sock
     - $HOME/.aws/:/home/.aws/
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/azure/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: azurite --blobHost 0.0.0.0 --loose
     volumes:
     - /var/run:/var/run
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/chronicle/compose.yaml

@@ -10,7 +10,3 @@ services:
     command:
     - -p
     - /public.pem
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/clickhouse/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   clickhouse:
     image: docker.io/yandex/clickhouse-server:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/databend/compose.yaml

@@ -19,7 +19,3 @@ services:
       - minio
     healthcheck:
       test: "curl -f localhost:8080/v1/health || exit 1"
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/datadog-agent/compose.yaml

@@ -26,7 +26,3 @@ services:
     - DD_CMD_PORT=5002
     - DD_USE_DOGSTATSD=false
     - DD_HOSTNAME=datadog-trace-agent
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/datadog-traces/compose.yaml

@@ -29,7 +29,3 @@ services:
     - DD_APM_MAX_MEMORY=0
     - DD_APM_MAX_CPU_PERCENT=0
     - DD_HOSTNAME=datadog-trace-agent-to-vector
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/dnstap/compose.yaml

@@ -11,9 +11,5 @@ services:
       - dnstap-sockets:/bind2/etc/bind/socket
       - dnstap-sockets:/bind3/etc/bind/socket
 
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-
 volumes:
   dnstap-sockets: {}

scripts/integration/elasticsearch/compose.yaml

@@ -25,7 +25,3 @@ services:
     - ES_JAVA_OPTS=-Xms400m -Xmx400m
     volumes:
     - ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/eventstoredb/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: --insecure --stats-period-sec=1
     volumes:
     - ../../../tests/data:/etc/vector:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/gcp/compose.yaml

@@ -6,7 +6,3 @@ services:
     environment:
     - PUBSUB_PROJECT1=testproject,topic1:subscription1
     - PUBSUB_PROJECT2=sourceproject,topic2:subscription2
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/http-client/compose.yaml

@@ -29,7 +29,3 @@ services:
     - ../../../tests/data/http-client/serve:/data
     - ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
     - ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/humio/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   humio:
     image: docker.io/humio/humio:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/influxdb/compose.yaml

@@ -19,7 +19,3 @@ services:
     command: influxd --reporting-disabled
     environment:
     - INFLUXDB_REPORTING_DISABLED=true
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/kafka/compose.yaml

@@ -33,7 +33,3 @@ services:
     volumes:
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/logstash/compose.yaml

@@ -12,7 +12,3 @@ services:
     - /dev/null:/usr/share/logstash/pipeline/logstash.yml
     - ../../../tests/data/host.docker.internal.crt:/tmp/logstash.crt
     - ../../../tests/data/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/loki/compose.yaml

@@ -4,7 +4,3 @@ services:
   loki:
     image: docker.io/grafana/loki:${CONFIG_VERSION}
     command: -config.file=/etc/loki/local-config.yaml -auth.enabled=true
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/mongodb/compose.yaml

@@ -30,7 +30,3 @@ services:
     - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017
     - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor
     - MONGODB_REPLICA_SET_KEY=vector
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/nats/compose.yaml

@@ -43,7 +43,3 @@ services:
     - /usr/share/nats/config/nats-jwt.conf
     volumes:
     - ../../../tests/data/nats:/usr/share/nats/config
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/nginx/compose.yaml

@@ -22,6 +22,4 @@ services:
     - proxy
 
 networks:
-  default:
-    name: ${VECTOR_NETWORK}
   proxy: {}

scripts/integration/opentelemetry/compose.yaml

@@ -5,7 +5,3 @@ services:
     image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION}
     volumes:
     - ../../../tests/data/opentelemetry/config.yaml:/etc/otelcol-contrib/config.yaml
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/postgres/compose.yaml

@@ -12,9 +12,5 @@ services:
     - ../../../tests/data/postgres-init.sh:/postgres-init.sh:ro
     - ../../../tests/data/ca:/certs:ro
 
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-
 volumes:
   socket: {}

scripts/integration/prometheus/compose.yaml

@@ -21,7 +21,3 @@ services:
     command: --config.file=/etc/vector/prometheus.yaml
     volumes:
     - ../../../tests/data:/etc/vector:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/pulsar/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: bin/pulsar standalone
     ports:
     - 6650:6650
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/redis/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   redis:
     image: docker.io/redis:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/shutdown/compose.yaml

@@ -31,7 +31,3 @@ services:
     volumes:
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/splunk/compose.yaml

@@ -13,7 +13,3 @@ services:
     - 8000:8000
     - 8088:8088
     - 8089:8089
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/webhdfs/compose.yaml

@@ -30,7 +30,3 @@ services:
       interval: 5s
       timeout: 5s
       retries: 3
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

vdev/Cargo.toml

@@ -35,5 +35,5 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.95"
 serde_yaml = "0.9.19"
 sha2 = "0.10.6"
-tempfile = "3.4.0"
+tempfile = "3.5.0"
 toml = { version = "0.7.2", default-features = false, features = ["parse"] }

vdev/src/testing/config.rs

@@ -5,7 +5,7 @@ use std::{env, fs};
 use anyhow::{bail, Context, Result};
 use hashlink::LinkedHashMap;
 use itertools::{self, Itertools};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_yaml::Value;
 
 use crate::{app, util};
@@ -37,20 +37,49 @@ impl RustToolchainConfig {
     }
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComposeConfig {
     pub services: BTreeMap<String, ComposeService>,
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub volumes: BTreeMap<String, Value>,
+    #[serde(default)]
+    pub networks: BTreeMap<String, BTreeMap<String, String>>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComposeService {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hostname: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub container_name: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub build: Option<Value>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub command: Option<Command>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ports: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub env_file: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volumes: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub environment: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub depends_on: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub healthcheck: Option<Value>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(untagged)]
+pub enum Command {
+    Single(String),
+    Multiple(Vec<String>),
 }
 
 impl ComposeConfig {
-    #[cfg(unix)]
     pub fn parse(path: &Path) -> Result<Self> {
         let contents =
             fs::read_to_string(path).with_context(|| format!("failed to read {path:?}"))?;

vdev/src/testing/integration.rs

@@ -1,8 +1,8 @@
-use std::{path::Path, path::PathBuf, process::Command};
+use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command};
 
 use anyhow::{bail, Context, Result};
+use tempfile::{Builder, NamedTempFile};
 
-#[cfg(unix)]
 use super::config::ComposeConfig;
 use super::config::{Environment, IntegrationTestConfig};
 use super::runner::{
@@ -135,30 +135,52 @@ impl IntegrationTest {
 }
 
 struct Compose {
-    path: PathBuf,
+    original_path: PathBuf,
     test_dir: PathBuf,
     env: Environment,
-    #[cfg(unix)]
+    #[cfg_attr(target_family = "windows", allow(dead_code))]
     config: ComposeConfig,
     network: String,
+    temp_file: NamedTempFile,
 }
 
 impl Compose {
     fn new(test_dir: PathBuf, env: Environment, network: String) -> Result<Option<Self>> {
-        let path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect();
-        match path.try_exists() {
-            Err(error) => Err(error).with_context(|| format!("Could not lookup {path:?}")),
+        let original_path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect();
+
+        match original_path.try_exists() {
+            Err(error) => Err(error).with_context(|| format!("Could not lookup {original_path:?}")),
             Ok(false) => Ok(None),
             Ok(true) => {
-                #[cfg(unix)]
-                let config = ComposeConfig::parse(&path)?;
+                let mut config = ComposeConfig::parse(&original_path)?;
+                // Inject the networks block
+                config.networks.insert(
+                    "default".to_string(),
+                    BTreeMap::from_iter([("name".to_string(), network.clone())]),
+                );
+
+                // Create a named tempfile, there may be resource leakage here in case of SIGINT
+                // Tried tempfile::tempfile() but this returns a File object without a usable path
+                // https://docs.rs/tempfile/latest/tempfile/#resource-leaking
+                let temp_file = Builder::new()
+                    .prefix("compose-temp-")
+                    .suffix(".yaml")
+                    .tempfile_in(&test_dir)
+                    .with_context(|| "Failed to create temporary compose file")?;
+
+                fs::write(
+                    temp_file.path(),
+                    serde_yaml::to_string(&config)
+                        .with_context(|| "Failed to serialize modified compose.yaml")?,
+                )?;
+
                 Ok(Some(Self {
-                    path,
+                    original_path,
                     test_dir,
                     env,
-                    #[cfg(unix)]
                     config,
                     network,
+                    temp_file,
                 }))
             }
         }
@@ -178,8 +200,19 @@ impl Compose {
         let mut command = CONTAINER_TOOL.clone();
         command.push("-compose");
         let mut command = Command::new(command);
+        // When the integration test environment is already active, the tempfile path does not
+        // exist because `Compose::new()` has not been called. In this case, the `stop` command
+        // needs to use the calculated path from the integration name instead of the nonexistent
+        // tempfile path. This is because `stop` doesn't go through the same logic as `start`
+        // and doesn't create a new tempfile before calling docker compose.
+        // If stop command needs to use some of the injected bits then we need to rebuild it
         command.arg("--file");
-        command.arg(&self.path);
+        if config.is_none() {
+            command.arg(&self.original_path);
+        } else {
+            command.arg(self.temp_file.path());
+        }
+
         command.args(args);
 
         command.current_dir(&self.test_dir);