chore(deps): Add 3rd party license file and CI checks (vectordotdev#17344)

* chore(deps): Add 3rd party license file and CI checks

* Add comments on the external crate license overrides

* Move installation of license tool to the environment prepare script

Author: bruceg

.github/workflows/test.yml

@@ -346,6 +346,9 @@ jobs:
       - name: Check cargo deny advisories/licenses
         if: needs.changes.outputs.dependencies == 'true' || needs.changes.outputs.deny == 'true'
         run: make check-deny
+      - name: Check that the 3rd-party license file is up to date
+        if: needs.changes.outputs.dependencies == 'true'
+        run: make check-licenses
       - name: Check Cue docs
         if: needs.changes.outputs.cue == 'true'
         run: make check-docs

LICENSE-3rdparty.csv

@@ -417,7 +417,7 @@ check: ## Run prerequisite code checks
 check-all: ## Check everything
 check-all: check-fmt check-clippy check-docs
 check-all: check-version check-examples check-component-features
-check-all: check-scripts check-deny check-component-docs
+check-all: check-scripts check-deny check-component-docs check-licenses
 
 .PHONY: check-component-features
 check-component-features: ## Check that all component features are setup properly
@@ -435,6 +435,10 @@ check-docs: ## Check that all /docs file are valid
 check-fmt: ## Check that all files are formatted properly
 	${MAYBE_ENVIRONMENT_EXEC} cargo vdev check fmt
 
+.PHONY: check-licenses
+check-licenses: ## Check that the 3rd-party license file is up to date
+	${MAYBE_ENVIRONMENT_EXEC} cargo vdev check licenses
+
 .PHONY: check-markdown
 check-markdown: ## Check that markdown is styled properly
 	${MAYBE_ENVIRONMENT_EXEC} cargo vdev check markdown

Makefile

@@ -0,0 +1,28 @@
+[overrides]
+"backon" = { origin = "https://github.com/Xuanwo/backon" }
+"bollard-stubs" = { origin = "https://github.com/fussybeaver/bollard" }
+"openssl-macros" = { origin = "https://github.com/sfackler/rust-openssl" }
+"serde_nanos" = { origin = "https://github.com/caspervonb/serde_nanos" }
+
+# These can go away once Vector starts using a release of the VRL crate with a
+# library field set up.
+"vrl" = { license = "MPL-2.0" }
+"vrl-compiler" = { license = "MPL-2.0" }
+"vrl-core" = { license = "MPL-2.0" }
+"vrl-diagnostic" = { license = "MPL-2.0" }
+"vrl-parser" = { license = "MPL-2.0" }
+"vrl-tests" = { license = "MPL-2.0" }
+
+# `ring` has a custom license that is mostly "ISC-style" but parts of it also fall under OpenSSL licensing.
+"ring-0.16.20" = { license = "ISC AND Custom" }
+
+# `rustls-webpki` doesn't specify their license in the metadata, but the file contains the ISC terms.
+"rustls-webpki-0.100.1" = { license = "ISC" }
+
+# `webpki` doesn't specify their license in the metadata, but the file contains the ISC terms.
+"webpki-0.21.4" = { license = "ISC" }
+"webpki-0.22.0" = { license = "ISC" }
+
+# `zerocopy` et al don't specify their licenses in the metadata, but the file contains the 2-clause BSD terms.
+"zerocopy-0.6.1" = { license = "BSD-2-Clause" }
+"zerocopy-derive-0.3.2" = { license = "BSD-2-Clause" }

license-tool.toml

@@ -17,6 +17,9 @@ fi
 if ! cargo deny --version >& /dev/null ; then
   rustup run stable cargo install cargo-deny --force --locked
 fi
+if ! rust-license-tool --help >& /dev/null ; then
+  cargo install --git https://github.com/DataDog/rust-license-tool
+fi
 
 cd scripts
 bundle install

scripts/environment/prepare.sh

@@ -0,0 +1,14 @@
+use anyhow::Result;
+
+use crate::app;
+
+/// Rebuild the 3rd-party license file.
+#[derive(clap::Args, Debug)]
+#[command()]
+pub struct Cli {}
+
+impl Cli {
+    pub fn exec(self) -> Result<()> {
+        app::exec("rust-license-tool", ["write"], true)
+    }
+}

vdev/src/commands/build/licenses.rs

@@ -1,6 +1,7 @@
 crate::cli_subcommands! {
     "Build, generate or regenerate components..."
     component_docs,
+    mod licenses,
     manifests,
     mod publish_metadata,
     release_cue,

vdev/src/commands/build/mod.rs

@@ -0,0 +1,14 @@
+use anyhow::Result;
+
+use crate::app;
+
+/// Check that the 3rd-party license file is up to date
+#[derive(clap::Args, Debug)]
+#[command()]
+pub struct Cli {}
+
+impl Cli {
+    pub fn exec(self) -> Result<()> {
+        app::exec("rust-license-tool", ["check"], true)
+    }
+}

vdev/src/commands/check/licenses.rs

@@ -7,6 +7,7 @@ crate::cli_subcommands! {
     events,
     mod examples,
     mod fmt,
+    mod licenses,
     mod markdown,
     mod rust,
     mod scripts,