diff --git a/.duvet/.gitignore b/.duvet/.gitignore new file mode 100644 index 0000000..a9a1bd3 --- /dev/null +++ b/.duvet/.gitignore @@ -0,0 +1 @@ +reports/ diff --git a/.duvet/config.toml b/.duvet/config.toml new file mode 100644 index 0000000..f6363ae --- /dev/null +++ b/.duvet/config.toml @@ -0,0 +1,12 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "duvet/**/*.rs" + +[report.html] +enabled = true +issue-link = "https://github.com/awslabs/duvet/issues" +blob-link = "https://github.com/awslabs/duvet/blob/${{ GITHUB_SHA || 'main' }}" + +[report.json] +enabled = true diff --git a/config/v0.4.0.json b/config/v0.4.0.json new file mode 100644 index 0000000..0b468cf --- /dev/null +++ b/config/v0.4.0.json @@ -0,0 +1,197 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://awslabs.github.io/duvet/config/v0.4.0.json", + "title": "Duvet Configuration", + "type": "object", + "properties": { + "$schema": { + "type": [ + "string", + "null" + ] + }, + "report": { + "$ref": "#/definitions/Report" + }, + "requirement": { + "type": "array", + "items": { + "$ref": "#/definitions/Requirement" + } + }, + "source": { + "type": "array", + "items": { + "$ref": "#/definitions/Source" + } + }, + "specification": { + "type": "array", + "items": { + "$ref": "#/definitions/Specification" + } + } + }, + "additionalProperties": false, + "definitions": { + "CommentStyle": { + "type": "object", + "properties": { + "content": { + "default": "//#", + "type": "string" + }, + "meta": { + "default": "//=", + "type": "string" + } + }, + "additionalProperties": false + }, + "DefaultType": { + "type": "string", + "enum": [ + "implementation", + "spec", + "test", + "exception", + "todo", + "implication" + ] + }, + "HtmlReport": { + "type": "object", + "properties": { + "blob-link": { + "anyOf": [ + { + "$ref": "#/definitions/TemplatedString" + }, + { + "type": "null" + } + ] + }, + "enabled": { + "default": true, + "type": "boolean" + }, + "issue-link": { + "anyOf": [ + { + "$ref": "#/definitions/TemplatedString" + }, + { + "type": "null" + } + ] + }, + "path": { + "default": ".duvet/reports/report.html", + "type": "string" + } + }, + "additionalProperties": false + }, + "JsonReport": { + "type": "object", + "properties": { + "enabled": { + "default": false, + "type": "boolean" + }, + "path": { + "default": ".duvet/reports/report.json", + "type": "string" + } + }, + "additionalProperties": false + }, + "Report": { + "type": "object", + "properties": { + "html": { + "anyOf": [ + { + "$ref": "#/definitions/HtmlReport" + }, + { + "type": "null" + } + ] + }, + "json": { + "anyOf": [ + { + "$ref": "#/definitions/JsonReport" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + "Requirement": { + "type": "object", + "required": [ + "pattern" + ], + "properties": { + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + "Source": { + "type": "object", + "required": [ + "pattern" + ], + "properties": { + "comment-style": { + "$ref": "#/definitions/CommentStyle" + }, + "pattern": { + "type": "string" + }, + "type": { + "$ref": "#/definitions/DefaultType" + } + }, + "additionalProperties": false + }, + "Specification": { + "type": "object", + "properties": { + "format": { + "anyOf": [ + { + "$ref": "#/definitions/SpecificationFormat" + }, + { + "type": "null" + } + ] + }, + "source": { + "default": "", + "type": "string" + } + }, + "additionalProperties": false + }, + "SpecificationFormat": { + "type": "string", + "enum": [ + "ietf", + "markdown" + ] + }, + "TemplatedString": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/config/v0.4.json b/config/v0.4.json new file mode 100644 index 0000000..862230a --- /dev/null +++ b/config/v0.4.json @@ -0,0 +1,197 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://awslabs.github.io/duvet/config/v0.4.json", + "title": "Duvet Configuration", + "type": "object", + "properties": { + "$schema": { + "type": [ + "string", + "null" + ] + }, + "report": { + "$ref": "#/definitions/Report" + }, + "requirement": { + "type": "array", + "items": { + "$ref": "#/definitions/Requirement" + } + }, + "source": { + "type": "array", + "items": { + "$ref": "#/definitions/Source" + } + }, + "specification": { + "type": "array", + "items": { + "$ref": "#/definitions/Specification" + } + } + }, + "additionalProperties": false, + "definitions": { + "CommentStyle": { + "type": "object", + "properties": { + "content": { + "default": "//#", + "type": "string" + }, + "meta": { + "default": "//=", + "type": "string" + } + }, + "additionalProperties": false + }, + "DefaultType": { + "type": "string", + "enum": [ + "implementation", + "spec", + "test", + "exception", + "todo", + "implication" + ] + }, + "HtmlReport": { + "type": "object", + "properties": { + "blob-link": { + "anyOf": [ + { + "$ref": "#/definitions/TemplatedString" + }, + { + "type": "null" + } + ] + }, + "enabled": { + "default": true, + "type": "boolean" + }, + "issue-link": { + "anyOf": [ + { + "$ref": "#/definitions/TemplatedString" + }, + { + "type": "null" + } + ] + }, + "path": { + "default": ".duvet/reports/report.html", + "type": "string" + } + }, + "additionalProperties": false + }, + "JsonReport": { + "type": "object", + "properties": { + "enabled": { + "default": false, + "type": "boolean" + }, + "path": { + "default": ".duvet/reports/report.json", + "type": "string" + } + }, + "additionalProperties": false + }, + "Report": { + "type": "object", + "properties": { + "html": { + "anyOf": [ + { + "$ref": "#/definitions/HtmlReport" + }, + { + "type": "null" + } + ] + }, + "json": { + "anyOf": [ + { + "$ref": "#/definitions/JsonReport" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, + "Requirement": { + "type": "object", + "required": [ + "pattern" + ], + "properties": { + "pattern": { + "type": "string" + } + }, + "additionalProperties": false + }, + "Source": { + "type": "object", + "required": [ + "pattern" + ], + "properties": { + "comment-style": { + "$ref": "#/definitions/CommentStyle" + }, + "pattern": { + "type": "string" + }, + "type": { + "$ref": "#/definitions/DefaultType" + } + }, + "additionalProperties": false + }, + "Specification": { + "type": "object", + "properties": { + "format": { + "anyOf": [ + { + "$ref": "#/definitions/SpecificationFormat" + }, + { + "type": "null" + } + ] + }, + "source": { + "default": "", + "type": "string" + } + }, + "additionalProperties": false + }, + "SpecificationFormat": { + "type": "string", + "enum": [ + "ietf", + "markdown" + ] + }, + "TemplatedString": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/duvet-core/src/artifact.rs b/duvet-core/src/artifact.rs new file mode 100644 index 0000000..b33b635 --- /dev/null +++ b/duvet-core/src/artifact.rs @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +/// Synchronizes a value to the file system +/// +/// When the `CI` environment variable is set, this method ensures the value matches +/// what is on disk. +pub fn sync(path: impl AsRef, value: impl AsRef) { + let path = path.as_ref(); + let value = value.as_ref(); + if std::env::var("CI").is_err() { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, value).unwrap(); + return; + } + + let actual = std::fs::read_to_string(path).unwrap(); + assert_eq!(actual, value); +} diff --git a/duvet-core/src/lib.rs b/duvet-core/src/lib.rs index c1df4ee..239d457 100644 --- a/duvet-core/src/lib.rs +++ b/duvet-core/src/lib.rs @@ -16,6 +16,8 @@ macro_rules! ensure { #[cfg(any(test, feature = "testing"))] pub mod testing; +#[cfg(any(test, feature = "testing"))] +pub mod artifact; mod cache; pub mod contents; pub mod diagnostic; diff --git a/duvet/Cargo.toml b/duvet/Cargo.toml index 725b145..b84b685 100644 --- a/duvet/Cargo.toml +++ b/duvet/Cargo.toml @@ -33,7 +33,9 @@ v_jsonescape = "0.7" [dev-dependencies] bolero = "0.12" +duvet-core = { version = "0.1", path = "../duvet-core", features = ["testing"] } insta = { version = "1", features = ["filters", "json"] } +schemars = "0.8" serde_json = "1" strip-ansi-escapes = "0.2" tempfile = "3" diff --git a/duvet/src/annotation.rs b/duvet/src/annotation.rs index a6d40dc..5ed108b 100644 --- a/duvet/src/annotation.rs +++ b/duvet/src/annotation.rs @@ -29,7 +29,7 @@ pub type AnnotationReferenceMap = pub async fn specifications( annotations: AnnotationSet, - spec_path: Option, + spec_path: Path, ) -> Result { let mut targets = TargetSet::new(); for anno in annotations.iter() { diff --git a/duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap b/duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap index 26693c6..a68956b 100644 --- a/duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__missing_new_line.snap @@ -1,6 +1,6 @@ --- source: duvet/src/comment/tests.rs -expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //# Here is my citation\"#)" +expression: "parse(\"//@=,//@#\",\nr#\"\n //@= https://example.com/spec.txt\n //@# Here is my citation\"#)" --- ( { @@ -8,7 +8,7 @@ expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n source: "file.rs", anno_line: 2, original_target: "https://example.com/spec.txt", - original_text: "https://example.com/spec.txt\n //# Here is my citation", + original_text: "https://example.com/spec.txt\n //@# Here is my citation", original_quote: "Here is my citation", anno: Citation, target: "https://example.com/spec.txt", diff --git a/duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap index 9bd0fd8..5874e34 100644 --- a/duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_citation.snap @@ -1,6 +1,6 @@ --- source: duvet/src/comment/tests.rs -expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //# Here is my citation\n \"#)" +expression: "parse(\"//@=,//@#\",\nr#\"\n //@= https://example.com/spec.txt\n //@# Here is my citation\n \"#)" --- ( { @@ -8,7 +8,7 @@ expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n source: "file.rs", anno_line: 2, original_target: "https://example.com/spec.txt", - original_text: "https://example.com/spec.txt\n //# Here is my citation", + original_text: "https://example.com/spec.txt\n //@# Here is my citation", original_quote: "Here is my citation", anno: Citation, target: "https://example.com/spec.txt", diff --git a/duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap index 2037c76..5bfb8fe 100644 --- a/duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_exception.snap @@ -1,6 +1,6 @@ --- source: duvet/src/comment/tests.rs -expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //= type=exception\n //= reason=This isn't possible currently\n //# Here is my citation\n \"#)" +expression: "parse(\"//@=,//@#\",\nr#\"\n //@= https://example.com/spec.txt\n //@= type=exception\n //@= reason=This isn't possible currently\n //@# Here is my citation\n \"#)" --- ( { @@ -8,7 +8,7 @@ expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n source: "file.rs", anno_line: 2, original_target: "https://example.com/spec.txt", - original_text: "https://example.com/spec.txt\n //= type=exception\n //= reason=This isn't possible currently\n //# Here is my citation", + original_text: "https://example.com/spec.txt\n //@= type=exception\n //@= reason=This isn't possible currently\n //@# Here is my citation", original_quote: "Here is my citation", anno: Exception, target: "https://example.com/spec.txt", diff --git a/duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap index a343bac..738967e 100644 --- a/duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_test.snap @@ -1,6 +1,6 @@ --- source: duvet/src/comment/tests.rs -expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //= type=test\n //# Here is my citation\n \"#)" +expression: "parse(\"//@=,//@#\",\nr#\"\n //@= https://example.com/spec.txt\n //@= type=test\n //@# Here is my citation\n \"#)" --- ( { @@ -8,7 +8,7 @@ expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n source: "file.rs", anno_line: 2, original_target: "https://example.com/spec.txt", - original_text: "https://example.com/spec.txt\n //= type=test\n //# Here is my citation", + original_text: "https://example.com/spec.txt\n //@= type=test\n //@# Here is my citation", original_quote: "Here is my citation", anno: Test, target: "https://example.com/spec.txt", diff --git a/duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap b/duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap index 9761f55..3aacbfa 100644 --- a/duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap +++ b/duvet/src/comment/snapshots/duvet__comment__tests__type_todo.snap @@ -1,6 +1,6 @@ --- source: duvet/src/comment/tests.rs -expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n //= type=todo\n //= feature=cool-things\n //= tracking-issue=123\n //# Here is my citation\n \"#)" +expression: "parse(\"//@=,//@#\",\nr#\"\n //@= https://example.com/spec.txt\n //@= type=todo\n //@= feature=cool-things\n //@= tracking-issue=123\n //@# Here is my citation\n \"#)" --- ( { @@ -8,7 +8,7 @@ expression: "parse(\"//=,//#\",\nr#\"\n //= https://example.com/spec.txt\n source: "file.rs", anno_line: 2, original_target: "https://example.com/spec.txt", - original_text: "https://example.com/spec.txt\n //= type=todo\n //= feature=cool-things\n //= tracking-issue=123\n //# Here is my citation", + original_text: "https://example.com/spec.txt\n //@= type=todo\n //@= feature=cool-things\n //@= tracking-issue=123\n //@# Here is my citation", original_quote: "Here is my citation", anno: Todo, target: "https://example.com/spec.txt", diff --git a/duvet/src/comment/tests.rs b/duvet/src/comment/tests.rs index 1a2b7c4..b2d806f 100644 --- a/duvet/src/comment/tests.rs +++ b/duvet/src/comment/tests.rs @@ -13,7 +13,8 @@ fn parse(pattern: &str, value: &str) -> (AnnotationSet, Vec) { macro_rules! snapshot { ($name:ident, $value:expr) => { - snapshot!($name, "//=,//#", $value); + // use a different pattern so we don't register these tests as part of the duvet report + snapshot!($name, "//@=,//@#", $value); }; ($name:ident, $pattern:expr, $value:expr) => { #[test] @@ -38,58 +39,58 @@ macro_rules! snapshot { snapshot!( content_without_meta, r#" - //# This is some content without meta + //@# This is some content without meta "# ); snapshot!( meta_without_content, r#" - //= type=todo + //@= type=todo "# ); snapshot!( type_citation, r#" - //= https://example.com/spec.txt - //# Here is my citation + //@= https://example.com/spec.txt + //@# Here is my citation "# ); snapshot!( type_test, r#" - //= https://example.com/spec.txt - //= type=test - //# Here is my citation + //@= https://example.com/spec.txt + //@= type=test + //@# Here is my citation "# ); snapshot!( type_todo, r#" - //= https://example.com/spec.txt - //= type=todo - //= feature=cool-things - //= tracking-issue=123 - //# Here is my citation + //@= https://example.com/spec.txt + //@= type=todo + //@= feature=cool-things + //@= tracking-issue=123 + //@# Here is my citation "# ); snapshot!( type_exception, r#" - //= https://example.com/spec.txt - //= type=exception - //= reason=This isn't possible currently - //# Here is my citation + //@= https://example.com/spec.txt + //@= type=exception + //@= reason=This isn't possible currently + //@# Here is my citation "# ); snapshot!( missing_new_line, r#" - //= https://example.com/spec.txt - //# Here is my citation"# + //@= https://example.com/spec.txt + //@# Here is my citation"# ); diff --git a/duvet/src/comment/tokenizer.rs b/duvet/src/comment/tokenizer.rs index 8efb3d5..dbafe0f 100644 --- a/duvet/src/comment/tokenizer.rs +++ b/duvet/src/comment/tokenizer.rs @@ -126,8 +126,8 @@ mod tests { $name, $input, Pattern { - meta: "//=".into(), - content: "//#".into(), + meta: "//@=".into(), + content: "//@#".into(), } ); }; @@ -146,25 +146,25 @@ mod tests { snapshot_test!( basic, r#" - //= thing goes here - //= meta=foo - //= meta2 = bar - //# content goes - //# here + //@= thing goes here + //@= meta=foo + //@= meta2 = bar + //@# content goes + //@# here "# ); snapshot_test!( only_unnamed, r#" - //= this is meta - //= this is other meta + //@= this is meta + //@= this is other meta "# ); snapshot_test!( duplicate_meta, r#" - //= meta=1 - //= meta=2 + //@= meta=1 + //@= meta=2 "# ); snapshot_test!( diff --git a/duvet/src/config.rs b/duvet/src/config.rs new file mode 100644 index 0000000..98e4ccf --- /dev/null +++ b/duvet/src/config.rs @@ -0,0 +1,128 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{extract::Extraction, Result}; +use duvet_core::{path::Path, vfs}; +use std::sync::Arc; + +pub mod schema; + +#[derive(Clone, Debug)] +pub struct Config { + pub sources: Vec, + pub requirements: Vec, + pub specifications: Vec, + pub report: Report, + pub requirements_path: Path, + pub download_path: Path, +} + +impl Config { + pub async fn load_specifications(&self) -> Result { + let download_path = &self.download_path; + let requirements_path = &self.requirements_path; + + for spec in &self.specifications { + Extraction { + download_path, + base_path: Some(download_path), + target: spec.target.clone(), + out: requirements_path, + extension: "toml", + // don't log to reduce noise + log: false, + } + .exec() + .await?; + } + + Ok(self.specifications.len()) + } +} + +#[derive(Clone, Debug)] +pub struct Source { + pub pattern: String, + pub root: Path, + pub comment_style: crate::comment::Pattern, + pub default_type: crate::annotation::AnnotationType, +} + +#[derive(Clone, Debug)] +pub struct Requirement { + pub pattern: String, + pub root: Path, +} + +#[derive(Clone, Debug)] +pub struct Report { + pub html: HtmlReport, + pub json: JsonReport, +} + +#[derive(Clone, Debug)] +pub struct HtmlReport { + pub enabled: bool, + pub path: Path, + pub blob_link: Option>, + pub issue_link: Option>, +} + +impl HtmlReport { + pub fn path(&self) -> Option<&Path> { + Some(&self.path).filter(|_| self.enabled) + } +} + +#[derive(Clone, Debug)] +pub struct JsonReport { + pub enabled: bool, + pub path: Path, +} + +impl JsonReport { + pub fn path(&self) -> Option<&Path> { + Some(&self.path).filter(|_| self.enabled) + } +} + +#[derive(Clone, Debug)] +pub struct Specification { + pub target: Arc, +} + +pub async fn load(path: Path, root: Path) -> Result> { + let file = vfs::read_string(path.clone()).await?; + let schema: Arc = file.as_toml().await?; + + let mut sources = vec![]; + let mut requirements = vec![]; + let mut specifications = vec![]; + + schema.load_sources(&mut sources, &root)?; + schema.load_requirements(&mut requirements, &root)?; + schema.load_specifications(&mut specifications, &root)?; + + let requirements_path = schema.requirements_path(&path, &root); + let download_path = schema.download_path(&path, &root); + let report = schema.report(&path, &root); + + Ok(Arc::new(Config { + sources, + requirements, + specifications, + requirements_path, + download_path, + report, + })) +} + +pub async fn default_path_and_root() -> Option<(Path, Path)> { + let root = duvet_core::env::current_dir().ok()?; + let path = root.join(".duvet").join("config.toml"); + + // check to see if it exists + let _ = vfs::read_metadata(&path).await.ok()?; + + Some((path, root)) +} diff --git a/duvet/src/config/schema.rs b/duvet/src/config/schema.rs new file mode 100644 index 0000000..d84c350 --- /dev/null +++ b/duvet/src/config/schema.rs @@ -0,0 +1,149 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{str::FromStr, sync::Arc}; + +use crate::{config, Result}; +use duvet_core::{error, path::Path}; +use serde::{de, Deserialize}; + +pub mod v0_4_0; + +#[derive(Debug, Deserialize)] +#[serde(tag = "$schema", deny_unknown_fields)] +pub enum Schema { + #[serde( + rename = "https://awslabs.github.io/duvet/config/v0.4.0.json", + alias = "https://awslabs.github.io/duvet/config/v0.4.0.json#", + alias = "https://awslabs.github.io/duvet/config/v0.4.json", + alias = "https://awslabs.github.io/duvet/config/v0.4.json#" + )] + V1_0_0(v0_4_0::Schema), +} + +impl Schema { + pub fn load_sources(&self, sources: &mut Vec, root: &Path) -> Result { + match self { + Schema::V1_0_0(schema) => schema.load_sources(sources, root), + } + } + + pub fn load_requirements( + &self, + requirements: &mut Vec, + root: &Path, + ) -> Result { + match self { + Schema::V1_0_0(schema) => schema.load_requirements(requirements, root), + } + } + + pub fn load_specifications( + &self, + specifications: &mut Vec, + root: &Path, + ) -> Result { + match self { + Schema::V1_0_0(schema) => schema.load_specifications(specifications, root), + } + } + + pub fn download_path(&self, config: &Path, root: &Path) -> Path { + match self { + Schema::V1_0_0(schema) => schema.download_path(config, root), + } + } + + pub fn requirements_path(&self, config: &Path, root: &Path) -> Path { + match self { + Schema::V1_0_0(schema) => schema.requirements_path(config, root), + } + } + + pub fn report(&self, config: &Path, root: &Path) -> config::Report { + match self { + Schema::V1_0_0(schema) => schema.report(config, root), + } + } +} + +#[derive(Clone, Debug)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct TemplatedString(Arc); + +impl FromStr for TemplatedString { + type Err = crate::Error; + + fn from_str(s: &str) -> std::result::Result { + let mut out = String::new(); + + for (idx, part) in s.split("${{").enumerate() { + if idx == 0 { + out.push_str(part); + continue; + } + + let close = "}}"; + let (expr, rest) = part + .split_once(close) + .ok_or_else(|| error!("expected {close:?}"))?; + + let mut value = None; + + for choice in expr.split("||") { + let choice = choice.trim(); + if let Some(choice) = choice.strip_prefix('\'') { + let choice = choice.trim_end_matches('\''); + value = Some(choice.to_string()); + break; + } else if let Ok(v) = std::env::var(choice) { + value = Some(v); + break; + } + } + + let Some(value) = value else { + return Err(error!("failed to evaluate expression: {expr:?}")); + }; + + out.push_str(&value); + out.push_str(rest); + } + + Ok(Self(out.into())) + } +} + +impl From<&TemplatedString> for Arc { + fn from(value: &TemplatedString) -> Self { + value.0.clone() + } +} + +impl<'de> serde::de::Deserialize<'de> for TemplatedString { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + use core::fmt; + + struct Visitor; + + impl de::Visitor<'_> for Visitor { + type Value = TemplatedString; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("string") + } + + fn visit_str(self, tmpl: &str) -> Result + where + E: de::Error, + { + tmpl.parse().map_err(de::Error::custom) + } + } + + deserializer.deserialize_str(Visitor) + } +} diff --git a/duvet/src/config/schema/v0_4_0.rs b/duvet/src/config/schema/v0_4_0.rs new file mode 100644 index 0000000..652e1ba --- /dev/null +++ b/duvet/src/config/schema/v0_4_0.rs @@ -0,0 +1,356 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + annotation::AnnotationType, + config, + target::{Target, TargetPath}, +}; +use duvet_core::{diagnostic::IntoDiagnostic, path::Path, Result}; +use serde::Deserialize; +use std::sync::Arc; + +use super::TemplatedString; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct Schema { + #[serde(default, rename = "source")] + pub sources: Arc<[Source]>, + + #[serde(default, rename = "requirement")] + pub requirements: Arc<[Requirement]>, + + #[serde(default)] + pub report: Arc, + + #[serde(default, rename = "specification")] + pub specifications: Arc<[Specification]>, + + #[serde(rename = "$schema")] + _schema: Option>, +} + +impl Schema { + pub fn load_sources(&self, sources: &mut Vec, root: &Path) -> Result { + for source in self.sources.iter() { + sources.push(config::Source { + // TODO add context to error + pattern: source.pattern.parse().into_diagnostic()?, + comment_style: (&source.comment_style).into(), + default_type: source.default_type.into(), + root: root.clone(), + }); + } + + Ok(()) + } + + pub fn load_requirements( + &self, + requirements: &mut Vec, + root: &Path, + ) -> Result { + // include several default paths + for pattern in [ + ".duvet/requirements/**/*.toml", + ".duvet/todos/**/*.toml", + ".duvet/exceptions/**/*.toml", + ] { + requirements.push(config::Requirement { + pattern: pattern.parse().into_diagnostic()?, + root: root.clone(), + }) + } + + for requirement in self.requirements.iter() { + requirements.push(config::Requirement { + // TODO add context to error + pattern: requirement.pattern.parse().into_diagnostic()?, + root: root.clone(), + }); + } + + Ok(()) + } + + pub fn load_specifications( + &self, + specifications: &mut Vec, + _root: &Path, + ) -> Result { + for spec in self.specifications.iter() { + let path = spec.source.parse::()?; + let format = spec + .format + .map(From::from) + .unwrap_or_else(|| crate::specification::Format::Auto); + + let target = Target { path, format }.into(); + specifications.push(config::Specification { target }); + } + + Ok(()) + } + + pub fn download_path(&self, config: &Path, _root: &Path) -> Path { + config.parent().unwrap().join("specifications").into() + } + + pub fn requirements_path(&self, config: &Path, _root: &Path) -> Path { + config.parent().unwrap().join("requirements").into() + } + + pub fn report(&self, _config: &Path, _root: &Path) -> config::Report { + (&*self.report).into() + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct Source { + pub pattern: String, + #[serde(default, rename = "comment-style")] + pub comment_style: CommentStyle, + #[serde(rename = "type", default)] + pub default_type: DefaultType, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub enum DefaultType { + #[default] + #[serde(rename = "implementation")] + Implementation, + #[serde(rename = "spec")] + Spec, + #[serde(rename = "test")] + Test, + #[serde(rename = "exception")] + Exception, + #[serde(rename = "todo")] + Todo, + #[serde(rename = "implication")] + Implication, +} + +impl From for AnnotationType { + fn from(value: DefaultType) -> Self { + match value { + DefaultType::Implementation => Self::Citation, + DefaultType::Spec => Self::Spec, + DefaultType::Test => Self::Test, + DefaultType::Todo => Self::Todo, + DefaultType::Exception => Self::Exception, + DefaultType::Implication => Self::Implication, + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct CommentStyle { + #[serde(default = "default_meta")] + pub meta: Arc, + #[serde(default = "default_content")] + pub content: Arc, +} + +fn default_meta() -> Arc { + Arc::from("//=") +} + +fn default_content() -> Arc { + Arc::from("//#") +} + +impl Default for CommentStyle { + fn default() -> Self { + Self { + meta: default_meta(), + content: default_content(), + } + } +} + +impl From<&CommentStyle> for crate::comment::Pattern { + fn from(value: &CommentStyle) -> Self { + Self { + meta: value.meta.clone(), + content: value.content.clone(), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct Requirement { + pub pattern: String, +} + +#[derive(Clone, Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct Report { + #[serde(default)] + pub html: Option, + #[serde(default)] + pub json: Option, +} + +impl From<&Report> for config::Report { + fn from(value: &Report) -> Self { + Self { + html: value + .html + .as_ref() + .map(From::from) + .unwrap_or_else(|| (&HtmlReport::default()).into()), + json: value + .json + .as_ref() + .map(From::from) + .unwrap_or_else(|| (&JsonReport::default()).into()), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct HtmlReport { + #[serde(default = "HtmlReport::default_enabled")] + pub enabled: bool, + #[serde(default = "HtmlReport::default_path")] + pub path: String, + #[serde(default, rename = "blob-link")] + pub blob_link: Option, + #[serde(default, rename = "issue-link")] + pub issue_link: Option, +} + +impl Default for HtmlReport { + fn default() -> Self { + Self { + enabled: Self::default_enabled(), + path: Self::default_path(), + blob_link: None, + issue_link: None, + } + } +} + +impl HtmlReport { + fn default_enabled() -> bool { + true + } + + fn default_path() -> String { + ".duvet/reports/report.html".into() + } +} + +impl From<&HtmlReport> for config::HtmlReport { + fn from(value: &HtmlReport) -> Self { + Self { + enabled: value.enabled, + path: value.path.as_str().into(), + issue_link: value.issue_link.as_ref().map(From::from), + blob_link: value.blob_link.as_ref().map(From::from), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct JsonReport { + #[serde(default = "JsonReport::default_enabled")] + pub enabled: bool, + #[serde(default = "JsonReport::default_path")] + pub path: String, +} + +impl Default for JsonReport { + fn default() -> Self { + Self { + enabled: Self::default_enabled(), + path: Self::default_path(), + } + } +} + +impl JsonReport { + fn default_enabled() -> bool { + false + } + + fn default_path() -> String { + ".duvet/reports/report.json".into() + } +} + +impl From<&JsonReport> for config::JsonReport { + fn from(value: &JsonReport) -> Self { + Self { + enabled: value.enabled, + path: value.path.as_str().into(), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub struct Specification { + #[serde(default)] + pub source: String, + pub format: Option, +} + +#[derive(Copy, Clone, Debug, Deserialize)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +pub enum SpecificationFormat { + #[serde(rename = "ietf", alias = "IETF")] + Ietf, + #[serde(rename = "markdown", alias = "md")] + Markdown, +} + +impl From for crate::specification::Format { + fn from(value: SpecificationFormat) -> Self { + match value { + SpecificationFormat::Ietf => Self::Ietf, + SpecificationFormat::Markdown => Self::Markdown, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_test() { + let mut schema = schemars::schema_for!(Schema); + + let metadata = schema.schema.metadata(); + metadata.title = Some("Duvet Configuration".into()); + metadata.id = Some("https://awslabs.github.io/duvet/config/v0.4.0.json".into()); + duvet_core::artifact::sync( + concat!(env!("CARGO_MANIFEST_DIR"), "/../config/v0.4.0.json"), + serde_json::to_string_pretty(&schema).unwrap(), + ); + + let metadata = schema.schema.metadata(); + metadata.id = Some("https://awslabs.github.io/duvet/config/v0.4.json".into()); + duvet_core::artifact::sync( + concat!(env!("CARGO_MANIFEST_DIR"), "/../config/v0.4.json"), + serde_json::to_string_pretty(&schema).unwrap(), + ); + } +} diff --git a/duvet/src/extract.rs b/duvet/src/extract.rs index 2586a0f..51e943a 100644 --- a/duvet/src/extract.rs +++ b/duvet/src/extract.rs @@ -5,15 +5,16 @@ use crate::{ annotation::AnnotationLevel, + project::Project, specification::{Format, Line, Section, Specification}, target::{self, Target, TargetPath}, Result, }; use clap::Parser; -use duvet_core::{diagnostic::IntoDiagnostic, path::Path}; +use duvet_core::{diagnostic::IntoDiagnostic, error, path::Path, progress}; use lazy_static::lazy_static; use regex::{Regex, RegexSet}; -use std::{fs::OpenOptions, io::BufWriter, path::PathBuf, sync::Arc}; +use std::{fs::OpenOptions, io::BufWriter, sync::Arc}; #[cfg(test)] mod tests; @@ -53,65 +54,118 @@ pub struct Extract { extension: String, #[clap(short, long, default_value = ".")] - out: PathBuf, + out: Path, - /// Path to store the collection of spec files - /// - /// The collection of spec files are stored in a folder called `specs`. The - /// `specs` folder is stored in the current directory by default. Use this - /// argument to override the default location. - #[clap(long = "spec-path")] - pub spec_path: Option, + #[clap(flatten)] + project: Project, - target: TargetPath, + target_path: TargetPath, } impl Extract { pub async fn exec(&self) -> Result { - let spec_path = self.spec_path.clone(); - let local_path = self.target.local(spec_path.as_ref()); + let download_path = self.project.download_path().await?; + let target = Arc::new(Target { format: self.format, - path: self.target.clone(), + path: self.target_path.clone(), }); - let spec = target::to_specification(target, spec_path).await?; - let sections = extract_sections(&spec); + Extraction { + download_path: &download_path, + base_path: None, + target, + out: &self.out, + extension: &self.extension, + log: true, + } + .exec() + .await + } +} + +pub struct Extraction<'a> { + pub download_path: &'a Path, + pub base_path: Option<&'a Path>, + pub target: Arc, + pub out: &'a Path, + pub extension: &'a str, + pub log: bool, +} + +impl Extraction<'_> { + pub async fn exec(self) -> Result { if self.out.extension().is_some() { // assume a path with an extension is a single file - // TODO output to single file - todo!("single file not implemented"); + // TODO output to single file? + return Err(error!( + "single file extraction not supported, got {}", + self.out.display() + )); + } + + let download_path = self.download_path; + let local_path = self.target.path.local(download_path); + + // The specification may be stored alongside the extracted TOML. + let out = match local_path.strip_prefix(self.out) { + Ok(path) => self.out.join(path), + Err(_e) => { + let local_path = self + .base_path + .and_then(|base| local_path.strip_prefix(base).ok()) + .unwrap_or(&local_path); + self.out.join(local_path) + } + }; + + let progress = if self.log { + Some(progress!( + "Extracting requirements from {}", + self.target.path + )) } else { - // output to directory - - sections.iter().try_for_each(|(section, features)| { - // The specification may be stored alongside the extracted TOML. - let mut out = match local_path.strip_prefix(&self.out) { - Ok(path) => self.out.join(path), - Err(_e) => self.out.join(&local_path), - }; - - out.set_extension(""); - let _ = std::fs::create_dir_all(&out); - out.push(format!("{}.{}", section.id, self.extension)); - - let file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(out)?; - let mut file = BufWriter::new(file); - - let target = &self.target; - - match &self.extension[..] { - "rs" => write_rust(&mut file, target, section, features)?, - "toml" => write_toml(&mut file, target, section, features)?, - ext => unimplemented!("{}", ext), - } + None + }; + + let spec = target::to_specification(self.target.clone(), download_path.clone()).await?; + let sections = extract_sections(&spec); + + // output to directory + + let mut total = 0; + + for (section, features) in sections.iter() { + total += features.len(); + + let mut out = out.to_path_buf(); + + out.set_extension(""); + let _ = std::fs::create_dir_all(&out); + out.push(format!("{}.{}", section.id, self.extension)); + + let file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(out)?; + let mut file = BufWriter::new(file); + + let target = &self.target.path; + + match self.extension { + "rs" => write_rust(&mut file, target, section, features)?, + "toml" => write_toml(&mut file, target, section, features)?, + ext => return Err(error!("unsupported extraction type: {ext:?}")), + } + } - ::Ok(()) - })?; + if let Some(progress) = progress { + progress!( + progress, + "Extracted {total} requirements across {} sections", + sections.len() + ); } Ok(()) @@ -139,17 +193,17 @@ fn extract_section(section: &Section) -> (&Section, Vec) { if KEY_WORDS_SET.is_match(line) { for (key_word, level) in KEY_WORDS.iter() { - for occurance in key_word.find_iter(line) { + for occurrence in key_word.find_iter(line) { // filter out any matches in quotes - these are definitions in the // document - if occurance.as_str().ends_with('"') { + if occurrence.as_str().ends_with('"') { continue; } let mut quote = vec![]; - let start = find_open(lines, lineno, occurance.start()); - let end = find_close(lines, lineno, occurance.end()); + let start = find_open(lines, lineno, occurrence.start()); + let end = find_close(lines, lineno, occurrence.end()); #[allow(clippy::needless_range_loop)] for i in start.0..=end.0 { diff --git a/duvet/src/lib.rs b/duvet/src/lib.rs index a75cc16..efe640a 100644 --- a/duvet/src/lib.rs +++ b/duvet/src/lib.rs @@ -6,6 +6,7 @@ use std::sync::Arc; mod annotation; mod comment; +mod config; mod extract; mod project; mod reference; diff --git a/duvet/src/project.rs b/duvet/src/project.rs index dd4a905..c49d6f1 100644 --- a/duvet/src/project.rs +++ b/duvet/src/project.rs @@ -1,83 +1,82 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::{comment, source::SourceFile, Result}; +use crate::{comment, config, source::SourceFile, Result}; use clap::Parser; use duvet_core::{diagnostic::IntoDiagnostic, path::Path}; use glob::glob; -use std::collections::HashSet; +use std::{collections::HashSet, sync::Arc}; #[derive(Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Parser)] pub struct Project { - /// Package to run tests for - #[clap(long, short = 'p')] - package: Option, - - /// Space or comma separated list of features to activate - #[clap(long)] - features: Vec, - - /// Build all packages in the workspace - #[clap(long)] - workspace: bool, - - /// Exclude packages from the test - #[clap(long = "exclude")] - excludes: Vec, - - /// Activate all available features - #[clap(long = "all-features")] - all_features: bool, - - /// Do not activate the `default` feature - #[clap(long = "no-default-features")] - no_default_features: bool, - - /// Disables running cargo commands - #[clap(long = "no-cargo")] - no_cargo: bool, + #[clap(flatten)] + deprecated: Deprecated, - /// TRIPLE #[clap(long)] - target: Option, + config_path: Option, +} - /// Directory for all generated artifacts - #[clap(long = "target-dir", default_value = "target/compliance")] - target_dir: String, +impl Project { + pub async fn download_path(&self) -> Result { + if let Some(config) = self.config().await? { + return Ok(config.download_path.clone()); + } - /// Path to Cargo.toml - #[clap(long = "manifest-path")] - manifest_path: Option, + if let Some(download_path) = self.deprecated.spec_path.as_ref() { + // the previous behavior always appended `specs` so we need to preserve that + return Ok(download_path.join("specs")); + } - /// Glob patterns for additional source files - #[clap(long = "source-pattern")] - source_patterns: Vec, + Ok("specs".into()) + } - /// Glob patterns for spec files - #[clap(long = "spec-pattern")] - spec_patterns: Vec, + pub async fn config(&self) -> Result>> { + let (path, root) = if let Some(path) = self.config_path.as_ref() { + let root = duvet_core::env::current_dir()?; + (path.clone(), root) + } else if let Some((path, root)) = config::default_path_and_root().await { + (path, root) + } else { + return Ok(None); + }; - /// Path to store the collection of spec files - /// - /// The collection of spec files are stored in a folder called `specs`. The - /// `specs` folder is stored in the current directory by default. Use this - /// argument to override the default location. - #[clap(long = "spec-path")] - pub spec_path: Option, -} + let config = config::load(path, root).await?; + Ok(Some(config)) + } -impl Project { pub async fn sources(&self) -> Result> { let mut sources = HashSet::new(); - for pattern in &self.source_patterns { + for pattern in &self.deprecated.source_patterns { self.source_file(pattern, &mut sources)?; } - for pattern in &self.spec_patterns { + for pattern in &self.deprecated.spec_patterns { self.toml_file(pattern, &mut sources)?; } + if let Some(config) = self.config().await? { + for source in &config.sources { + // TODO switch from `glob` to `duvet_core::glob` + let _ = &source.root; + for entry in glob(&source.pattern).into_diagnostic()? { + sources.insert(SourceFile::Text { + pattern: source.comment_style.clone(), + default_type: source.default_type, + path: entry.into_diagnostic()?.into(), + }); + } + } + + for requirement in &config.requirements { + // TODO switch from `glob` to `duvet_core::glob` + let _ = &requirement.root; + for entry in glob(&requirement.pattern).into_diagnostic()? { + sources.insert(SourceFile::Toml(entry.into_diagnostic()?.into())); + } + } + } + Ok(sources) } @@ -95,10 +94,11 @@ impl Project { }; for entry in glob(file_pattern).into_diagnostic()? { - files.insert(SourceFile::Text( - compliance_pattern.clone(), - entry.into_diagnostic()?.into(), - )); + files.insert(SourceFile::Text { + pattern: compliance_pattern.clone(), + default_type: Default::default(), + path: entry.into_diagnostic()?.into(), + }); } Ok(()) @@ -112,3 +112,47 @@ impl Project { Ok(()) } } + +/// Set of options that are preserved for backwards compatibility but either +/// don't do anything or are undocumented +#[derive(Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Parser)] +struct Deprecated { + #[clap(long, short = 'p', hide = true)] + package: Option, + + #[clap(long, hide = true)] + features: Vec, + + #[clap(long, hide = true)] + workspace: bool, + + #[clap(long = "exclude", hide = true)] + excludes: Vec, + + #[clap(long = "all-features", hide = true)] + all_features: bool, + + #[clap(long = "no-default-features", hide = true)] + no_default_features: bool, + + #[clap(long = "no-cargo", hide = true)] + no_cargo: bool, + + #[clap(long, hide = true)] + target: Option, + + #[clap(long = "target-dir", default_value = "target/compliance", hide = true)] + target_dir: String, + + #[clap(long = "manifest-path", hide = true)] + manifest_path: Option, + + #[clap(long = "source-pattern", hide = true)] + source_patterns: Vec, + + #[clap(long = "spec-pattern", hide = true)] + spec_patterns: Vec, + + #[clap(long = "spec-path", hide = true)] + spec_path: Option, +} diff --git a/duvet/src/report.rs b/duvet/src/report.rs index 0543a09..91f2367 100644 --- a/duvet/src/report.rs +++ b/duvet/src/report.rs @@ -54,18 +54,38 @@ pub struct Report { impl Report { pub async fn exec(&self) -> Result { + let config = self.project.config().await?; + let config = config.as_ref(); + + if let Some(config) = config { + let progress = progress!("Extracting requirements"); + let count = config.load_specifications().await?; + if count > 0 { + progress!( + progress, + "Extracted requirements from {count} specifications" + ); + } else { + progress!( + progress, + "Extracted no requirements - config does not include any specifications" + ) + } + } + let progress = progress!("Scanning sources"); let project_sources = self.project.sources().await?; let project_sources = Arc::new(project_sources); progress!(progress, "Scanned {} sources", project_sources.len()); - let progress = progress!("Extracing annotations"); + let progress = progress!("Parsing annotations"); let annotations = crate::annotation::query(project_sources.clone()).await?; - progress!(progress, "Extracted {} annotations", annotations.len()); + progress!(progress, "Parsed {} annotations", annotations.len()); let progress = progress!("Loading specifications"); - let spec_path = self.project.spec_path.clone(); - let specifications = annotation::specifications(annotations.clone(), spec_path).await?; + let download_path = self.project.download_path().await?; + let specifications = + annotation::specifications(annotations.clone(), download_path.clone()).await?; progress!(progress, "Loaded {} specifications", specifications.len()); let progress = progress!("Mapping sections"); @@ -73,11 +93,20 @@ impl Report { progress!(progress, "Mapped {} sections", reference_map.len()); let progress = progress!("Matching references"); + let blob_link = self + .blob_link + .as_deref() + .or_else(|| config.and_then(|config| config.report.html.blob_link.as_deref())); + let issue_link = self + .issue_link + .as_deref() + .or_else(|| config.and_then(|config| config.report.html.issue_link.as_deref())); let mut report = ReportResult { targets: Default::default(), annotations, - blob_link: self.blob_link.as_deref(), - issue_link: self.issue_link.as_deref(), + blob_link, + issue_link, + download_path: &download_path, }; let references = reference::query(reference_map.clone(), specifications.clone()).await?; progress!(progress, "Matched {} references", references.len()); @@ -110,18 +139,23 @@ impl Report { type ReportFn = fn(&ReportResult, &Path) -> crate::Result<()>; - let reports: &[(&Option<_>, ReportFn)] = &[ - (&self.json, json::report), - (&self.html, html::report), - (&self.lcov, lcov::report), + let internal_json = std::env::var("DUVET_INTERNAL_CI_JSON").ok().map(Path::from); + let internal_html = std::env::var("DUVET_INTERNAL_CI_HTML").ok().map(Path::from); + + let reports: &[(Option<&_>, ReportFn)] = &[ + (self.json.as_ref(), json::report), + (self.html.as_ref(), html::report), + (self.lcov.as_ref(), lcov::report), ( - &std::env::var("DUVET_INTERNAL_CI_JSON").ok().map(Path::from), + config.and_then(|config| config.report.json.path()), json::report, ), ( - &std::env::var("DUVET_INTERNAL_CI_HTML").ok().map(Path::from), + config.and_then(|config| config.report.html.path()), html::report, ), + (internal_json.as_ref(), json::report), + (internal_html.as_ref(), html::report), ]; for (path, report_fn) in reports { @@ -162,6 +196,7 @@ pub struct ReportResult<'a> { pub annotations: AnnotationSet, pub blob_link: Option<&'a str>, pub issue_link: Option<&'a str>, + pub download_path: &'a Path, } #[derive(Debug)] diff --git a/duvet/src/report/lcov.rs b/duvet/src/report/lcov.rs index 98d38fb..5be1783 100644 --- a/duvet/src/report/lcov.rs +++ b/duvet/src/report/lcov.rs @@ -39,6 +39,7 @@ macro_rules! record { pub fn report(report: &ReportResult, dir: &Path) -> Result { std::fs::create_dir_all(dir)?; let lcov_dir = dir.canonicalize()?; + let download_path = &report.download_path; report .targets .iter() @@ -46,7 +47,7 @@ pub fn report(report: &ReportResult, dir: &Path) -> Result { .try_for_each(|(id, (source, report))| { let path = lcov_dir.join(format!("compliance.{}.lcov", id)); let mut output = BufWriter::new(std::fs::File::create(path)?); - report_source(source, report, &mut output)?; + report_source(source, report, download_path, &mut output)?; ::Ok(()) })?; Ok(()) @@ -56,6 +57,7 @@ pub fn report(report: &ReportResult, dir: &Path) -> Result { fn report_source( source: &Target, report: &TargetReport, + download_path: &Path, output: &mut Output, ) -> Result { macro_rules! put { @@ -65,7 +67,7 @@ fn report_source( } put!("TN:Compliance"); - let relative = source.path.local(None); + let relative = source.path.local(download_path); put!("SF:{}", relative.display()); // record all sections diff --git a/duvet/src/source.rs b/duvet/src/source.rs index e3b79c7..70beb15 100644 --- a/duvet/src/source.rs +++ b/duvet/src/source.rs @@ -1,22 +1,33 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::{annotation::AnnotationSet, comment, Error}; +use crate::{ + annotation::{AnnotationSet, AnnotationType}, + comment, Error, +}; use duvet_core::path::Path; pub mod toml; #[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)] pub enum SourceFile { - Text(comment::Pattern, Path), + Text { + pattern: comment::Pattern, + default_type: AnnotationType, + path: Path, + }, Toml(Path), } impl SourceFile { pub async fn annotations(&self) -> (AnnotationSet, Vec) { match self { - Self::Text(pattern, file) => match duvet_core::vfs::read_string(file).await { - Ok(text) => comment::extract(&text, pattern, Default::default()), + Self::Text { + pattern, + default_type, + path, + } => match duvet_core::vfs::read_string(path).await { + Ok(text) => comment::extract(&text, pattern, *default_type), Err(err) => (Default::default(), vec![err]), }, Self::Toml(file) => toml::load(file).await, diff --git a/duvet/src/target.rs b/duvet/src/target.rs index 3247ace..85904f9 100644 --- a/duvet/src/target.rs +++ b/duvet/src/target.rs @@ -16,7 +16,7 @@ use url::Url; pub type SpecificationMap = Arc, Arc>>; -pub async fn query(targets: &TargetSet, spec_path: Option) -> Result { +pub async fn query(targets: &TargetSet, spec_path: Path) -> Result { let mut errors = vec![]; let mut tasks = tokio::task::JoinSet::new(); @@ -52,10 +52,9 @@ pub async fn query(targets: &TargetSet, spec_path: Option) -> Result, - spec_path: Option, + spec_path: duvet_core::path::Path, ) -> Result> { - let spec_path = spec_path.as_ref(); - let contents = target.path.load(spec_path).await?; + let contents = target.path.load(&spec_path).await?; let spec = target.format.parse(&contents)?; let spec = Arc::new(spec); Ok(spec) @@ -124,7 +123,7 @@ impl TargetPath { Ok(Self::Path(path.into())) } - pub async fn load(&self, spec_download_path: Option<&Path>) -> Result { + pub async fn load(&self, spec_download_path: &Path) -> Result { match self { Self::Url(url) => { let canonical_url = Self::canonical_url(url.as_str()); @@ -148,16 +147,10 @@ impl TargetPath { } } - pub fn local(&self, spec_download_path: Option<&Path>) -> Path { + pub fn local(&self, spec_download_path: &Path) -> Path { match self { Self::Url(url) => { - let mut path = if let Some(path_to_spec) = spec_download_path { - path_to_spec.clone() - } else { - duvet_core::env::current_dir().unwrap() - } - .to_path_buf(); - path.push("specs"); + let mut path = spec_download_path.to_path_buf(); path.push(url.host_str().expect("url should have host")); path.extend(url.path_segments().expect("url should have path")); path.set_extension("txt"); diff --git a/duvet/src/text/find.rs b/duvet/src/text/find.rs index 1df4160..4ade37c 100644 --- a/duvet/src/text/find.rs +++ b/duvet/src/text/find.rs @@ -50,7 +50,7 @@ fn fast_find(needle: &str, haystack: &str) -> Option> { }) } -/// TODO we should probaly deprecate this - it's better to enforce strict matching +/// TODO we should probably deprecate this - it's better to enforce strict matching fn fuzzy_find(needle: &str, haystack: &str) -> Option> { text_search( needle.as_bytes(),