Run Rust library test suite as well as integration tests

.github/workflows/ci.yml

@@ -5,6 +5,32 @@ on:
     branches:
     - master
 jobs:
+  cargo_test:
+    name: Cargo Test
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Install latest nightly
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly
+            override: true
+      - name: Cache cargo directories
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-test-lib-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-test-lib-
+            ${{ runner.os }}-cargo-test-
+
+      - run: cargo test --lib
+
   regressions:
     name: CSL Test Suite Regressions
     runs-on: ubuntu-18.04
@@ -64,6 +90,7 @@ jobs:
           key: ${{ runner.os }}-cargo-test-suite-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: |
             ${{ runner.os }}-cargo-test-suite-
+            ${{ runner.os }}-cargo-test-
       - name: "Build tools package"
         run: cargo build --package tools
       - name: "Pull locales"

crates/citeproc/src/test.rs

@@ -186,7 +186,7 @@ mod preview {
         let mut db = mk_db();
         assert_cluster!(db.get_cluster(1), Some("Book one"));
         let cites = vec![Cite::basic("two")];
-        let preview = db.preview_citation_cluster(cites, PreviewPosition::ReplaceCluster(1));
+        let preview = db.preview_citation_cluster(cites, PreviewPosition::ReplaceCluster(1), None);
         assert_cluster!(db.get_cluster(1), Some("Book one"));
         assert_cluster!(preview.ok(), Some("Book two"));
     }
@@ -196,7 +196,7 @@ mod preview {
         let mut db = mk_db();
         assert_cluster!(db.get_cluster(2), Some("Book two"));
         let cites = vec![Cite::basic("one")];
-        let preview = db.preview_citation_cluster(cites, PreviewPosition::ReplaceCluster(2));
+        let preview = db.preview_citation_cluster(cites, PreviewPosition::ReplaceCluster(2), None);
         assert_cluster!(db.get_cluster(2), Some("Book two"));
         assert_cluster!(preview.ok(), Some("Book one, ibid"));
     }
@@ -210,7 +210,7 @@ mod preview {
             ClusterPosition { id: 2, note: Some(2) },
             ClusterPosition { id: 0, note: Some(3) }, // Append at the end
         ];
-        let preview = db.preview_citation_cluster(cites, PreviewPosition::MarkWithZero(positions));
+        let preview = db.preview_citation_cluster(cites, PreviewPosition::MarkWithZero(positions), None);
         assert_cluster!(preview.ok(), Some("Book one, subsequent"));
         assert_cluster!(db.get_cluster(1), Some("Book one"));
         assert_cluster!(db.get_cluster(2), Some("Book two"));
@@ -225,7 +225,7 @@ mod preview {
             ClusterPosition { id: 1, note: Some(1) },
             ClusterPosition { id: 2, note: Some(2) },
         ];
-        let preview = db.preview_citation_cluster(cites, PreviewPosition::MarkWithZero(positions));
+        let preview = db.preview_citation_cluster(cites, PreviewPosition::MarkWithZero(positions), None);
         assert_cluster!(preview.ok(), Some("Book one; Book three"));
         assert_cluster!(db.get_cluster(1), Some("Book one"));
         assert_cluster!(db.get_cluster(2), Some("Book two"));
@@ -239,7 +239,7 @@ mod preview {
             ClusterPosition { id: 0, note: Some(1) }, // Replace cluster #1
             ClusterPosition { id: 2, note: Some(2) },
         ];
-        let preview = db.preview_citation_cluster(cites, PreviewPosition::MarkWithZero(positions));
+        let preview = db.preview_citation_cluster(cites, PreviewPosition::MarkWithZero(positions), None);
         assert_cluster!(preview.ok(), Some("Book three"));
         assert_cluster!(db.get_cluster(1), Some("Book one"));
         assert_cluster!(db.get_cluster(2), Some("Book two"));

crates/io/src/lib.rs

@@ -282,8 +282,11 @@ fn stopword_regex() -> &'static regex::Regex {
         // Skip the | on the last one
         ")(?:\\s|$)",
         // John d’Doe
-        "|(?-i)d\u{2019}",
-        "|(?-i)of-"
+        "|^(?-i)d\u{2019}",
+        "|^(?-i)d'",
+        "|^(?-i)l\u{2019}",
+        "|^(?-i)l'",
+        "|^(?-i)of-"
     ];
 
     CITEPROC_JS_STOPWORD_REGEX.get_or_init(|| regex::Regex::new(re).unwrap())
@@ -299,6 +302,7 @@ fn stopwords() {
     assert!(!is_stopword("grandiloquent "));
     assert!(is_stopword("l’Anglais "));
     assert!(is_stopword("l’Égypte "));
+    assert!(!is_stopword("this word followed by l’Égypte "));
 }
 
 /// Returns the length of the matched word
@@ -351,12 +355,11 @@ fn title_case_word<'a>(
     entire_is_uppercase: bool,
     no_stopword: bool,
 ) -> (Cow<'a, str>, Option<usize>) {
-    let expect = "only called with nonempty words";
-    trace!("title_case_word {}", word);
     if !no_stopword {
         if let Some(mut match_len) = is_stopword(word_and_rest) {
             // drop the trailing whitespace
             let matched = &word_and_rest[..match_len];
+            debug!("title_case_word -- is_stopword: {}", matched);
             let last_char = matched.chars().rev().nth(0).map_or(0, |c| if c == '-' || c.is_whitespace() { c.len_utf8() } else { 0 });
             match_len = match_len - last_char;
             let lowered = word_and_rest[..match_len].to_lowercase();

crates/io/src/output/markup/move_punctuation.rs

@@ -14,11 +14,11 @@ fn normalise() {
         InlineElement::Micro(MicroNode::parse("b", &Default::default())),
     ];
     normalise_text_elements(&mut nodes);
-    assert_eq!(&nodes[..], &[InlineElement::Micro(MicroNode::parse("ab", &Default::default()))][..]);
+    assert_eq!(&nodes[..], &[InlineElement::Text("ab".to_owned())][..]);
 }
 
 fn smash_string_push(base: &mut String, mut suff: &str) {
-    info!("smash_string_push {:?} <- {:?}", base, suff);
+    trace!("smash_string_push {:?} <- {:?}", base, suff);
     let suff_trimmed = suff.trim_start();
     if base.trim_end().chars().rev().nth(0).map_or(false, is_punc)
         && suff_trimmed.chars().nth(0).map_or(false, is_punc)
@@ -40,14 +40,14 @@ fn smash_string_push(base: &mut String, mut suff: &str) {
         let start = base_len - last_width;
         let range = start .. start + width;
         if let Some(Some(replacement)) = FULL_MONTY_PLAIN.get(&base.as_bytes()[range.clone()]) {
-            info!("smash_string_push REPLACING {:?} with {:?}", &base[range.clone()], *replacement);
+            trace!("smash_string_push REPLACING {:?} with {:?}", &base[range.clone()], *replacement);
             base.replace_range(range, *replacement);
         }
     }
 }
 
 fn smash_just_punc(base: &mut String, suff: &mut String) {
-    info!("smash_just_punc {:?} <- {:?}", base, suff);
+    trace!("smash_just_punc {:?} <- {:?}", base, suff);
     let mut suff_append: &str = suff.as_ref();
     let suff_trimmed = suff_append.trim_start();
     if base.trim_end().chars().rev().nth(0).map_or(false, is_punc)
@@ -70,7 +70,7 @@ fn smash_just_punc(base: &mut String, suff: &mut String) {
         let start = base_len - last_width;
         let range = start .. start + width;
         if let Some(Some(replacement)) = FULL_MONTY_PLAIN.get(&base.as_bytes()[range.clone()]) {
-            info!("smash_just_punc REPLACING {:?} with {:?}", &base[range.clone()], *replacement);
+            trace!("smash_just_punc REPLACING {:?} with {:?}", &base[range.clone()], *replacement);
             base.replace_range(range, *replacement);
             suff.replace_range(..first_width, "");
         } else {
@@ -143,7 +143,7 @@ pub fn normalise_text_elements(slice: &mut Vec<InlineElement>) {
                         }
                     }
                     (InlineElement::Formatted(children, _), InlineElement::Micro(ms2)) => {
-                        info!("formatted, micro");
+                        trace!("formatted, micro");
                         match children.last_mut().and_then(find_string_right_f) {
                             Some(s1) => match ms2.first_mut().and_then(find_string_left_micro) {
                                 Some(s2) => smash_just_punc(s1, s2),
@@ -223,7 +223,7 @@ enum Motion {
 // Basically, affixes go outside Quoted elements. So we can just look for text elements that come
 // right after quoted ones.
 pub fn move_punctuation(slice: &mut Vec<InlineElement>, punctuation_in_quote: Option<bool>) {
-    info!("move_punctuation {:?} {:?}", slice, punctuation_in_quote);
+    trace!("move_punctuation {:?} {:?}", slice, punctuation_in_quote);
     normalise_text_elements(slice);
 
     if slice.len() > 1 {

crates/io/src/output/markup/parse_quotes.rs

@@ -197,7 +197,7 @@ fn stamp<'a>(
 
 #[test]
 fn test_stamp() {
-    env_logger::init();
+    // env_logger::init();
     let mut orig = vec![MicroNode::Text("hi".into()), MicroNode::Text("ho".into())];
     let options = IngestOptions::default_with_quotes(LocalizedQuotes::simple());
     let inters = vec![

crates/proc/src/date.rs

@@ -722,17 +722,23 @@ fn dp_render_either<'c, O: OutputFormat, I: OutputFormat>(
 
 fn dp_render_sort_string(part: &DatePart, date: &Date, key: &SortKey, is_filtered: bool) -> Option<String> {
     match part.form {
-        DatePartForm::Year(_form) => Some(format!("{:04}_", date.year)),
-        DatePartForm::Month(_form, _strip_periods) => {
+        DatePartForm::Year(_) => Some(format!("{:04}_", date.year)),
+        DatePartForm::Month(..) => {
+            if is_filtered {
+                return None;
+            }
             // Sort strings do not compare seasons
-            if !is_filtered && date.month > 0 && date.month <= 12 {
+            if date.month > 0 && date.month <= 12 {
                 Some(format!("{:02}", date.month))
             } else {
                 Some("00".to_owned())
             }
         }
-        DatePartForm::Day(_form) => {
-            if !is_filtered && date.day > 0 {
+        DatePartForm::Day(_) => {
+            if is_filtered {
+                return None;
+            }
+            if date.day > 0 {
                 Some(format!("{:02}", date.day))
             } else {
                 Some("00".to_owned())

crates/proc/src/disamb/test.rs

@@ -12,7 +12,7 @@ use citeproc_io::output::markup::Markup;
 use citeproc_io::{ClusterNumber, Reference};
 
 use csl::Variable;
-use csl::{Cond, CslType, Position};
+use csl::CslType;
 
 macro_rules! style_text_layout {
     ($ex:expr) => {{
@@ -137,15 +137,15 @@ fn test() {
         .ordinary
         .insert(Variable::ContainerTitle, "Title".into());
 
-    let vec = create_ref_ir::<Markup, MockProcessor>(db, &refr);
+    let vec = create_ref_ir::<Markup>(db, &refr);
     for (fc, ir) in &vec {
         println!("{:?}:\n    {}", fc, ir.debug(db));
     }
-    let dfa = create_dfa::<Markup, MockProcessor>(db, &refr);
+    let dfa = create_dfa::<Markup>(db, &refr);
     println!("{}", dfa.debug_graph(db));
 
-    let _vec = create_ref_ir::<Markup, MockProcessor>(db, &refr2);
-    let dfa2 = create_dfa::<Markup, MockProcessor>(db, &refr2);
+    let _vec = create_ref_ir::<Markup>(db, &refr2);
+    let dfa2 = create_dfa::<Markup>(db, &refr2);
     println!("{}", dfa2.debug_graph(db));
 
     use citeproc_io::{Cite, Cluster, IntraNote};

crates/proc/src/sort.rs

@@ -346,13 +346,12 @@ fn test_date_as_macro_strip_delims() {
     let mut db = MockProcessor::new();
     let mut refr = citeproc_io::Reference::empty("ref_id".into(), CslType::Book);
     use citeproc_io::{Date, DateOrRange};
-    let mac = "indep";
     refr.ordinary.insert(Variable::Title, String::from("title"));
     refr.date.insert(
         DateVariable::Issued,
         DateOrRange::Single(Date::new(2000, 1, 1)),
     );
-    db.insert_reference(refr);
+    db.insert_references(vec![refr]);
     db.set_style_text(r#"<?xml version="1.0" encoding="utf-8"?>
         <style version="1.0" class="note">
            <macro name="year-date">

crates/proc/src/test.rs

@@ -1,5 +1,5 @@
 use crate::prelude::*;
-use citeproc_db::{LocaleFetcher, PredefinedLocales, StyleDatabase};
+use citeproc_db::{LocaleFetcher, PredefinedLocales, StyleDatabase, CiteData};
 use citeproc_io::{output::markup::Markup, Cluster, Reference};
 use csl::Atom;
 
@@ -45,22 +45,18 @@ pub fn with_test_citation<T>(f: impl Fn(Style) -> T, s: &str) -> T {
     crate::db::IrDatabaseStorage
 )]
 pub struct MockProcessor {
-    runtime: salsa::Runtime<Self>,
+    storage: salsa::Storage<Self>,
     fetcher: Arc<dyn LocaleFetcher>,
 }
 
+impl salsa::Database for MockProcessor {}
+
 impl HasFormatter for MockProcessor {
     fn get_formatter(&self) -> Markup {
         Markup::html()
     }
 }
 
-impl salsa::Database for MockProcessor {
-    fn salsa_runtime(&self) -> &salsa::Runtime<MockProcessor> {
-        &self.runtime
-    }
-}
-
 impl citeproc_db::HasFetcher for MockProcessor {
     fn get_fetcher(&self) -> Arc<dyn LocaleFetcher> {
         self.fetcher.clone()
@@ -71,7 +67,7 @@ impl MockProcessor {
     pub fn new() -> Self {
         let fetcher = Arc::new(PredefinedLocales::bundled_en_us());
         let mut db = MockProcessor {
-            runtime: Default::default(),
+            storage: Default::default(),
             fetcher,
         };
         citeproc_db::safe_default(&mut db);
@@ -94,7 +90,11 @@ impl MockProcessor {
             } = cluster;
             let mut ids = Vec::new();
             for (index, cite) in cites.into_iter().enumerate() {
-                let cite_id = self.cite(cluster_id, index as u32, Arc::new(cite));
+                let cite_id = self.cite(CiteData::RealCite {
+                    cluster: cluster_id,
+                    index: index as u32,
+                    cite: Arc::new(cite),
+                });
                 ids.push(cite_id);
             }
             self.set_cluster_cites(cluster_id, Arc::new(ids));