From 0ff61a93e766882016fb502780ee2448ce15e055 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Wed, 25 Nov 2020 05:23:58 +1100 Subject: [PATCH 01/22] Wrap most wasm APIs in a JS version of Result, to avoid wasm-bindgen's stack overflows This way we never call wasm_bindgen's throw_val, which doesn't clean up after itself. Before: driver.batchedUpdates() After: driver.batchedUpdates().unwrap() // may throw an error The object with the unwrap() method is shaped like { Ok: T } or { Err: Error }. This object is included in `crates/wasm/src/include.js`, which most builds can import/bundle by themselves, but not no-modules, which doesn't work yet. --- Cargo.lock | 14 +- crates/citeproc/src/processor.rs | 16 +- crates/csl/src/error.rs | 5 +- crates/csl/src/locale.rs | 6 + crates/csl/src/locale/lang.rs | 13 +- crates/csl/src/style/info.rs | 1 + crates/db/src/cite.rs | 15 - crates/db/src/lib.rs | 4 +- crates/db/src/xml.rs | 39 +- crates/proc/src/db.rs | 31 +- crates/proc/src/disamb/mod.rs | 2 +- crates/wasm/Cargo.toml | 2 +- crates/wasm/js-demo/js/App.tsx | 6 +- crates/wasm/js-demo/js/Document.ts | 29 +- crates/wasm/js-demo/js/DocumentEditor.tsx | 17 +- crates/wasm/js-demo/js/useDocument.ts | 33 +- crates/wasm/js-demo/webpack.config.js | 2 +- crates/wasm/src/include.js | 81 ++++ crates/wasm/src/lib.rs | 565 +++++++++++++++------- crates/wasm/src/options.rs | 74 +++ crates/wasm/src/typings.d.ts | 3 + crates/wasm/src/utils.rs | 17 +- crates/wasm/src/wasm_result.rs | 204 ++++++++ 23 files changed, 886 insertions(+), 293 deletions(-) create mode 100644 crates/wasm/src/include.js create mode 100644 crates/wasm/src/options.rs create mode 100644 crates/wasm/src/typings.d.ts create mode 100644 crates/wasm/src/wasm_result.rs diff --git a/Cargo.lock b/Cargo.lock index 9c55a57a..fa832c22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1978,9 +1978,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.42" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -2070,18 +2070,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" dependencies = [ "proc-macro2", "quote", @@ -2392,7 +2392,6 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" name = "wasm" version = "0.0.1" dependencies = [ - "anyhow", "cfg-if", "citeproc", "citeproc-io", @@ -2406,6 +2405,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "thiserror", "wasm-bindgen", "wasm-bindgen-futures", ] diff --git a/crates/citeproc/src/processor.rs b/crates/citeproc/src/processor.rs index 7d1066f4..9da382a2 100644 --- a/crates/citeproc/src/processor.rs +++ b/crates/citeproc/src/processor.rs @@ -618,17 +618,11 @@ impl Processor { } pub fn get_langs_in_use(&self) -> Vec { - let mut langs: HashSet = self - .all_keys() - .iter() - .filter_map(|ref_id| self.reference(ref_id.clone())) - .filter_map(|refr| refr.language.clone()) - .collect(); - let style = self.style(); - if let Some(dl) = style.default_locale.as_ref() { - langs.insert(dl.clone()); - } - langs.into_iter().collect() + let dl = self.default_lang(); + let mut vec: Vec = dl.iter_fetchable_langs().collect(); + vec.sort(); + vec.dedup(); + vec } pub fn has_cached_locale(&self, lang: &Lang) -> bool { diff --git a/crates/csl/src/error.rs b/crates/csl/src/error.rs index c25ccfd2..ae86203b 100644 --- a/crates/csl/src/error.rs +++ b/crates/csl/src/error.rs @@ -37,11 +37,12 @@ where #[derive(thiserror::Error, Debug)] #[cfg_attr(feature = "serde", derive(Serialize))] +#[cfg_attr(feature = "serde", serde(tag = "tag", content = "content"))] #[non_exhaustive] pub enum StyleError { #[error("invalid style: {0}")] Invalid(#[from] CslError), - #[error("could not parse style")] + #[error("could not parse style: {0}")] ParseError( #[from] #[cfg_attr(feature = "serde", serde(serialize_with = "rox_error_serialize"))] @@ -50,6 +51,7 @@ pub enum StyleError { #[error( "incorrectly supplied a dependent style, which refers to a parent {required_parent:?}" )] + #[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] DependentStyle { required_parent: String }, } @@ -98,6 +100,7 @@ pub struct InvalidCsl { // TODO: serialize_with or otherwise get this into the output pub range: Range, pub message: String, + #[serde(skip_serializing_if = "String::is_empty")] pub hint: String, } diff --git a/crates/csl/src/locale.rs b/crates/csl/src/locale.rs index 26841b04..76e3f266 100644 --- a/crates/csl/src/locale.rs +++ b/crates/csl/src/locale.rs @@ -78,6 +78,12 @@ pub struct Locale { impl FromStr for Locale { type Err = StyleError; fn from_str(xml: &str) -> Result { + Locale::parse(xml) + } +} + +impl Locale { + pub fn parse(xml: &str) -> Result { let doc = Document::parse(&xml)?; let info = ParseInfo::default(); let locale = Locale::from_node(&doc.root_element(), &info)?; diff --git a/crates/csl/src/locale/lang.rs b/crates/csl/src/locale/lang.rs index c4d4c5b5..5dc7f688 100644 --- a/crates/csl/src/locale/lang.rs +++ b/crates/csl/src/locale/lang.rs @@ -17,7 +17,7 @@ pub enum LocaleSource { /// A parsable representation of `xml:lang`. /// /// See http://www.datypic.com/sc/xsd/t-xsd_language.html -#[derive(Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, PartialOrd, Ord)] pub enum Lang { /// ISO 639 language code, + optional hyphen and 2-letter ISO 3166 country code. /// @@ -87,6 +87,13 @@ impl Lang { .map(LocaleSource::Inline) .chain(self.file_iter().map(LocaleSource::File)) } + pub fn iter_fetchable_langs(&self) -> impl Iterator { + self.iter() + .filter_map(|source| match source { + LocaleSource::File(l) => Some(l), + _ => None, + }) + } fn file_iter(&self) -> FileIter { FileIter { current: Some(self.clone()), @@ -154,7 +161,7 @@ fn test_french() { /// Language codes for `Lang::Iso`. /// /// The 3-character codes are ISO 639-3. -#[derive(Debug, Clone, Eq, PartialEq, Hash, EnumString)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, EnumString, PartialOrd, Ord)] pub enum IsoLang { #[strum(serialize = "en", serialize = "eng")] English, @@ -212,7 +219,7 @@ impl fmt::Display for IsoLang { /// These countries are used to do dialect fallback. Countries not used in that can be represented /// as `IsoCountry::Other`. If a country is in the list, you don't need to allocate to refer to it, /// so there are some non-participating countries in the list simply because it's faster. -#[derive(Debug, Clone, Eq, PartialEq, Hash, EnumString)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, EnumString, PartialOrd, Ord)] pub enum IsoCountry { /// United States US, diff --git a/crates/csl/src/style/info.rs b/crates/csl/src/style/info.rs index 8839b0f9..484d7698 100644 --- a/crates/csl/src/style/info.rs +++ b/crates/csl/src/style/info.rs @@ -169,6 +169,7 @@ impl EnumGetAttribute for CitationFormat {} #[cfg_attr(feature = "serde", serde(rename_all = "kebab-case"))] pub enum Rel { #[strum(serialize = "self")] + #[cfg_attr(feature = "serde", serde(rename = "self"))] RelSelf, Documentation, /// Not allowed in dependent styles diff --git a/crates/db/src/cite.rs b/crates/db/src/cite.rs index 63e91fcc..4ea35bb4 100644 --- a/crates/db/src/cite.rs +++ b/crates/db/src/cite.rs @@ -47,9 +47,6 @@ pub trait CiteDatabase: LocaleDatabase + StyleDatabase { // All cite ids, in the order they are cited in the document fn all_cite_ids(&self) -> Arc>; - fn locale_by_cite(&self, id: CiteId) -> Arc; - fn locale_by_reference(&self, ref_id: Atom) -> Arc; - /// Create ghost cites for disambiguation only as needed. /// These are subsequently interned into CiteIds. fn ghost_cite(&self, ref_id: Atom) -> Arc>; @@ -123,18 +120,6 @@ fn reference(db: &dyn CiteDatabase, key: Atom) -> Option> { } } -fn locale_by_cite(db: &dyn CiteDatabase, id: CiteId) -> Arc { - let cite = id.lookup(db); - db.locale_by_reference(cite.ref_id.clone()) -} - -fn locale_by_reference(db: &dyn CiteDatabase, ref_id: Atom) -> Arc { - let refr = db.reference(ref_id); - refr.and_then(|r| r.language.clone()) - .map(|l| db.merged_locale(l)) - .unwrap_or_else(|| db.default_locale()) -} - /// Type to represent which references should appear in a bibiliography even if they are not cited /// in the document. The default is that references only appear if they are cited. #[derive(Debug, Clone, Eq, PartialEq)] diff --git a/crates/db/src/lib.rs b/crates/db/src/lib.rs index 1e0beca7..d7df3e2b 100644 --- a/crates/db/src/lib.rs +++ b/crates/db/src/lib.rs @@ -17,6 +17,6 @@ pub fn safe_default(db: &mut (impl cite::CiteDatabase + xml::LocaleDatabase + xm db.set_all_keys_with_durability(Default::default(), Durability::MEDIUM); db.set_all_uncited(Default::default()); db.set_cluster_ids(Arc::new(Default::default())); - db.set_locale_input_langs(Default::default()); - db.set_default_lang_override(Default::default()); + db.set_locale_input_langs_with_durability(Default::default(), Durability::HIGH); + db.set_default_lang_override_with_durability(Default::default(), Durability::HIGH); } diff --git a/crates/db/src/xml.rs b/crates/db/src/xml.rs index 928b8c6e..b69d9bcc 100644 --- a/crates/db/src/xml.rs +++ b/crates/db/src/xml.rs @@ -170,30 +170,34 @@ pub trait LocaleDatabase: StyleDatabase + HasFetcher { fn default_lang_override(&self) -> Option; /// Backed by the LocaleFetcher implementation + #[salsa::transparent] fn locale_xml(&self, key: Lang) -> Option>; /// Derived from a `Style` + #[salsa::transparent] fn inline_locale(&self, key: Option) -> Option>; /// A locale object, which may be `Default::default()` - fn locale(&self, key: LocaleSource) -> Option>; + fn parsed_locale(&self, key: LocaleSource) -> Option>; /// Derives the full lang inheritance chain, and merges them into one + #[salsa::transparent] fn merged_locale(&self, key: Lang) -> Arc; - /// Even though we already have a merged `LocaleOptionsNode` struct, all its fields are - /// `Option`. To avoid having to unwrap each field later on, we merge whatever options did - /// get provided into a non-`Option` defaults struct. - fn locale_options(&self, key: Lang) -> Arc; - fn default_locale(&self) -> Arc; + + #[salsa::transparent] + fn default_lang(&self) -> Lang; } -fn default_locale(db: &dyn LocaleDatabase) -> Arc { - let overrider = db +fn default_lang(db: &dyn LocaleDatabase) -> Lang { + db .default_lang_override() - .unwrap_or_else(|| db.style().default_locale.clone().unwrap_or_else(Default::default)); - db.merged_locale(overrider) + .unwrap_or_else(|| db.style().default_locale.clone().unwrap_or_else(Default::default)) +} + +fn default_locale(db: &dyn LocaleDatabase) -> Arc { + db.merged_locale(db.default_lang()) } fn locale_xml(db: &dyn LocaleDatabase, key: Lang) -> Option> { @@ -216,12 +220,12 @@ fn inline_locale(db: &dyn LocaleDatabase, key: Option) -> Option Option> { +fn parsed_locale(db: &dyn LocaleDatabase, key: LocaleSource) -> Option> { match key { LocaleSource::File(ref lang) => { let string = db.locale_xml(lang.clone()); string - .and_then(|s| match Locale::from_str(&s) { + .and_then(|s| match Locale::parse(&s) { Ok(l) => Some(l), Err(e) => { error!("failed to parse locale for lang {}: {:?}", lang, e); @@ -238,7 +242,7 @@ fn merged_locale(db: &dyn LocaleDatabase, key: Lang) -> Arc { debug!("requested locale {:?}", key); let locales = key .iter() - .filter_map(|src| db.locale(src)) + .filter_map(|src| db.parsed_locale(src)) .collect::>(); Arc::new( locales @@ -259,11 +263,6 @@ fn merged_locale(db: &dyn LocaleDatabase, key: Lang) -> Arc { ) } -fn locale_options(db: &dyn LocaleDatabase, key: Lang) -> Arc { - let merged = &db.merged_locale(key).options_node; - Arc::new(LocaleOptions::from_merged(merged)) -} - cfg_if::cfg_if! { if #[cfg(feature = "parallel")] { pub trait LocaleFetcher: Send + Sync { @@ -271,7 +270,7 @@ cfg_if::cfg_if! { fn fetch_locale(&self, lang: &Lang) -> Option { use std::str::FromStr; let s = self.fetch_string(lang).ok()??; - Some(Locale::from_str(&s).ok()?) + Some(Locale::parse(&s).ok()?) } } } else { @@ -280,7 +279,7 @@ cfg_if::cfg_if! { fn fetch_locale(&self, lang: &Lang) -> Option { use std::str::FromStr; let s = self.fetch_string(lang).ok()??; - Some(Locale::from_str(&s).ok()?) + Some(Locale::parse(&s).ok()?) } } } diff --git a/crates/proc/src/db.rs b/crates/proc/src/db.rs index 0cc8ef96..1c99902c 100644 --- a/crates/proc/src/db.rs +++ b/crates/proc/src/db.rs @@ -360,7 +360,6 @@ fn ref_not_found(db: &dyn IrDatabase, ref_id: &Atom, log: bool) -> Arc { // style // cite // reference -// locale_by_cite // cite_position // - // bib_number @@ -368,7 +367,7 @@ fn ref_not_found(db: &dyn IrDatabase, ref_id: &Atom, log: bool) -> Arc { macro_rules! preamble { ($style:ident, $locale:ident, $cite:ident, $refr:ident, $ctx:ident, $db:expr, $id:expr, $pass:expr) => {{ $style = $db.style(); - $locale = $db.locale_by_cite($id); + $locale = $db.default_locale(); // Avoid making bibliography ghosts all depend any positional / note num info let cite_stuff = match $db.lookup_cite($id) { CiteData::RealCite { cite, .. } => (cite, $db.cite_position($id)), @@ -785,11 +784,12 @@ fn expand_one_name_ir( } use crate::disamb::names::MatchKey; - let name_ambiguity_number = |edge: &EdgeData, match_key: Option<&MatchKey>, slot: &[NameVariantMatcher]| -> u32 { - slot.iter() - .filter(|matcher| matcher.accepts(edge, None)) - .count() as u32 - }; + let name_ambiguity_number = + |edge: &EdgeData, match_key: Option<&MatchKey>, slot: &[NameVariantMatcher]| -> u32 { + slot.iter() + .filter(|matcher| matcher.accepts(edge, None)) + .count() as u32 + }; let mut n = 0usize; for dnr in nir.disamb_names.iter_mut() { @@ -846,7 +846,9 @@ fn disambiguate_add_givennames( ctx: &mut CiteContext<'_, Markup>, also_add: bool, ) -> Option { - ctx.disamb_pass = Some(DisambPass::AddGivenName(ctx.style.citation.givenname_disambiguation_rule)); + ctx.disamb_pass = Some(DisambPass::AddGivenName( + ctx.style.citation.givenname_disambiguation_rule, + )); let _fmt = db.get_formatter(); let refs = refs_accepting_cite( db, @@ -992,7 +994,14 @@ fn ir_gen2_matching_refs(db: &dyn IrDatabase, id: CiteId) -> Arc> { let gndr = style.citation.givenname_disambiguation_rule; let cite = id.lookup(db); let gen2 = db.ir_gen2_add_given_name(id); - let refs = refs_accepting_cite(db, gen2.root, &gen2.arena, Some(id), &cite.ref_id, Some(DisambPass::AddGivenName(gndr))); + let refs = refs_accepting_cite( + db, + gen2.root, + &gen2.arena, + Some(id), + &cite.ref_id, + Some(DisambPass::AddGivenName(gndr)), + ); Arc::new(refs) } @@ -1464,7 +1473,7 @@ pub fn with_cite_context( f: impl FnOnce(CiteContext) -> T, ) -> Option { let style = db.style(); - let locale = db.locale_by_cite(id); + let locale = db.default_locale(); let cite = id.lookup(db); let refr = db.reference(cite.ref_id.clone())?; let (names_delimiter, name_el) = db.name_info_citation(); @@ -1505,7 +1514,7 @@ pub fn with_bib_context( ) -> Option { let style = db.style(); let bib = style.bibliography.as_ref()?; - let locale = db.locale_by_reference(ref_id.clone()); + let locale = db.default_locale(); let cite = Cite::basic(ref_id.clone()); let refr_arc = db.reference(ref_id); let null_ref = citeproc_io::Reference::empty("empty_ref".into(), csl::CslType::Article); diff --git a/crates/proc/src/disamb/mod.rs b/crates/proc/src/disamb/mod.rs index 17334f7b..308cf476 100644 --- a/crates/proc/src/disamb/mod.rs +++ b/crates/proc/src/disamb/mod.rs @@ -273,7 +273,7 @@ pub fn create_ref_ir( refr: &Reference, ) -> Vec<(FreeCond, RefIR)> { let style = db.style(); - let locale = db.locale_by_reference(refr.id.clone()); + let locale = db.default_locale(); let ysh_explicit_edge = EdgeData::YearSuffixExplicit; let ysh_plain_edge = EdgeData::YearSuffixPlain; let ysh_edge = EdgeData::YearSuffix; diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml index 865a8133..e4bfbbbb 100644 --- a/crates/wasm/Cargo.toml +++ b/crates/wasm/Cargo.toml @@ -53,7 +53,7 @@ js-sys = "0.3.39" serde = "1.0.116" serde_derive = "1.0.116" serde_json = "1.0.57" -anyhow = "1.0.32" +thiserror = "1.0.22" [dependencies.rand] version = "0.7.3" diff --git a/crates/wasm/js-demo/js/App.tsx b/crates/wasm/js-demo/js/App.tsx index 59cd2b61..5a66d418 100644 --- a/crates/wasm/js-demo/js/App.tsx +++ b/crates/wasm/js-demo/js/App.tsx @@ -102,11 +102,11 @@ async function loadEditor() {

Style

-