Support more newlines

Corresponds to Section 5.8 of the Unicode Standard, as well as `chumsky::text::newline`.
zesterer · Jan 26, 2023 · 1e98de7 · 1e98de7
1 parent 06b1748
commit 1e98de7
Showing 1 changed file with 26 additions and 9 deletions.
diff --git a/src/source.rs b/src/source.rs
@@ -66,18 +66,35 @@ impl<S: AsRef<str>> From<S> for Source {
     /// Note that this function can be expensive for long strings. Use an implementor of [`Cache`] where possible.
     fn from(s: S) -> Self {
         let mut offset = 0;
+        let mut last_ends_with_cr = false;
         Self {
             lines: s
                 .as_ref()
-                .split_terminator('\n') // TODO: Handle non-\n newlines
-                .map(|line| {
-                    let l = Line {
-                        offset,
-                        len: line.chars().count() + 1,
-                        chars: line.trim_end().to_owned(),
-                    };
-                    offset += l.len;
-                    l
+                .split_inclusive([
+                    '\r', // Carriage return
+                    '\n', // Line feed
+                    '\x0B', // Vertical tab
+                    '\x0C', // Line separator
+                    '\u{0085}', // Next line
+                    '\u{2028}', // Line separator
+                    '\u{2029}' // Paragraph separator
+                ])
+                .flat_map(|line| {
+                    if line == "\n" && last_ends_with_cr {
+                        offset += 1;
+                        last_ends_with_cr = false;
+                        None
+                    } else {
+                        let len = line.chars().count();
+                        let l = Line {
+                            offset,
+                            len,
+                            chars: line.trim_end().to_owned(),
+                        };
+                        offset += len;
+                        last_ends_with_cr = line.ends_with('\r');
+                        Some(l)
+                    }
                 })
                 .collect(),
             len: offset,