Fix Unicode 15.0 sentence segmentation

components/segmenter/Cargo.toml

@@ -40,6 +40,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = "1.0"
 icu = { path = "../../components/icu", default-features = false }
 itertools = "0.10"
+icu_properties = { workspace = true }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 criterion = "0.4"
@@ -51,7 +52,7 @@ serde = ["dep:serde", "zerovec/serde", "icu_collections/serde", "icu_provider/se
 datagen = ["serde", "dep:databake", "zerovec/databake", "icu_collections/databake"]
 lstm = ["dep:core_maths"]
 auto = ["lstm"] # Enabled try_new_auto_unstable constructors
-compiled_data = ["dep:icu_segmenter_data"]
+compiled_data = ["dep:icu_segmenter_data", "icu_properties/compiled_data"]
 bench = []
 
 [lib]

components/segmenter/src/rule_segmenter.rs

@@ -133,6 +133,9 @@ impl<'l, 's, Y: RuleBreakType<'l, 's> + ?Sized> Iterator for RuleBreakIterator<'
                 let mut previous_pos_data = self.current_pos_data;
                 let mut previous_left_prop = left_prop;
 
+                if (break_state & INTERMEDIATE_MATCH_RULE) != 0 {
+                    break_state &= !INTERMEDIATE_MATCH_RULE;
+                }
                 loop {
                     self.advance_iter();
 

components/segmenter/tests/spec_test.rs

@@ -213,11 +213,42 @@ fn run_grapheme_break_test() {
 fn sentence_break_test(filename: &str) {
     let test_iter = TestContentIterator::new(filename);
     let segmenter = SentenceSegmenter::new();
-    for test in test_iter {
+    for (i, test) in test_iter.enumerate() {
         let s: String = test.utf8_vec.into_iter().collect();
         let iter = segmenter.segment_str(&s);
         let result: Vec<usize> = iter.collect();
-        assert_eq!(result, test.break_result_utf8, "{}", test.original_line);
+        if result != test.break_result_utf8 {
+            let sb = icu::properties::maps::sentence_break();
+            let sb_name = icu::properties::SentenceBreak::enum_to_long_name_mapper();
+            let mut iter = segmenter.segment_str(&s);
+            // TODO(egg): It would be really nice to have Name here.
+            println!("  | A | E | Code pt. | Sentence_Break | State | Literal");
+            for (i, c) in s.char_indices() {
+                let expected_break = test.break_result_utf8.contains(&i);
+                let actual_break = result.contains(&i);
+                if actual_break {
+                    iter.next();
+                }
+                println!(
+                    "{}| {} | {} | {:>8} | {:>14} | {} | {}",
+                    if actual_break != expected_break {
+                        "😭"
+                    } else {
+                        "  "
+                    },
+                    if actual_break { "÷" } else { "×" },
+                    if expected_break { "÷" } else { "×" },
+                    format!("{:04X}", c as u32),
+                    sb_name
+                        .get(sb.get(c))
+                        .unwrap_or(&format!("{:?}", sb.get(c))),
+                    "?".repeat(5),
+                    c
+                )
+            }
+            println!("Test case #{}", i);
+            panic!()
+        }
 
         let iter = segmenter.segment_utf16(&test.utf16_vec);
         let result: Vec<usize> = iter.collect();

provider/datagen/src/transform/segmenter/rules/sentence.toml

@@ -181,6 +181,18 @@ name = "Lower_ATerm"
 left = "Lower_ATerm"
 right = "Format"
 
+[[tables]]
+# SB5
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close_Sp_SB8"
+right = "Extend"
+
+[[tables]]
+# SB5
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close_Sp_SB8"
+right = "Format"
+
 [[tables]]
 # SB7
 name = "Upper_ATerm"
@@ -385,6 +397,41 @@ name = "STerm_Close_Sp_ParaSep"
 left = "STerm_Close_Sp"
 right = "LF"
 
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm"
+right = "Unknown"
+interm_break_state = true
+
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "Upper_ATerm"
+right = "Unknown"
+interm_break_state = true
+
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "Lower_ATerm"
+right = "Unknown"
+interm_break_state = true
+
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close"
+right = "Numeric"
+interm_break_state = true
+
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close"
+right = "Unknown"
+interm_break_state = true
+
 [[tables]]
 # SB8
 name = "ATerm_Close_Sp_SB8"
@@ -399,6 +446,13 @@ left = "ATerm_Close_Sp"
 right = "Numeric"
 interm_break_state = true
 
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close_Sp"
+right = "Unknown"
+interm_break_state = true
+
 [[tables]]
 # SB8
 name = "ATerm_Close_Sp_SB8"
@@ -417,6 +471,24 @@ name = "ATerm_Close_Sp_SB8"
 left = "ATerm_Close_Sp_SB8"
 right = "Numeric"
 
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close_Sp_SB8"
+right = "Sp"
+
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close_Sp_SB8"
+right = "Unknown"
+
+[[tables]]
+# SB8
+name = "ATerm_Close_Sp_SB8"
+left = "ATerm_Close_Sp_SB8"
+right = "SContinue"
+
 [[rules]]
 # SB1
 left = [ "sot" ]
@@ -561,19 +633,20 @@ left = [
   "STerm_Close_Sp_ParaSep",
   "STerm_Close_Sp_CR"
 ]
-right = [ "ATerm", "Lower", "OLetter", "Upper", "Numeric", "STerm" ]
+# NOTE(egg): left=Meow_CR, right=LF would be wrong, but it is prevented by another rule above.
+right = [ "ATerm", "Lower", "OLetter", "Upper", "Numeric", "STerm", "CR", "LF", "Sep" ]
 break_state = true
 
 [[rules]]
 # SB11
 left = [ "ATerm_Close_Sp", "STerm_Close_Sp" ]
-right = [ "Numeric", "Upper", "Close", "Unknown", "OLetter" ]
+right = [ "Numeric", "Upper", "Lower", "Close", "Unknown", "OLetter" ]
 break_state = true
 
 [[rules]]
 # SB11
 left = [ "ATerm_Close", "STerm", "STerm_Close" ]
-right = [ "Numeric", "Upper" ]
+right = [ "Numeric", "Upper", "Lower", "OLetter", "Unknown" ]
 break_state = true
 
 [[rules]]