From 767ebad054588ec2b99433cef6922283042de2fe Mon Sep 17 00:00:00 2001
From: WhiredPlanck <whiredplanck@outlook.com>
Date: Mon, 27 Nov 2023 00:12:55 +0800
Subject: [PATCH] fix: don't compress the token during collecting dict entries
 (#762) (#768)

follow the default split behavior of boost::split
---
 src/rime/algo/encoder.cc | 3 ++-
 src/rime/algo/strings.cc | 6 +++---
 src/rime/algo/strings.h  | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/rime/algo/encoder.cc b/src/rime/algo/encoder.cc
index 8f117a96d5..64290424a8 100644
--- a/src/rime/algo/encoder.cc
+++ b/src/rime/algo/encoder.cc
@@ -20,7 +20,8 @@ string RawCode::ToString() const {
 }
 
 void RawCode::FromString(const string& code_str) {
-  *dynamic_cast<vector<string>*>(this) = strings::split(code_str, " ");
+  *dynamic_cast<vector<string>*>(this) =
+      strings::split(code_str, " ", strings::SplitBehavior::SkipToken);
 }
 
 TableEncoder::TableEncoder(PhraseCollector* collector)
diff --git a/src/rime/algo/strings.cc b/src/rime/algo/strings.cc
index a0ab82626a..acbb040ae9 100644
--- a/src/rime/algo/strings.cc
+++ b/src/rime/algo/strings.cc
@@ -8,7 +8,7 @@ vector<string> split(const string& str,
                      SplitBehavior behavior) {
   vector<string> strings;
   size_t lastPos, pos;
-  if (behavior == SplitBehavior::SkipEmpty) {
+  if (behavior == SplitBehavior::SkipToken) {
     lastPos = str.find_first_not_of(delim, 0);
   } else {
     lastPos = 0;
@@ -17,7 +17,7 @@ vector<string> split(const string& str,
 
   while (std::string::npos != pos || std::string::npos != lastPos) {
     strings.emplace_back(str.substr(lastPos, pos - lastPos));
-    if (behavior == SplitBehavior::SkipEmpty) {
+    if (behavior == SplitBehavior::SkipToken) {
       lastPos = str.find_first_not_of(delim, pos);
     } else {
       if (pos == std::string::npos) {
@@ -31,7 +31,7 @@ vector<string> split(const string& str,
 };
 
 vector<string> split(const string& str, const string& delim) {
-  return split(str, delim, SplitBehavior::SkipEmpty);
+  return split(str, delim, SplitBehavior::KeepToken);
 };
 
 }  // namespace strings
diff --git a/src/rime/algo/strings.h b/src/rime/algo/strings.h
index 2bd570f1c6..222476d2fa 100644
--- a/src/rime/algo/strings.h
+++ b/src/rime/algo/strings.h
@@ -7,7 +7,7 @@
 namespace rime {
 namespace strings {
 
-enum class SplitBehavior { KeepEmpty, SkipEmpty };
+enum class SplitBehavior { KeepToken, SkipToken };
 
 vector<string> split(const string& str,
                      const string& delim,