feat: compress data at build time

Cargo.toml

@@ -30,6 +30,7 @@ required-features = ["tfidf", "textrank"]
 cedarwood = "0.4"
 derive_builder = { version = "0.20.0", optional = true }
 fxhash = "0.2.1"
+include-flate = "0.3.0"
 lazy_static = "1.0"
 ordered-float = { version = "4.0", optional = true }
 phf = "0.11"
@@ -45,8 +46,4 @@ tfidf = ["dep:ordered-float", "dep:derive_builder"]
 textrank = ["dep:ordered-float", "dep:derive_builder"]
 
 [workspace]
-members = [
-    ".",
-    "capi",
-    "examples/weicheng"
-]
+members = [".", "capi", "examples/weicheng"]

src/keywords/tfidf.rs

@@ -2,13 +2,14 @@ use std::cmp::Ordering;
 use std::collections::{BTreeSet, BinaryHeap};
 use std::io::{self, BufRead, BufReader};
 
+use include_flate::flate;
 use ordered_float::OrderedFloat;
 
 use super::{Keyword, KeywordExtract, KeywordExtractConfig, KeywordExtractConfigBuilder};
 use crate::FxHashMap as HashMap;
 use crate::Jieba;
 
-static DEFAULT_IDF: &str = include_str!("../data/idf.txt");
+flate!(static DEFAULT_IDF: str from "src/data/idf.txt");
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 struct HeapNode<'a> {

src/lib.rs

@@ -71,6 +71,7 @@
 //! ```
 //!
 
+use include_flate::flate;
 use lazy_static::lazy_static;
 
 use std::cmp::Ordering;
@@ -97,7 +98,7 @@ mod keywords;
 mod sparse_dag;
 
 #[cfg(feature = "default-dict")]
-static DEFAULT_DICT: &str = include_str!("data/dict.txt");
+flate!(static DEFAULT_DICT: str from "src/data/dict.txt");
 
 use sparse_dag::StaticSparseDAG;