fix tests

milli/src/index.rs

@@ -808,7 +808,7 @@ pub(crate) mod tests {
     use maplit::btreemap;
     use tempfile::TempDir;
 
-    use crate::update::{IndexDocuments, UpdateFormat};
+    use crate::update::{IndexDocuments};
     use crate::Index;
 
     pub(crate) struct TempIndex {
@@ -844,13 +844,12 @@ pub(crate) mod tests {
         let index = Index::new(options, &path).unwrap();
 
         let mut wtxn = index.write_txn().unwrap();
-        let content = &br#"[
+        let content = documents!([
             { "id": 1, "name": "kevin" },
             { "id": 2, "name": "bob", "age": 20 },
             { "id": 2, "name": "bob", "age": 20 }
-        ]"#[..];
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        ]);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
         builder.execute(content, |_, _| ()).unwrap();
         wtxn.commit().unwrap();
 
@@ -869,8 +868,12 @@ pub(crate) mod tests {
         // we add all the documents a second time. we are supposed to get the same
         // field_distribution in the end
         let mut wtxn = index.write_txn().unwrap();
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
+        let content = documents!([
+            { "id": 1, "name": "kevin" },
+            { "id": 2, "name": "bob", "age": 20 },
+            { "id": 2, "name": "bob", "age": 20 }
+        ]);
         builder.execute(content, |_, _| ()).unwrap();
         wtxn.commit().unwrap();
 
@@ -887,13 +890,12 @@ pub(crate) mod tests {
         );
 
         // then we update a document by removing one field and another by adding one field
-        let content = &br#"[
+        let content = documents!([
             { "id": 1, "name": "kevin", "has_dog": true },
             { "id": 2, "name": "bob" }
-        ]"#[..];
+        ]);
         let mut wtxn = index.write_txn().unwrap();
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
         builder.execute(content, |_, _| ()).unwrap();
         wtxn.commit().unwrap();
 

milli/src/lib.rs

@@ -1,6 +1,9 @@
 #[macro_use]
 extern crate pest_derive;
 
+#[macro_use]
+pub mod documents;
+
 mod criterion;
 mod error;
 mod external_documents_ids;
@@ -12,7 +15,6 @@ pub mod proximity;
 mod search;
 pub mod tree_level;
 pub mod update;
-pub mod documents;
 
 use std::borrow::Cow;
 use std::collections::{BTreeMap, HashMap};

milli/src/search/distinct/mod.rs

@@ -27,21 +27,24 @@ pub trait Distinct {
 #[cfg(test)]
 mod test {
     use std::collections::HashSet;
+    use std::io::Cursor;
 
+    use bimap::BiHashMap;
     use once_cell::sync::Lazy;
     use rand::seq::SliceRandom;
     use rand::Rng;
     use roaring::RoaringBitmap;
     use serde_json::{json, Value};
 
+    use crate::documents::{DocumentsBuilder, DocumentsReader};
     use crate::index::tests::TempIndex;
     use crate::index::Index;
-    use crate::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat};
+    use crate::update::{IndexDocumentsMethod, UpdateBuilder};
     use crate::{DocumentId, FieldId, BEU32};
 
-    static JSON: Lazy<Value> = Lazy::new(generate_json);
+    static JSON: Lazy<Vec<u8>> = Lazy::new(generate_json);
 
-    fn generate_json() -> Value {
+    fn generate_json() -> Vec<u8> {
         let mut rng = rand::thread_rng();
         let num_docs = rng.gen_range(10..30);
 
@@ -69,7 +72,13 @@ mod test {
             documents.push(doc);
         }
 
-        Value::Array(documents)
+        let mut cursor = Cursor::new(Vec::new());
+        let mut builder = DocumentsBuilder::new(&mut cursor, BiHashMap::new()).unwrap();
+
+        builder.add_documents(documents).unwrap();
+        builder.finish().unwrap();
+
+        cursor.into_inner()
     }
 
     /// Returns a temporary index populated with random test documents, the FieldId for the
@@ -89,13 +98,14 @@ mod test {
         let mut addition = builder.index_documents(&mut txn, &index);
 
         addition.index_documents_method(IndexDocumentsMethod::ReplaceDocuments);
-        addition.update_format(UpdateFormat::Json);
-        addition.execute(JSON.to_string().as_bytes(), |_, _| ()).unwrap();
+        let reader = crate::documents::DocumentsReader::from_reader(Cursor::new(&*JSON)).unwrap();
+        addition.execute(reader, |_, _| ()).unwrap();
 
         let fields_map = index.fields_ids_map(&txn).unwrap();
         let fid = fields_map.id(&distinct).unwrap();
 
-        let map = (0..JSON.as_array().unwrap().len() as u32).collect();
+        let documents = DocumentsReader::from_reader(Cursor::new(&*JSON)).unwrap();
+        let map = (0..documents.len() as u32).collect();
 
         txn.commit().unwrap();
 

milli/src/update/clear_documents.rs

@@ -80,7 +80,7 @@ mod tests {
     use heed::EnvOpenOptions;
 
     use super::*;
-    use crate::update::{IndexDocuments, UpdateFormat};
+    use crate::update::{IndexDocuments};
 
     #[test]
     fn clear_documents() {
@@ -90,13 +90,12 @@ mod tests {
         let index = Index::new(options, &path).unwrap();
 
         let mut wtxn = index.write_txn().unwrap();
-        let content = &br#"[
+        let content = documents!([
             { "id": 0, "name": "kevin", "age": 20 },
             { "id": 1, "name": "kevina" },
             { "id": 2, "name": "benoit", "country": "France" }
-        ]"#[..];
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        ]);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
         builder.execute(content, |_, _| ()).unwrap();
 
         // Clear all documents from the database.

milli/src/update/delete_documents.rs

@@ -548,7 +548,7 @@ mod tests {
     use maplit::hashset;
 
     use super::*;
-    use crate::update::{IndexDocuments, Settings, UpdateFormat};
+    use crate::update::{IndexDocuments, Settings};
     use crate::FilterCondition;
 
     #[test]
@@ -559,13 +559,12 @@ mod tests {
         let index = Index::new(options, &path).unwrap();
 
         let mut wtxn = index.write_txn().unwrap();
-        let content = &br#"[
+        let content = documents!([
             { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
             { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
             { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
-        ]"#[..];
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        ]);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
         builder.execute(content, |_, _| ()).unwrap();
 
         // delete those documents, ids are synchronous therefore 0, 1, and 2.
@@ -590,13 +589,12 @@ mod tests {
         let index = Index::new(options, &path).unwrap();
 
         let mut wtxn = index.write_txn().unwrap();
-        let content = &br#"[
+        let content = documents!([
             { "mysuperid": 0, "name": "kevin" },
             { "mysuperid": 1, "name": "kevina" },
             { "mysuperid": 2, "name": "benoit" }
-        ]"#[..];
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        ]);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
         builder.execute(content, |_, _| ()).unwrap();
 
         // Delete not all of the documents but some of them.
@@ -621,7 +619,7 @@ mod tests {
         builder.set_filterable_fields(hashset! { S("label") });
         builder.execute(|_, _| ()).unwrap();
 
-        let content = &br#"[
+        let content = documents!([
             {"docid":"1_4","label":"sign"},
             {"docid":"1_5","label":"letter"},
             {"docid":"1_7","label":"abstract,cartoon,design,pattern"},
@@ -642,9 +640,8 @@ mod tests {
             {"docid":"1_58","label":"abstract,art,cartoon"},
             {"docid":"1_68","label":"design"},
             {"docid":"1_69","label":"geometry"}
-        ]"#[..];
-        let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
-        builder.update_format(UpdateFormat::Json);
+        ]);
+        let builder = IndexDocuments::new(&mut wtxn, &index, 0);
         builder.execute(content, |_, _| ()).unwrap();
 
         // Delete not all of the documents but some of them.