Fix support for larger than 2 GB databases on Windows

Cargo.toml

@@ -28,7 +28,7 @@ members = [
 [dependencies]
 bitflags = "1"
 libc = "0.2"
-lmdb-sys = "0.8.0"
+lmdb-sys = { path = "lmdb-sys" }
 
 [dev-dependencies]
 rand = "0.4"

lmdb-sys/build.rs

@@ -16,6 +16,20 @@ fn main() {
         if target.contains("android") {
             build.define("ANDROID", "1");
         }
+        if target.contains("windows") {
+            // LMDB on Windows has an issue with ´off_t´ being defined as ´long´ 32-bit signed
+            // which caused a max size of the database of 2 GB which we work 
+            // around by redefining ´off_t´ to 64-bit
+            // 
+            // was discussed here and allgedly fixed in January 2016 but fix doesn't work as 
+            // Windows doesn't support _FILE_OFFSET_BITS
+            // https://www.openldap.org/lists/openldap-bugs/201605/msg00015.htm
+            // https://github.com/LMDB/lmdb/commit/20dec1f69bf4860202c764ce92b1fbbe3d11a065
+            build.define("_OFF_T_DEFINED", "1");
+            build.define("off_t", "__int64");
+            build.define("_off_t", "__int64");
+        }
+
         build
             .file(lmdb.join("mdb.c"))
             .file(lmdb.join("midl.c"))

src/lib.rs

@@ -120,4 +120,58 @@ mod test_utils {
             tx.commit().expect("tx.commit")
         }
     }
+
+    // verify that the map file is sparsly allocated
+    // this used to fail on earlier versions of LMDB on Windows, before ITS#8324
+    #[test]
+    fn verify_sparse() {
+        const HEIGHT_KEY: [u8; 1] = [0];
+
+        let dir = TempDir::new("test").unwrap();
+
+        {
+            let env = {
+                let mut builder = Environment::new();
+                builder.set_map_size(1_000_000_000);
+                builder.open(dir.path()).expect("open lmdb env")
+            };
+            let db = env.open_db(None).unwrap();
+
+            for height in 0..1000 {
+                let mut value = [0u8; 8];
+                LittleEndian::write_u64(&mut value, height);
+                let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
+                tx.put(db, &HEIGHT_KEY, &value, WriteFlags::empty())
+                    .expect("tx.put");
+                tx.commit().expect("tx.commit")
+            }
+        }
+
+        let size = std::fs::metadata(dir.path().join("data.mdb"))
+            .expect("get file size")
+            .len();
+        assert!(size < 1_000_000);
+    }
+
+
+    // Verify that one can create a database larger than 2 GB
+    #[test]
+    fn verify_2gb_plus() {
+        let dir = TempDir::new("test").unwrap();
+            let env = Environment::new()
+                .set_map_size(4_000_000_000)
+                .open(dir.path()).expect("open lmdb env");
+            let db = env.open_db(None).unwrap();
+
+            let data: Vec<u8> = (0..1_000_000).into_iter().map(|i| i as u8).collect();
+
+            // try to write 3 GB of data
+            let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
+            for i in 0..3000 {
+                let key = &data[i..(i+32)];
+                tx.put(db, &key, &data, WriteFlags::empty()).expect("tx.put");
+            }
+            tx.commit().expect("tx.commit")
+    }
+
 }