test(turborepo): Create Cache Tests (#5371)

### Description Tests for creating a cache. These do have different hashes than the Go ones, but I think that's fine since there's no expectation that tar construction or zstd compression is necessarily deterministic. ### Testing Instructions --------- Co-authored-by: --global <Nicholas Yang>
vercel · Jul 6, 2023 · 0a92abc · 0a92abc
1 parent 7ee94f5
commit 0a92abc
Show file tree

Hide file tree

Showing 7 changed files with 278 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -177,6 +177,7 @@ dialoguer = "0.10.3"
 dunce = "1.0.3"
 futures = "0.3.26"
 futures-retry = "0.6.0"
+hex = "0.4.3"
 httpmock = { version = "0.6.7", default-features = false }
 image = { version = "0.24.6", default-features = false }
 indexmap = "1.9.2"

diff --git a/crates/turborepo-cache/Cargo.toml b/crates/turborepo-cache/Cargo.toml
@@ -12,6 +12,7 @@ rustls-tls = ["turborepo-api-client/rustls-tls"]
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] }
+libc = "0.2.146"
 tempfile = { workspace = true }
 test-case = { workspace = true }
 
@@ -21,6 +22,7 @@ bytes.workspace = true
 camino = { workspace = true }
 chrono = { workspace = true }
 dunce = { workspace = true }
+hex = { workspace = true }
 lazy_static = { workspace = true }
 os_str_bytes = "6.5.0"
 path-clean = { workspace = true }

diff --git a/crates/turborepo-cache/src/cache_archive/create.rs b/crates/turborepo-cache/src/cache_archive/create.rs
@@ -26,6 +26,10 @@ impl CacheWriter {
         Ok(self.builder.append_data(header, path, body)?)
     }
 
+    pub fn finish(mut self) -> Result<(), CacheError> {
+        Ok(self.builder.finish()?)
+    }
+
     // Makes a new CacheArchive at the specified path
     // Wires up the chain of writers:
     // tar::Builder -> zstd::Encoder (optional) -> BufWriter -> File
@@ -133,9 +137,229 @@ mod tests {
 
     use anyhow::Result;
     use tempfile::tempdir;
-    use turbopath::AbsoluteSystemPath;
+    use test_case::test_case;
+    use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf};
 
     use super::*;
+    use crate::cache_archive::restore::CacheReader;
+
+    #[derive(Debug)]
+    enum FileType {
+        Dir,
+        Symlink { linkname: String },
+        Fifo,
+        File,
+    }
+
+    #[derive(Debug)]
+    struct CreateFileDefinition {
+        path: AnchoredSystemPathBuf,
+        mode: u32,
+        file_type: FileType,
+    }
+
+    fn create_entry(anchor: &AbsoluteSystemPath, file: &CreateFileDefinition) -> Result<()> {
+        match &file.file_type {
+            FileType::Dir => create_dir(anchor, file),
+            FileType::Symlink { linkname } => create_symlink(anchor, file, &linkname),
+            FileType::Fifo => create_fifo(anchor, file),
+            FileType::File => create_file(anchor, file),
+        }
+    }
+
+    fn create_dir(anchor: &AbsoluteSystemPath, file: &CreateFileDefinition) -> Result<()> {
+        let path = anchor.resolve(&file.path);
+        path.create_dir_all()?;
+
+        #[cfg(unix)]
+        {
+            path.set_mode(file.mode & 0o777)?;
+        }
+
+        Ok(())
+    }
+
+    fn create_symlink(
+        anchor: &AbsoluteSystemPath,
+        file: &CreateFileDefinition,
+        linkname: &str,
+    ) -> Result<()> {
+        let path = anchor.resolve(&file.path);
+        path.symlink_to_file(&linkname)?;
+
+        Ok(())
+    }
+
+    #[cfg(unix)]
+    fn create_fifo(anchor: &AbsoluteSystemPath, file: &CreateFileDefinition) -> Result<()> {
+        use std::ffi::CString;
+
+        let path = anchor.resolve(&file.path);
+        let path_cstr = CString::new(path.as_str())?;
+
+        unsafe {
+            libc::mkfifo(path_cstr.as_ptr(), 0o644);
+        }
+
+        Ok(())
+    }
+
+    #[cfg(windows)]
+    fn create_fifo(_: &AbsoluteSystemPath, _: &CreateFileDefinition) -> Result<()> {
+        Err(CacheError::CreateUnsupportedFileType(Backtrace::capture()).into())
+    }
+
+    fn create_file(anchor: &AbsoluteSystemPath, file: &CreateFileDefinition) -> Result<()> {
+        let path = anchor.resolve(&file.path);
+        fs::write(&path, b"file contents")?;
+        #[cfg(unix)]
+        {
+            path.set_mode(file.mode & 0o777)?;
+        }
+
+        Ok(())
+    }
+
+    #[test_case(
+      vec![
+         CreateFileDefinition {
+           path: AnchoredSystemPathBuf::from_raw("hello world.txt").unwrap(),
+           mode: 0o644,
+           file_type: FileType::File,
+         }
+      ],
+      "db05810ef8714bc849a27d2b78a267c03862cd5259a5c7fb916e92a1ef912da68a4c92032d8e984e241e12fb85a4b41574009922d740c7e66faf50a00682003c",
+      "db05810ef8714bc849a27d2b78a267c03862cd5259a5c7fb916e92a1ef912da68a4c92032d8e984e241e12fb85a4b41574009922d740c7e66faf50a00682003c",
+      "224fda5e3b1db1e4a7ede1024e09ea70add3243ce1227d28b3f8fc40bca98e14d381efe4e8affc4fef8eb21b4ff42753f9923aac60038680602c117b15748ca1",
+      None
+    )]
+    #[test_case(
+        vec![
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("one").unwrap(),
+                mode: 0o777,
+                file_type: FileType::Symlink { linkname: "two".to_string() },
+            },
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("two").unwrap(),
+                mode: 0o777,
+                file_type: FileType::Symlink { linkname: "three".to_string() },
+            },
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("three").unwrap(),
+                mode: 0o777,
+                file_type: FileType::Symlink { linkname: "real".to_string() },
+            },
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("real").unwrap(),
+                mode: 0o777,
+                file_type: FileType::File,
+            }
+        ],
+        "7cb91627c62368cfa15160f9f018de3320ee0cf267625d37265d162ae3b0dea64b8126aac9769922796e3eb864189efd7c5555c4eea8999c91cbbbe695852111",
+        "04f27e900a4a189cf60ce21e1864ac3f77c3bc9276026a94329a5314e20a3f2671e2ac949025840f46dc9fe72f9f566f1f2c0848a3f203ba77564fae204e886c",
+        "1a618c123f9f09bbca9052121d13eea3192fa3addc61eb11f6dcb794f1093abba204510d126ca1f974d5db9a6e728c1e5d3b7c099faf904e494476277178d657",
+        None
+    )]
+    #[test_case(
+        vec![
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("parent").unwrap(),
+                mode: 0o777,
+                file_type: FileType::Dir,
+            },
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("parent/child").unwrap(),
+                mode: 0o644,
+                file_type: FileType::File,
+            },
+        ],
+        "919de777e4d43eb072939d2e0664f9df533bd24ec357eacab83dcb8a64e2723f3ee5ecb277d1cf24538339fe06d210563188052d08dab146a8463fdb6898d655",
+        "919de777e4d43eb072939d2e0664f9df533bd24ec357eacab83dcb8a64e2723f3ee5ecb277d1cf24538339fe06d210563188052d08dab146a8463fdb6898d655",
+        "f12ff4c12722f2c901885c67d232c325b604d54e5b67c35da01ab133fd36e637bf8d2501b463ffb6e4438efaf2a59526a85218e00c0f6b7b5594c8f4154c1ece",
+        None
+    )]
+    #[test_case(
+        vec![
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("one").unwrap(),
+                mode: 0o644,
+                file_type: FileType::Symlink { linkname: "two".to_string() },
+            },
+        ],
+        "40ce0d42109bb5e5a6b1d4ba9087a317b4c1c6c51822a57c9cb983f878b0ff765637c05fadd4bac32c8dd2b496c2a24825b183d9720b0cdd5b33f9248b692cc1",
+        "c113763393a9fb498cc676e1fe4843206cda665afe2144829fe7434da9e81f0cf6d11386fa79877d3c514d108f9696740256af952b57d32216fbed2eb2fb049d",
+        "fe692a000551a60da6cc303a9552a16d7ed5c462e33153a96824e96596da6d642fc671448f06f34e9685a13fe5bbb4220f59db73a856626b8a0962916a8f5ea3",
+        None
+    )]
+    #[test_case(
+        vec![
+            CreateFileDefinition {
+                path: AnchoredSystemPathBuf::from_raw("one").unwrap(),
+                mode: 0o644,
+                file_type: FileType::Fifo,
+            }
+        ],
+        "",
+        "",
+        "",
+        Some("attempted to create unsupported file type")
+    )]
+    fn test_create(
+        files: Vec<CreateFileDefinition>,
+        #[allow(unused_variables)] expected_darwin: &str,
+        #[allow(unused_variables)] expected_unix: &str,
+        #[allow(unused_variables)] expected_windows: &str,
+        #[allow(unused_variables)] expected_err: Option<&str>,
+    ) -> Result<()> {
+        'outer: for compressed in [false, true] {
+            let input_dir = tempdir()?;
+            let archive_dir = tempdir()?;
+            let input_dir_path = AbsoluteSystemPathBuf::try_from(input_dir.path())?;
+            let archive_path = if compressed {
+                AbsoluteSystemPathBuf::try_from(archive_dir.path().join("out.tar.zst"))?
+            } else {
+                AbsoluteSystemPathBuf::try_from(archive_dir.path().join("out.tar"))?
+            };
+
+            let mut cache_archive = CacheWriter::create(&archive_path)?;
+
+            for file in files.iter() {
+                let result = create_entry(&input_dir_path, file);
+                if let Err(err) = result {
+                    assert!(expected_err.is_some());
+                    assert_eq!(err.to_string(), expected_err.unwrap());
+                    continue 'outer;
+                }
+
+                let result = cache_archive.add_file(&input_dir_path, &file.path);
+                if let Err(err) = result {
+                    assert!(expected_err.is_some());
+                    assert_eq!(err.to_string(), expected_err.unwrap());
+                    continue 'outer;
+                }
+            }
+
+            cache_archive.finish()?;
+
+            if compressed {
+                let opened_cache_archive = CacheReader::open(&archive_path)?;
+                let sha_one = opened_cache_archive.get_sha()?;
+                let snapshot = hex::encode(&sha_one);
+
+                #[cfg(target_os = "macos")]
+                assert_eq!(snapshot, expected_darwin);
+
+                #[cfg(windows)]
+                assert_eq!(snapshot, expected_windows);
+
+                #[cfg(all(unix, not(target_os = "macos")))]
+                assert_eq!(snapshot, expected_unix);
+            }
+        }
+
+        Ok(())
+    }
 
     #[test]
     #[cfg(unix)]
@@ -178,4 +402,21 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_compression() -> Result<()> {
+        let mut buffer = Vec::new();
+        let mut encoder = zstd::Encoder::new(&mut buffer, 0)?.auto_finish();
+        encoder.write(b"hello world")?;
+        // Should finish encoding on drop
+        drop(encoder);
+
+        let mut decoder = zstd::Decoder::new(&buffer[..])?;
+        let mut out = String::new();
+        decoder.read_to_string(&mut out)?;
+
+        assert_eq!(out, "hello world");
+
+        Ok(())
+    }
 }
diff --git a/crates/turborepo-cache/src/cache_archive/restore.rs b/crates/turborepo-cache/src/cache_archive/restore.rs
@@ -1,6 +1,7 @@
 use std::{backtrace::Backtrace, collections::HashMap, io::Read};
 
 use petgraph::graph::DiGraph;
+use ring::digest::{Context, SHA512};
 use tar::Entry;
 use turbopath::{AbsoluteSystemPath, AbsoluteSystemPathBuf, AnchoredSystemPathBuf};
 
@@ -15,7 +16,7 @@ use crate::{
     CacheError,
 };
 
-struct CacheReader {
+pub struct CacheReader {
     reader: Box<dyn Read>,
 }
 
@@ -33,8 +34,9 @@ impl CacheReader {
 
     pub fn open(path: &AbsoluteSystemPathBuf) -> Result<Self, CacheError> {
         let file = path.open()?;
+        let is_compressed = path.extension() == Some("zst");
 
-        let reader: Box<dyn Read> = if path.extension() == Some("zst") {
+        let reader: Box<dyn Read> = if is_compressed {
             Box::new(zstd::Decoder::new(file)?)
         } else {
             Box::new(file)
@@ -43,6 +45,20 @@ impl CacheReader {
         Ok(CacheReader { reader })
     }
 
+    pub fn get_sha(mut self) -> Result<Vec<u8>, CacheError> {
+        let mut context = Context::new(&SHA512);
+        let mut buffer = [0; 8192];
+        loop {
+            let n = self.reader.read(&mut buffer)?;
+            if n == 0 {
+                break;
+            }
+            context.update(&buffer[..n]);
+        }
+
+        Ok(context.finish().as_ref().to_vec())
+    }
+
     pub fn restore(
         &mut self,
         anchor: &AbsoluteSystemPath,

diff --git a/crates/turborepo-paths/src/absolute_system_path.rs b/crates/turborepo-paths/src/absolute_system_path.rs
@@ -293,6 +293,16 @@ impl AbsoluteSystemPath {
     pub fn open_with_options(&self, open_options: OpenOptions) -> Result<File, io::Error> {
         open_options.open(&self.0)
     }
+
+    #[cfg(unix)]
+    pub fn set_mode(&self, mode: u32) -> Result<(), io::Error> {
+        use std::{fs::Permissions, os::unix::fs::PermissionsExt};
+
+        let permissions = Permissions::from_mode(mode);
+        fs::set_permissions(&self.0, permissions)?;
+
+        Ok(())
+    }
 }
 
 #[cfg(test)]

diff --git a/crates/turborepo-scm/Cargo.toml b/crates/turborepo-scm/Cargo.toml
@@ -10,7 +10,7 @@ license = "MPL-2.0"
 bstr = "1.4.0"
 git2 = { version = "0.16.1", default-features = false }
 globwalk = { path = "../turborepo-globwalk" }
-hex = "0.4.3"
+hex = { workspace = true }
 ignore = "0.4.20"
 itertools.workspace = true
 nom = "7.1.3"