Skip to content

Commit

Permalink
Add real values for atime, ctime, and mtime
Browse files Browse the repository at this point in the history
  • Loading branch information
@dannycjones
dannycjones committed Jan 16, 2023
1 parent 58b896a commit d2a4910
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 33 deletions.
20 changes: 5 additions & 15 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 12 additions & 6 deletions s3-client/src/mock_client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ impl MockClient {
pub struct MockObject {
generator: Box<dyn Fn(u64, usize) -> Box<[u8]> + Send + Sync>,
size: usize,
pub last_modified: OffsetDateTime,
}

impl MockObject {
Expand All @@ -84,13 +85,15 @@ impl MockObject {
Self {
size: bytes.len(),
generator: Box::new(move |offset, size| bytes[offset as usize..offset as usize + size].into()),
last_modified: OffsetDateTime::now_utc(),
}
}

pub fn constant(v: u8, size: usize) -> Self {
Self {
generator: Box::new(move |_offset, size| vec![v; size].into_boxed_slice()),
size,
last_modified: OffsetDateTime::now_utc(),
}
}

Expand All @@ -108,9 +111,14 @@ impl MockObject {
vec.into_boxed_slice()
}),
size,
last_modified: OffsetDateTime::now_utc(),
}
}

pub fn with_last_modified(&mut self, last_modified: OffsetDateTime) {
self.last_modified = last_modified;
}

pub fn len(&self) -> usize {
self.size
}
Expand Down Expand Up @@ -244,8 +252,7 @@ impl ObjectClient for MockClient {
object: ObjectInfo {
key: key.to_string(),
size: object.size as u64,
// TODO real mtimes
last_modified: OffsetDateTime::from_unix_timestamp(0).unwrap(),
last_modified: object.last_modified,
etag: "TODO".to_string(),
storage_class: None,
},
Expand Down Expand Up @@ -283,7 +290,7 @@ impl ObjectClient for MockClient {
// start at the beginning of the bucket.
let object_iterator = objects.range(continuation_token.unwrap_or("").to_string()..);

for (key, value) in object_iterator {
for (key, object) in object_iterator {
// If the prefix is `n` characters long, and we encounter a key whose first `n`
// characters are lexicographically larger than the prefix, then we can stop iterating.
// Note that we cannot just do a direct comparison between the full key and prefix. For
Expand Down Expand Up @@ -340,9 +347,8 @@ impl ObjectClient for MockClient {
} else {
object_vec.push(ObjectInfo {
key: key.to_string(),
size: value.len() as u64,
// TODO real mtimes
last_modified: OffsetDateTime::from_unix_timestamp(0).unwrap(),
size: object.len() as u64,
last_modified: object.last_modified,
etag: "TODO".to_string(),
storage_class: None,
});
Expand Down
1 change: 1 addition & 0 deletions s3-file-connector/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ sha2 = "0.10.6"
shuttle = { version = "0.5.0" }
tempfile = "3.3.0"
test-case = "2.2.2"
time = { version = "0.3.17", features = ["macros"] }
tokio = { version = "1.23.1", features = ["rt", "macros"] }

[features]
Expand Down
6 changes: 3 additions & 3 deletions s3-file-connector/src/fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,9 +199,9 @@ where
ino,
size: stat.size as u64,
blocks: stat.size as u64 / BLOCK_SIZE,
atime: UNIX_EPOCH,
mtime: UNIX_EPOCH, // TODO
ctime: UNIX_EPOCH,
atime: stat.atime,
mtime: stat.mtime,
ctime: stat.ctime,
crtime: UNIX_EPOCH,
kind: (&stat.kind).into(),
perm,
Expand Down
66 changes: 57 additions & 9 deletions s3-file-connector/src/inode.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
use std::collections::{HashMap, VecDeque};
use std::ffi::{OsStr, OsString};
use std::os::unix::prelude::OsStrExt;
use std::time::Instant;
use std::time::{Instant, SystemTime, UNIX_EPOCH};

use fuser::FileType;
use futures::{select_biased, FutureExt};
Expand Down Expand Up @@ -83,6 +83,9 @@ impl Superblock {
stat_cache: RwLock::new(InodeStat {
kind: InodeStatKind::Directory {},
size: 0,
atime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
}),
stat_cache_expiry: Instant::now(),
data: InodeData::Directory {
Expand Down Expand Up @@ -177,6 +180,9 @@ impl Superblock {
let stat = InodeStat {
kind: InodeStatKind::Directory {},
size: 0,
atime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
};
let ino =
self.inner
Expand All @@ -194,9 +200,13 @@ impl Superblock {
.map(|object| object.key == full_path.to_str().unwrap())
.unwrap_or(false)
{
let last_modified: SystemTime = result.objects[0].last_modified.into();
let stat = InodeStat {
kind: InodeStatKind::File {},
size: result.objects[0].size as usize,
atime: last_modified,
ctime: last_modified,
mtime: last_modified,
};
file_state = Some(stat);
}
Expand Down Expand Up @@ -244,6 +254,9 @@ impl Superblock {
let stat = InodeStat {
kind: InodeStatKind::Directory {},
size: 0,
atime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
};
let ino =
self.inner
Expand Down Expand Up @@ -532,6 +545,9 @@ impl ReaddirHandle {
let stat = InodeStat {
kind: InodeStatKind::Directory {},
size: 0,
atime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
};
let stat_clone = stat.clone();

Expand All @@ -556,9 +572,13 @@ impl ReaddirHandle {
// Hide keys that end with '/', since they can be confused with directories
.filter(|(name, _object)| valid_inode_name(name))
.flat_map(|(name, object)| {
let last_modified: SystemTime = object.last_modified.into();
let stat = InodeStat {
kind: InodeStatKind::File {},
size: object.size as usize,
atime: last_modified,
ctime: last_modified,
mtime: last_modified,
};
let stat_clone = stat.clone();

Expand Down Expand Up @@ -674,9 +694,16 @@ enum InodeData {
/// Public inode stat data that can expire
#[derive(Debug, Clone)]
pub struct InodeStat {
// common metadata: mtime, ctime, ...
/// Size in bytes
pub size: usize,

/// Time of last file content modification
pub mtime: SystemTime,
/// Time of last file metadata (or content) change
pub ctime: SystemTime,
/// Time of last access
pub atime: SystemTime,

/// Per-kind metadata
pub kind: InodeStatKind,
}
Expand Down Expand Up @@ -730,6 +757,8 @@ pub struct DirEntryPlus {
mod tests {
use s3_client::mock_client::{MockClient, MockClientConfig, MockObject};
use test_case::test_case;
use time::macros::datetime;
use time::OffsetDateTime;

use crate::fs::FUSE_ROOT_INODE;

Expand Down Expand Up @@ -759,8 +788,11 @@ mod tests {
format!("{}dir1/sdir3/file1.txt", prefix),
];

const SOME_LAST_MODIFIED_TIME: OffsetDateTime = datetime!(2023-01-01 0:00 +0);
for key in keys {
client.add_object(key, MockObject::constant(0xaa, 30));
let mut obj = MockObject::constant(0xaa, 30);
obj.with_last_modified(SOME_LAST_MODIFIED_TIME);
client.add_object(key, obj);
}

let superblock = Superblock::new("test_bucket".to_string(), OsString::from(prefix));
Expand All @@ -771,44 +803,60 @@ mod tests {
.lookup(&client, FUSE_ROOT_INODE, &OsString::from("dir0"))
.await
.expect("should exist");
assert_eq!(dir0.stat.kind, InodeStatKind::Directory {});

let dir_stat_assertions = |stat: InodeStat| {
assert_eq!(stat.kind, InodeStatKind::Directory {});
assert_eq!(stat.atime, OffsetDateTime::UNIX_EPOCH);
assert_eq!(stat.ctime, OffsetDateTime::UNIX_EPOCH);
assert_eq!(stat.mtime, OffsetDateTime::UNIX_EPOCH);
};

dir_stat_assertions(dir0.stat);
assert_eq!(dir0.full_key, OsString::from(format!("{}dir0", prefix)));

let dir1 = superblock
.lookup(&client, FUSE_ROOT_INODE, &OsString::from("dir1"))
.await
.expect("should exist");
assert_eq!(dir1.stat.kind, InodeStatKind::Directory {});
dir_stat_assertions(dir1.stat);
assert_eq!(dir1.full_key, OsString::from(format!("{}dir1", prefix)));

let sdir0 = superblock
.lookup(&client, dir0.ino, &OsString::from("sdir0"))
.await
.expect("should exist");
assert_eq!(sdir0.stat.kind, InodeStatKind::Directory {});
dir_stat_assertions(sdir0.stat);
assert_eq!(sdir0.full_key, OsString::from(format!("{}dir0/sdir0", prefix)));

let sdir1 = superblock
.lookup(&client, dir0.ino, &OsString::from("sdir1"))
.await
.expect("should exist");
assert_eq!(sdir1.stat.kind, InodeStatKind::Directory {});
dir_stat_assertions(sdir1.stat);
assert_eq!(sdir1.full_key, OsString::from(format!("{}dir0/sdir1", prefix)));

let sdir2 = superblock
.lookup(&client, dir1.ino, &OsString::from("sdir2"))
.await
.expect("should exist");
assert_eq!(sdir2.stat.kind, InodeStatKind::Directory {});
dir_stat_assertions(sdir2.stat);
assert_eq!(sdir2.full_key, OsString::from(format!("{}dir1/sdir2", prefix)));

let sdir3 = superblock
.lookup(&client, dir1.ino, &OsString::from("sdir3"))
.await
.expect("should exist");
assert_eq!(sdir3.stat.kind, InodeStatKind::Directory {});
dir_stat_assertions(sdir3.stat);
assert_eq!(sdir3.full_key, OsString::from(format!("{}dir1/sdir3", prefix)));

let file0 = superblock
.lookup(&client, dir0.ino, &OsString::from("file0.txt"))
.await
.expect("should exist");
assert_eq!(file0.stat.kind, InodeStatKind::File {});
assert_eq!(file0.stat.atime, SOME_LAST_MODIFIED_TIME);
assert_eq!(file0.stat.ctime, SOME_LAST_MODIFIED_TIME);
assert_eq!(file0.stat.mtime, SOME_LAST_MODIFIED_TIME);
assert_eq!(file0.full_key, OsString::from(format!("{}dir0/file0.txt", prefix)));

for (dir, sdir, ino, n) in &[
Expand Down
1 change: 1 addition & 0 deletions s3-file-connector/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ fn main() -> anyhow::Result<()> {
MountOption::RO,
MountOption::DefaultPermissions,
MountOption::FSName("fuse_sync".to_string()),
MountOption::NoAtime,
];
if args.auto_unmount {
options.push(MountOption::AutoUnmount);
Expand Down

0 comments on commit d2a4910

Please sign in to comment.