Skip to content

Commit

Permalink
Read the whole sector instead of record chunks on Windows to speed up…
Browse files Browse the repository at this point in the history
… proving process
  • Loading branch information
nazar-pc committed Mar 6, 2024
1 parent 8335376 commit e8a2461
Showing 1 changed file with 23 additions and 0 deletions.
23 changes: 23 additions & 0 deletions crates/subspace-farmer-components/src/reading.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,32 @@ where
let sector_contents_map_size = SectorContentsMap::encoded_size(pieces_in_sector) as u64;
match sector {
ReadAt::Sync(sector) => {
// TODO: Random reads are slow on Windows due to a variety of bugs with its disk
// subsystem:
// * https://learn.microsoft.com/en-us/answers/questions/1601862/windows-is-leaking-memory-when-reading-random-chun
// * https://learn.microsoft.com/en-us/answers/questions/1608540/getfileinformationbyhandle-followed-by-read-with-f
// * and likely some more that are less obvious
// As a workaround, read the whole sector at once instead of individual record chunks,
// which while results in higher data transfer from SSD, reads data in larger blocks
// and ends up being faster in many cases even though it uses more RAM while doing so.
// It is also likely possible to read large parts of the sector instead of the whole
// sector if code below is not parallelized, but logic will become even more
// convoluted, so for now it is not done.
#[cfg(windows)]
let sector_bytes = {
let mut sector_bytes = vec![0u8; crate::sector::sector_size(pieces_in_sector)];
sector.read_at(&mut sector_bytes, 0)?;
sector_bytes
};
read_chunks_inputs.into_par_iter().flatten().try_for_each(
|(maybe_record_chunk, chunk_location, encoded_chunk_used, s_bucket)| {
let mut record_chunk = [0; Scalar::FULL_BYTES];
#[cfg(windows)]
record_chunk.copy_from_slice(
&sector_bytes[sector_contents_map_size as usize
+ chunk_location as usize * Scalar::FULL_BYTES..][..Scalar::FULL_BYTES],
);
#[cfg(not(windows))]
sector
.read_at(
&mut record_chunk,
Expand Down

0 comments on commit e8a2461

Please sign in to comment.