From a42fbbc40c012bf1abc264b4613ca1d065be9688 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 1 Mar 2024 16:36:07 +0200 Subject: [PATCH 1/4] Add `FILE_FLAG_WRITE_THROUGH` to Windows random access flag in order to make sure it doesn't use as much memory --- crates/subspace-farmer-components/src/file_ext.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/subspace-farmer-components/src/file_ext.rs b/crates/subspace-farmer-components/src/file_ext.rs index 83516356a0..164d32932c 100644 --- a/crates/subspace-farmer-components/src/file_ext.rs +++ b/crates/subspace-farmer-components/src/file_ext.rs @@ -30,7 +30,14 @@ impl OpenOptionsExt for OpenOptions { #[cfg(windows)] fn advise_random_access(&mut self) -> &mut Self { use std::os::windows::fs::OpenOptionsExt; - self.custom_flags(winapi::um::winbase::FILE_FLAG_RANDOM_ACCESS) + // `FILE_FLAG_WRITE_THROUGH` below is a bit of a hack, especially in `advise_random_access`, + // but it helps with memory usage and feels like should be default. Since `.custom_flags()` + // overrides previous value, we need to set bitwise OR of two flags rather that two flags + // separately. + self.custom_flags( + winapi::um::winbase::FILE_FLAG_RANDOM_ACCESS + | winapi::um::winbase::FILE_FLAG_WRITE_THROUGH, + ) } #[cfg(target_os = "linux")] From 61a1e3dacfef51c6f4bfba1e453981548892e52b Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 1 Mar 2024 16:41:51 +0200 Subject: [PATCH 2/4] Do not store in plot cache pieces that are already in piece cache --- crates/subspace-farmer/src/farmer_cache.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/subspace-farmer/src/farmer_cache.rs b/crates/subspace-farmer/src/farmer_cache.rs index 93b798bf0f..66856af392 100644 --- a/crates/subspace-farmer/src/farmer_cache.rs +++ b/crates/subspace-farmer/src/farmer_cache.rs @@ -886,10 +886,18 @@ impl FarmerCache { let should_store_fut = tokio::task::spawn_blocking({ let plot_caches = Arc::clone(&self.plot_caches); + let piece_caches = Arc::clone(&self.piece_caches); let next_plot_cache = Arc::clone(&self.next_plot_cache); let piece = piece.clone(); move || { + for cache in piece_caches.read().iter() { + if cache.stored_pieces.contains_key(&key) { + // Already stored in normal piece cache, no need to store it again + return; + } + } + let plot_caches = plot_caches.read(); let plot_caches_len = plot_caches.len(); From 3bfff27aaa4246706c5f5156871f0241830ee123 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 1 Mar 2024 17:10:34 +0200 Subject: [PATCH 3/4] Avoid plot cache for very large plots (piece cache should be quite large in that case anyway) --- .../src/single_disk_farm/plot_cache.rs | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs b/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs index a22221fa33..f74f810825 100644 --- a/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs +++ b/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs @@ -16,6 +16,13 @@ use subspace_networking::utils::multihash::ToMultihash; use thiserror::Error; use tracing::{debug, info, warn}; +/// Max plot space for which to use caching, for larger gaps between the plotted part and the end of +/// the file it will result in very long period of writing zeroes on Windows, see +/// https://stackoverflow.com/q/78058306/3806795 +/// +/// Currently set to 2TiB. +const MAX_WINDOWS_PLOT_SPACE_FOR_CACHE: u64 = 2 * 1024 * 1024 * 1024 * 1024; + /// Disk plot cache open error #[derive(Debug, Error)] pub enum DiskPlotCacheError { @@ -68,33 +75,36 @@ impl DiskPlotCache { // Clippy complains about `RecordKey`, but it is not changing here, so it is fine #[allow(clippy::mutable_key_type)] let mut map = HashMap::new(); + let mut next_offset = None; let file_size = sector_size * u64::from(target_sector_count); let plotted_size = sector_size * sectors_metadata.len() as u64; - // Step over all free potential offsets for pieces that could have been cached - let from_offset = (plotted_size / Self::element_size() as u64) as u32; - let to_offset = (file_size / Self::element_size() as u64) as u32; - let mut next_offset = None; - // TODO: Parallelize or read in larger batches - for offset in (from_offset..to_offset).rev() { - match Self::read_piece_internal(file, offset, &mut element) { - Ok(maybe_piece_index) => match maybe_piece_index { - Some(piece_index) => { - map.insert(RecordKey::from(piece_index.to_multihash()), offset); - } - None => { + // Avoid writing over large gaps on Windows that is very lengthy process + if !cfg!(windows) || (file_size - plotted_size) <= MAX_WINDOWS_PLOT_SPACE_FOR_CACHE { + // Step over all free potential offsets for pieces that could have been cached + let from_offset = (plotted_size / Self::element_size() as u64) as u32; + let to_offset = (file_size / Self::element_size() as u64) as u32; + // TODO: Parallelize or read in larger batches + for offset in (from_offset..to_offset).rev() { + match Self::read_piece_internal(file, offset, &mut element) { + Ok(maybe_piece_index) => match maybe_piece_index { + Some(piece_index) => { + map.insert(RecordKey::from(piece_index.to_multihash()), offset); + } + None => { + next_offset.replace(offset); + break; + } + }, + Err(DiskPlotCacheError::ChecksumMismatch) => { next_offset.replace(offset); break; } - }, - Err(DiskPlotCacheError::ChecksumMismatch) => { - next_offset.replace(offset); - break; - } - Err(error) => { - warn!(%error, %offset, "Failed to read plot cache element"); - break; + Err(error) => { + warn!(%error, %offset, "Failed to read plot cache element"); + break; + } } } } From df919f963c236f4cfb0366cda8f5409a8a3fb301 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 1 Mar 2024 22:47:30 +0200 Subject: [PATCH 4/4] Fix compilation on macOS --- crates/subspace-farmer/src/utils.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/subspace-farmer/src/utils.rs b/crates/subspace-farmer/src/utils.rs index 0fb73e7ce9..6b003db43f 100644 --- a/crates/subspace-farmer/src/utils.rs +++ b/crates/subspace-farmer/src/utils.rs @@ -20,9 +20,7 @@ use std::{io, thread}; use thread_priority::{set_current_thread_priority, ThreadPriority}; use tokio::runtime::Handle; use tokio::task; -use tracing::debug; -#[cfg(feature = "numa")] -use tracing::warn; +use tracing::{debug, warn}; /// It doesn't make a lot of sense to have a huge number of farming threads, 32 is plenty const MAX_DEFAULT_FARMING_THREADS: usize = 32;