Skip to content

Commit

Permalink
Merge pull request #313 from autonomys/update-subspace
Browse files Browse the repository at this point in the history 
Update Subspace to latest `main` revision + related changes
  • Loading branch information
@nazar-pc
nazar-pc authored Oct 23, 2024
2 parents 674cfc3 + b9647ed commit e9659cc
Show file tree
Hide file tree
Showing 10 changed files with 340 additions and 271 deletions.
386 changes: 218 additions & 168 deletions Cargo.lock
View file

Large diffs are not rendered by default.

50 changes: 25 additions & 25 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ merge-assets.append = [
include = ["res/windows/wix/space-acres.wxs"]

[dependencies]
anyhow = "1.0.89"
anyhow = "1.0.91"
arc-swap = "1.7.1"
async-lock = "3.4.0"
async-oneshot = "0.5.9"
Expand All @@ -53,7 +53,7 @@ futures = "0.3.31"
futures-timer = "3.0.3"
gtk = { version = "0.7.3", package = "gtk4", features = ["v4_14"] }
hex = "0.4.3"
image = { version = "0.25.2", default-features = false, features = ["png"] }
image = { version = "0.25.4", default-features = false, features = ["png"] }
mimalloc = "0.1.43"
names = "0.14.0"
notify-rust = { version = "4.11.3", features = ["images"] }
Expand All @@ -68,47 +68,47 @@ reqwest = { version = "0.12.8", default-features = false, features = ["json", "r
sc-client-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-client-db = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-consensus-slots = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
sc-informant = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-network = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-network-types = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-rpc = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-service = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-storage-monitor = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
sc-utils = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
schnellru = "0.2.3"
semver = "1.0.23"
serde = { version = "1.0.210", features = ["derive"] }
serde_json = "1.0.128"
serde = { version = "1.0.213", features = ["derive"] }
serde_json = "1.0.132"
simple_moving_average = "1.0.2"
sp-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sp-blockchain = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sp-consensus = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
sp-core = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
sp-objects = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
sp-objects = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
sp-runtime = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42", default-features = false }
subspace-archiving = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-fake-runtime-api = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0", default-features = false }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-kzg = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-proof-of-space-gpu = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0", optional = true }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-service = { git = "https://github.com/subspace/subspace", rev = "6c0c789a674cba083718bc09512c32923fd5fde0" }
subspace-archiving = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-fake-runtime-api = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c", default-features = false }
subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-kzg = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-networking = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-proof-of-space-gpu = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c", optional = true }
subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
subspace-service = { git = "https://github.com/subspace/subspace", rev = "7c0db0ffe2f97e025b28de5f3b6dc878258ff20c" }
supports-color = "3.0.1"
sys-locale = "0.3.1"
tempfile = "3.13.0"
thiserror = "1.0.64"
thiserror = "1.0.65"
thread-priority = "1.1.0"
tokio = { version = "1.40.0", features = ["fs", "time"] }
tokio = { version = "1.41.0", features = ["fs", "time"] }
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
tracker = "0.2.2"
Expand All @@ -118,7 +118,7 @@ native-dialog = "0.7.0"
tracing-panic = "0.1.2"

[target.'cfg(any(target_os = "macos", target_os = "windows"))'.dependencies]
tray-icon = "0.19.0"
tray-icon = "0.19.1"

[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies]
ksni = "0.2.2"
Expand Down
2 changes: 1 addition & 1 deletion rust-toolchain.toml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[toolchain]
channel = "nightly-2024-09-26"
channel = "nightly-2024-10-22"
components = ["rust-src"]
targets = []
profile = "default"
48 changes: 29 additions & 19 deletions src/backend.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ use async_lock::RwLock as AsyncRwLock;
use backoff::ExponentialBackoff;
use future::FutureExt;
use futures::channel::mpsc;
use futures::{future, select, SinkExt, StreamExt};
use futures::{future, select, SinkExt, Stream, StreamExt};
use sc_subspace_chain_specs::DEVNET_CHAIN_SPEC;
use sp_consensus_subspace::ChainConstants;
use std::error::Error;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::num::{NonZeroU8, NonZeroUsize};
use std::num::NonZeroU8;
use std::path::{Path, PathBuf};
use std::pin::pin;
use std::sync::{Arc, Weak};
Expand Down Expand Up @@ -60,9 +60,6 @@ pub type CacheIndex = u8;

/// Get piece retry attempts number.
const PIECE_GETTER_MAX_RETRIES: u16 = 7;
/// Global limit on combined piece getter, a nice number that should result in enough pieces
/// downloading successfully during DSN sync
const PIECE_GETTER_MAX_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(128).expect("Not zero; qed");
/// Defines initial duration between get_piece calls.
const GET_PIECE_INITIAL_INTERVAL: Duration = Duration::from_secs(5);
/// Defines max duration between get_piece calls.
Expand Down Expand Up @@ -90,12 +87,21 @@ impl DsnSyncPieceGetter for PieceGetterWrapper {

#[async_trait::async_trait]
impl PieceGetter for PieceGetterWrapper {
async fn get_piece(
&self,
piece_index: PieceIndex,
) -> Result<Option<Piece>, Box<dyn Error + Send + Sync + 'static>> {
async fn get_piece(&self, piece_index: PieceIndex) -> anyhow::Result<Option<Piece>> {
self.0.get_piece(piece_index).await
}

async fn get_pieces<'a, PieceIndices>(
&'a self,
piece_indices: PieceIndices,
) -> anyhow::Result<
Box<dyn Stream<Item = (PieceIndex, anyhow::Result<Option<Piece>>)> + Send + Unpin + 'a>,
>
where
PieceIndices: IntoIterator<Item = PieceIndex, IntoIter: Send> + Send + 'a,
{
self.0.get_pieces(piece_indices).await
}
}

impl PieceGetterWrapper {
Expand Down Expand Up @@ -127,12 +133,21 @@ struct WeakPieceGetterWrapper(

#[async_trait::async_trait]
impl PieceGetter for WeakPieceGetterWrapper {
async fn get_piece(
&self,
piece_index: PieceIndex,
) -> Result<Option<Piece>, Box<dyn Error + Send + Sync + 'static>> {
async fn get_piece(&self, piece_index: PieceIndex) -> anyhow::Result<Option<Piece>> {
self.0.get_piece(piece_index).await
}

async fn get_pieces<'a, PieceIndices>(
&'a self,
piece_indices: PieceIndices,
) -> anyhow::Result<
Box<dyn Stream<Item = (PieceIndex, anyhow::Result<Option<Piece>>)> + Send + Unpin + 'a>,
>
where
PieceIndices: IntoIterator<Item = PieceIndex, IntoIter: Send> + Send + 'a,
{
self.0.get_pieces(piece_indices).await
}
}

/// Major steps in application loading progress
Expand Down Expand Up @@ -423,11 +438,7 @@ async fn load(
let kzg = Kzg::new();
let piece_provider = PieceProvider::new(
node.clone(),
Some(SegmentCommitmentPieceValidator::new(
node.clone(),
maybe_node_client.clone(),
kzg.clone(),
)),
SegmentCommitmentPieceValidator::new(node.clone(), maybe_node_client.clone(), kzg.clone()),
);

let piece_getter = PieceGetterWrapper::new(FarmerPieceGetter::new(
Expand All @@ -446,7 +457,6 @@ async fn load(
..ExponentialBackoff::default()
},
},
PIECE_GETTER_MAX_CONCURRENCY,
));

let create_consensus_node_fut = create_consensus_node(
Expand Down
2 changes: 2 additions & 0 deletions src/backend/farmer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ pub(super) const CACHE_PERCENTAGE: NonZeroU8 = NonZeroU8::MIN;
/// very long period of writing zeroes on Windows, see https://stackoverflow.com/q/78058306/3806795
const MAX_SPACE_PLEDGED_FOR_PLOT_CACHE_ON_WINDOWS: u64 = ByteSize::tib(7).as_u64();
const FARM_ERROR_PRINT_INTERVAL: Duration = Duration::from_secs(30);
const MAX_PLOTTING_SECTORS_PER_FARM: NonZeroUsize = NonZeroUsize::new(4).expect("Not zero; qed");

#[derive(Debug, Default, Copy, Clone, PartialEq)]
pub struct InitialFarmState {
Expand Down Expand Up @@ -452,6 +453,7 @@ where
farming_thread_pool_size: recommended_number_of_farming_threads(),
plotting_delay: Some(plotting_delay_receiver),
global_mutex,
max_plotting_sectors_per_farm: MAX_PLOTTING_SECTORS_PER_FARM,
disable_farm_locking: false,
read_sector_record_chunks_mode: None,
faster_read_sector_record_chunks_mode_barrier,
Expand Down
34 changes: 17 additions & 17 deletions src/backend/farmer/direct_node_client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use subspace_core_primitives::segments::{HistorySize, SegmentHeader, SegmentInde
use subspace_core_primitives::solutions::Solution;
use subspace_core_primitives::{BlockHash, PublicKey, SlotNumber};
use subspace_erasure_coding::ErasureCoding;
use subspace_farmer::node_client::{Error, NodeClient, NodeClientExt};
use subspace_farmer::node_client::{NodeClient, NodeClientExt};
use subspace_farmer_components::FarmerProtocolInfo;
use subspace_kzg::Kzg;
use subspace_networking::libp2p::Multiaddr;
Expand Down Expand Up @@ -189,7 +189,7 @@ where
+ 'static,
Client::Api: SubspaceApi<Block, PublicKey> + ObjectsApi<Block> + 'static,
{
async fn farmer_app_info(&self) -> Result<FarmerAppInfo, Error> {
async fn farmer_app_info(&self) -> anyhow::Result<FarmerAppInfo> {
let last_segment_index = self
.segment_headers_store
.max_segment_index()
Expand Down Expand Up @@ -219,14 +219,14 @@ where

farmer_app_info.map_err(|error| {
error!("Failed to get data from runtime API: {}", error);
"Internal error".to_string().into()
anyhow::anyhow!("Internal error")
})
}

async fn submit_solution_response(
&self,
solution_response: SolutionResponse,
) -> Result<(), Error> {
) -> anyhow::Result<()> {
let slot = solution_response.slot_number;
let mut solution_response_senders = self.solution_response_senders.lock();

Expand All @@ -241,14 +241,14 @@ where
"Solution was ignored, likely because farmer was too slow"
);

return Err("Solution was ignored".to_string().into());
return Err(anyhow::anyhow!("Solution was ignored"));
}

Ok(())
}
async fn subscribe_slot_info(
&self,
) -> Result<Pin<Box<dyn Stream<Item = SlotInfo> + Send + 'static>>, Error> {
) -> anyhow::Result<Pin<Box<dyn Stream<Item = SlotInfo> + Send + 'static>>> {
let executor = self.subscription_executor.clone();
let solution_response_senders = self.solution_response_senders.clone();

Expand Down Expand Up @@ -330,7 +330,7 @@ where

async fn subscribe_reward_signing(
&self,
) -> Result<Pin<Box<dyn Stream<Item = RewardSigningInfo> + Send + 'static>>, Error> {
) -> anyhow::Result<Pin<Box<dyn Stream<Item = RewardSigningInfo> + Send + 'static>>> {
let executor = self.subscription_executor.clone();
let reward_signature_senders = self.reward_signature_senders.clone();

Expand Down Expand Up @@ -393,7 +393,7 @@ where
async fn submit_reward_signature(
&self,
reward_signature: RewardSignatureResponse,
) -> Result<(), Error> {
) -> anyhow::Result<()> {
let reward_signature_senders = self.reward_signature_senders.clone();

let mut reward_signature_senders = reward_signature_senders.lock();
Expand All @@ -409,7 +409,7 @@ where

async fn subscribe_archived_segment_headers(
&self,
) -> Result<Pin<Box<dyn Stream<Item = SegmentHeader> + Send + 'static>>, Error> {
) -> anyhow::Result<Pin<Box<dyn Stream<Item = SegmentHeader> + Send + 'static>>> {
let archived_segment_acknowledgement_senders =
self.archived_segment_acknowledgement_senders.clone();

Expand Down Expand Up @@ -494,7 +494,7 @@ where
async fn acknowledge_archived_segment_header(
&self,
segment_index: SegmentIndex,
) -> Result<(), Error> {
) -> anyhow::Result<()> {
let archived_segment_acknowledgement_senders =
self.archived_segment_acknowledgement_senders.clone();

Expand Down Expand Up @@ -529,7 +529,7 @@ where
Ok(())
}

async fn piece(&self, piece_index: PieceIndex) -> Result<Option<Piece>, Error> {
async fn piece(&self, piece_index: PieceIndex) -> anyhow::Result<Option<Piece>> {
let archived_segment = {
let mut cached_archived_segment = self.cached_archived_segment.lock().await;

Expand Down Expand Up @@ -569,14 +569,14 @@ where
Ok(Err(error)) => {
error!(%error, "Failed to re-create genesis segment");

return Err("Failed to re-create genesis segment".to_string().into());
return Err(anyhow::anyhow!("Failed to re-create genesis segment"));
}
Err(error) => {
error!(%error, "Blocking task failed to re-create genesis segment");

return Err("Blocking task failed to re-create genesis segment"
.to_string()
.into());
return Err(anyhow::anyhow!(
"Blocking task failed to re-create genesis segment"
));
}
}
}
Expand All @@ -594,7 +594,7 @@ where
async fn segment_headers(
&self,
segment_indexes: Vec<SegmentIndex>,
) -> Result<Vec<Option<SegmentHeader>>, Error> {
) -> anyhow::Result<Vec<Option<SegmentHeader>>> {
Ok(segment_indexes
.into_iter()
.map(|segment_index| self.segment_headers_store.get_segment_header(segment_index))
Expand All @@ -614,7 +614,7 @@ where
+ 'static,
Client::Api: SubspaceApi<Block, PublicKey> + ObjectsApi<Block> + 'static,
{
async fn last_segment_headers(&self, limit: u32) -> Result<Vec<Option<SegmentHeader>>, Error> {
async fn last_segment_headers(&self, limit: u32) -> anyhow::Result<Vec<Option<SegmentHeader>>> {
let last_segment_index = self
.segment_headers_store
.max_segment_index()
Expand Down
Loading

0 comments on commit e9659cc

Please sign in to comment.