From 6d864a813cacc6b29c42fb0e0c53accba40c0450 Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Sun, 17 Nov 2019 21:12:10 +0000 Subject: [PATCH] fix: split state validation status into kernel and rproof updates. (#3096) * fix: split state validation status into kernel and rproof updates. And fix sync status for these two states * fix: show correct number of leaves for pruned MMR as well as unpruned * docs: better docs for kernel/range proof validation * fix: ordering of kernel and rproofs validation in TUI * fix: typo in rangeproofs api and comments --- api/src/handlers/server_api.rs | 19 ++- chain/src/txhashset/txhashset.rs | 12 +- chain/src/types.rs | 69 +++----- core/src/core/pmmr/backend.rs | 3 + core/src/core/pmmr/pmmr.rs | 6 +- core/tests/vec_backend.rs | 4 + servers/src/grin/sync/syncer.rs | 3 +- src/bin/tui/status.rs | 262 +++++++++++++++---------------- store/src/pmmr.rs | 8 + store/tests/pmmr.rs | 8 +- 10 files changed, 195 insertions(+), 199 deletions(-) diff --git a/api/src/handlers/server_api.rs b/api/src/handlers/server_api.rs index 5678577e5..fe50fb4f2 100644 --- a/api/src/handlers/server_api.rs +++ b/api/src/handlers/server_api.rs @@ -113,16 +113,19 @@ fn sync_status_to_api(sync_status: SyncStatus) -> (String, Option ( - "txhashset_validation".to_string(), - Some( - json!({ "kernels": kernels, "kernel_total": kernel_total ,"rproofs": rproofs, "rproof_total": rproof_total }), - ), + "txhashset_rangeproofs_validation".to_string(), + Some(json!({ "rproofs": rproofs, "rproofs_total": rproofs_total })), + ), + SyncStatus::TxHashsetKernelsValidation { + kernels, + kernels_total, + } => ( + "txhashset_kernels_validation".to_string(), + Some(json!({ "kernels": kernels, "kernels_total": kernels_total })), ), SyncStatus::BodySync { current_height, diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 68d233d52..11adcff23 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -1280,7 +1280,7 @@ impl<'a> Extension<'a> { TxKernel::batch_sig_verify(&tx_kernels)?; kern_count += tx_kernels.len() as u64; tx_kernels.clear(); - status.on_validation(kern_count, total_kernels, 0, 0); + status.on_validation_kernels(kern_count, total_kernels); debug!( "txhashset: verify_kernel_signatures: verified {} signatures", kern_count, @@ -1305,7 +1305,8 @@ impl<'a> Extension<'a> { let mut proofs: Vec = Vec::with_capacity(1_000); let mut proof_count = 0; - let total_rproofs = pmmr::n_leaves(self.output_pmmr.unpruned_size()); + let total_rproofs = self.output_pmmr.n_unpruned_leaves(); + for pos in self.output_pmmr.leaf_pos_iter() { let output = self.output_pmmr.get_data(pos); let proof = self.rproof_pmmr.get_data(pos); @@ -1331,10 +1332,9 @@ impl<'a> Extension<'a> { "txhashset: verify_rangeproofs: verified {} rangeproofs", proof_count, ); - } - - if proof_count % 1_000 == 0 { - status.on_validation(0, 0, proof_count, total_rproofs); + if proof_count % 1_000 == 0 { + status.on_validation_rproofs(proof_count, total_rproofs); + } } } diff --git a/chain/src/types.rs b/chain/src/types.rs index beba016f6..b672ee02c 100644 --- a/chain/src/types.rs +++ b/chain/src/types.rs @@ -65,12 +65,15 @@ pub enum SyncStatus { }, /// Setting up before validation TxHashsetSetup, - /// Validating the full state - TxHashsetValidation { + /// Validating the kernels + TxHashsetKernelsValidation { kernels: u64, - kernel_total: u64, + kernels_total: u64, + }, + /// Validating the range proofs + TxHashsetRangeProofsValidation { rproofs: u64, - rproof_total: u64, + rproofs_total: u64, }, /// Finalizing the new state TxHashsetSave, @@ -155,43 +158,18 @@ impl TxHashsetWriteStatus for SyncState { self.update(SyncStatus::TxHashsetSetup); } - fn on_validation(&self, vkernels: u64, vkernel_total: u64, vrproofs: u64, vrproof_total: u64) { - let mut status = self.current.write(); - match *status { - SyncStatus::TxHashsetValidation { - kernels, - kernel_total, - rproofs, - rproof_total, - } => { - let ks = if vkernels > 0 { vkernels } else { kernels }; - let kt = if vkernel_total > 0 { - vkernel_total - } else { - kernel_total - }; - let rps = if vrproofs > 0 { vrproofs } else { rproofs }; - let rpt = if vrproof_total > 0 { - vrproof_total - } else { - rproof_total - }; - *status = SyncStatus::TxHashsetValidation { - kernels: ks, - kernel_total: kt, - rproofs: rps, - rproof_total: rpt, - }; - } - _ => { - *status = SyncStatus::TxHashsetValidation { - kernels: 0, - kernel_total: 0, - rproofs: 0, - rproof_total: 0, - } - } - } + fn on_validation_kernels(&self, kernels: u64, kernels_total: u64) { + self.update(SyncStatus::TxHashsetKernelsValidation { + kernels, + kernels_total, + }); + } + + fn on_validation_rproofs(&self, rproofs: u64, rproofs_total: u64) { + self.update(SyncStatus::TxHashsetRangeProofsValidation { + rproofs, + rproofs_total, + }); } fn on_save(&self) { @@ -315,8 +293,10 @@ pub trait ChainAdapter { pub trait TxHashsetWriteStatus { /// First setup of the txhashset fn on_setup(&self); - /// Starting validation - fn on_validation(&self, kernels: u64, kernel_total: u64, rproofs: u64, rproof_total: u64); + /// Starting kernel validation + fn on_validation_kernels(&self, kernels: u64, kernel_total: u64); + /// Starting rproof validation + fn on_validation_rproofs(&self, rproofs: u64, rproof_total: u64); /// Starting to save the txhashset and related data fn on_save(&self); /// Done writing a new txhashset @@ -328,7 +308,8 @@ pub struct NoStatus; impl TxHashsetWriteStatus for NoStatus { fn on_setup(&self) {} - fn on_validation(&self, _ks: u64, _kts: u64, _rs: u64, _rt: u64) {} + fn on_validation_kernels(&self, _ks: u64, _kts: u64) {} + fn on_validation_rproofs(&self, _rs: u64, _rt: u64) {} fn on_save(&self) {} fn on_done(&self) {} } diff --git a/core/src/core/pmmr/backend.rs b/core/src/core/pmmr/backend.rs index c9317c4b6..6c0aae6eb 100644 --- a/core/src/core/pmmr/backend.rs +++ b/core/src/core/pmmr/backend.rs @@ -55,6 +55,9 @@ pub trait Backend { /// Iterator over current (unpruned, unremoved) leaf positions. fn leaf_pos_iter(&self) -> Box + '_>; + /// Number of leaves + fn n_unpruned_leaves(&self) -> u64; + /// Remove Hash by insertion position. An index is also provided so the /// underlying backend can implement some rollback of positions up to a /// given index (practically the index is the height of a block that diff --git a/core/src/core/pmmr/pmmr.rs b/core/src/core/pmmr/pmmr.rs index e0ae155fc..106a91d61 100644 --- a/core/src/core/pmmr/pmmr.rs +++ b/core/src/core/pmmr/pmmr.rs @@ -79,6 +79,11 @@ where self.backend.leaf_pos_iter() } + /// Number of leafs in the MMR + pub fn n_unpruned_leaves(&self) -> u64 { + self.backend.n_unpruned_leaves() + } + /// Returns a vec of the peaks of this MMR. pub fn peaks(&self) -> Vec { let peaks_pos = peaks(self.last_pos); @@ -490,7 +495,6 @@ pub fn peak_map_height(mut pos: u64) -> (u64, u64) { /// The height of a node in a full binary tree from its postorder traversal /// index. This function is the base on which all others, as well as the MMR, /// are built. - pub fn bintree_postorder_height(num: u64) -> u64 { if num == 0 { return 0; diff --git a/core/tests/vec_backend.rs b/core/tests/vec_backend.rs index 9943bef24..84427c689 100644 --- a/core/tests/vec_backend.rs +++ b/core/tests/vec_backend.rs @@ -113,6 +113,10 @@ impl Backend for VecBackend { unimplemented!() } + fn n_unpruned_leaves(&self) -> u64 { + unimplemented!() + } + fn remove(&mut self, position: u64) -> Result<(), String> { self.remove_list.push(position); Ok(()) diff --git a/servers/src/grin/sync/syncer.rs b/servers/src/grin/sync/syncer.rs index f22bc9b35..578e98fc1 100644 --- a/servers/src/grin/sync/syncer.rs +++ b/servers/src/grin/sync/syncer.rs @@ -192,7 +192,8 @@ impl SyncRunner { match self.sync_state.status() { SyncStatus::TxHashsetDownload { .. } | SyncStatus::TxHashsetSetup - | SyncStatus::TxHashsetValidation { .. } + | SyncStatus::TxHashsetRangeProofsValidation { .. } + | SyncStatus::TxHashsetKernelsValidation { .. } | SyncStatus::TxHashsetSave | SyncStatus::TxHashsetDone => check_state_sync = true, _ => { diff --git a/src/bin/tui/status.rs b/src/bin/tui/status.rs index 27596fe63..90e1bb414 100644 --- a/src/bin/tui/status.rs +++ b/src/bin/tui/status.rs @@ -31,6 +31,109 @@ const NANO_TO_MILLIS: f64 = 1.0 / 1_000_000.0; pub struct TUIStatusView; +impl TUIStatusView { + fn update_sync_status(sync_status: SyncStatus) -> String { + match sync_status { + SyncStatus::Initial => "Initializing".to_string(), + SyncStatus::NoSync => "Running".to_string(), + SyncStatus::AwaitingPeers(_) => "Waiting for peers".to_string(), + SyncStatus::HeaderSync { + current_height, + highest_height, + } => { + let percent = if highest_height == 0 { + 0 + } else { + current_height * 100 / highest_height + }; + format!("Sync step 1/7: Downloading headers: {}%", percent) + } + SyncStatus::TxHashsetDownload { + start_time, + prev_update_time, + update_time: _, + prev_downloaded_size, + downloaded_size, + total_size, + } => { + if total_size > 0 { + let percent = if total_size > 0 { + downloaded_size * 100 / total_size + } else { + 0 + }; + let start = prev_update_time.timestamp_nanos(); + let fin = Utc::now().timestamp_nanos(); + let dur_ms = (fin - start) as f64 * NANO_TO_MILLIS; + + format!("Sync step 2/7: Downloading {}(MB) chain state for state sync: {}% at {:.1?}(kB/s)", + total_size / 1_000_000, + percent, + if dur_ms > 1.0f64 { (downloaded_size - prev_downloaded_size) as f64 / dur_ms as f64 } else { 0f64 }, + ) + } else { + let start = start_time.timestamp_millis(); + let fin = Utc::now().timestamp_millis(); + let dur_secs = (fin - start) / 1000; + + format!("Sync step 2/7: Downloading chain state for state sync. Waiting remote peer to start: {}s", + dur_secs, + ) + } + } + SyncStatus::TxHashsetSetup => { + "Sync step 3/7: Preparing chain state for validation".to_string() + } + SyncStatus::TxHashsetRangeProofsValidation { + rproofs, + rproofs_total, + } => { + let r_percent = if rproofs_total > 0 { + (rproofs * 100) / rproofs_total + } else { + 0 + }; + format!( + "Sync step 4/7: Validating chain state - range proofs: {}%", + r_percent + ) + } + SyncStatus::TxHashsetKernelsValidation { + kernels, + kernels_total, + } => { + let k_percent = if kernels_total > 0 { + (kernels * 100) / kernels_total + } else { + 0 + }; + format!( + "Sync step 5/7: Validating chain state - kernels: {}%", + k_percent + ) + } + SyncStatus::TxHashsetSave => { + "Sync step 6/7: Finalizing chain state for state sync".to_string() + } + SyncStatus::TxHashsetDone => { + "Sync step 6/7: Finalized chain state for state sync".to_string() + } + SyncStatus::BodySync { + current_height, + highest_height, + } => { + let percent = if highest_height == 0 { + 0 + } else { + current_height * 100 / highest_height + }; + format!("Sync step 7/7: Downloading blocks: {}%", percent) + } + SyncStatus::Shutdown => "Shutting down, closing connections".to_string(), + } + } +} + impl TUIStatusListener for TUIStatusView { /// Create basic status view fn create() -> Box { @@ -143,137 +246,9 @@ impl TUIStatusListener for TUIStatusView { Box::new(basic_status_view.with_id(VIEW_BASIC_STATUS)) } - /// update fn update(c: &mut Cursive, stats: &ServerStats) { - //find and update here as needed - let basic_status = { - match stats.sync_status { - SyncStatus::Initial => "Initializing".to_string(), - SyncStatus::NoSync => "Running".to_string(), - SyncStatus::AwaitingPeers(_) => "Waiting for peers".to_string(), - SyncStatus::HeaderSync { - current_height, - highest_height, - } => { - let percent = if highest_height == 0 { - 0 - } else { - current_height * 100 / highest_height - }; - format!("Downloading headers: {}%, step 1/4", percent) - } - SyncStatus::TxHashsetDownload { - start_time, - prev_update_time, - update_time: _, - prev_downloaded_size, - downloaded_size, - total_size, - } => { - if total_size > 0 { - let percent = if total_size > 0 { - downloaded_size * 100 / total_size - } else { - 0 - }; - let start = prev_update_time.timestamp_nanos(); - let fin = Utc::now().timestamp_nanos(); - let dur_ms = (fin - start) as f64 * NANO_TO_MILLIS; + let basic_status = TUIStatusView::update_sync_status(stats.sync_status); - format!("Downloading {}(MB) chain state for state sync: {}% at {:.1?}(kB/s), step 2/4", - total_size / 1_000_000, - percent, - if dur_ms > 1.0f64 { (downloaded_size - prev_downloaded_size) as f64 / dur_ms as f64 } else { 0f64 }, - ) - } else { - let start = start_time.timestamp_millis(); - let fin = Utc::now().timestamp_millis(); - let dur_secs = (fin - start) / 1000; - - format!("Downloading chain state for state sync. Waiting remote peer to start: {}s, step 2/4", - dur_secs, - ) - } - } - SyncStatus::TxHashsetSetup => { - "Preparing chain state for validation, step 3/4".to_string() - } - SyncStatus::TxHashsetValidation { - kernels, - kernel_total, - rproofs, - rproof_total, - } => { - // 10% of overall progress is attributed to kernel validation - // 90% to range proofs (which are much longer) - let mut percent = if kernel_total > 0 { - kernels * 10 / kernel_total - } else { - 0 - }; - percent += if rproof_total > 0 { - rproofs * 90 / rproof_total - } else { - 0 - }; - format!("Validating chain state: {}%, step 3/4", percent) - } - SyncStatus::TxHashsetSave => { - "Finalizing chain state for state sync, step 3/4".to_string() - } - SyncStatus::TxHashsetDone => { - "Finalized chain state for state sync, step 3/4".to_string() - } - SyncStatus::BodySync { - current_height, - highest_height, - } => { - let percent = if highest_height == 0 { - 0 - } else { - current_height * 100 / highest_height - }; - format!("Downloading blocks: {}%, step 4/4", percent) - } - SyncStatus::Shutdown => "Shutting down, closing connections".to_string(), - } - }; - /*let basic_mining_config_status = { - if stats.mining_stats.is_enabled { - "Configured as mining node" - } else { - "Configured as validating node only (not mining)" - } - }; - let (basic_mining_status, basic_network_info) = { - if stats.mining_stats.is_enabled { - if stats.is_syncing { - ( - "Mining Status: Paused while syncing".to_string(), - " ".to_string(), - ) - } else if stats.mining_stats.combined_gps == 0.0 { - ( - "Mining Status: Starting miner and awaiting first solution...".to_string(), - " ".to_string(), - ) - } else { - ( - format!( - "Mining Status: Mining at height {} at {:.*} GPS", - stats.mining_stats.block_height, 4, stats.mining_stats.combined_gps - ), - format!( - "Cuckoo {} - Network Difficulty {}", - stats.mining_stats.edge_bits, - stats.mining_stats.network_difficulty.to_string() - ), - ) - } - } else { - (" ".to_string(), " ".to_string()) - } - };*/ c.call_on_id("basic_current_status", |t: &mut TextView| { t.set_content(basic_status); }); @@ -321,14 +296,25 @@ impl TUIStatusListener for TUIStatusView { c.call_on_id("stem_pool_kernels", |t: &mut TextView| { t.set_content(stats.tx_stats.stem_pool_kernels.to_string()); }); - /*c.call_on_id("basic_mining_config_status", |t: &mut TextView| { - t.set_content(basic_mining_config_status); - }); - c.call_on_id("basic_mining_status", |t: &mut TextView| { - t.set_content(basic_mining_status); - }); - c.call_on_id("basic_network_info", |t: &mut TextView| { - t.set_content(basic_network_info); - });*/ } } + +#[test] +fn test_status_txhashset_kernels() { + let status = SyncStatus::TxHashsetKernelsValidation { + kernels: 201, + kernels_total: 5000, + }; + let basic_status = TUIStatusView::update_sync_status(status); + assert!(basic_status.contains("4%"), basic_status); +} + +#[test] +fn test_status_txhashset_rproofs() { + let status = SyncStatus::TxHashsetRangeProofsValidation { + rproofs: 643, + rproofs_total: 1000, + }; + let basic_status = TUIStatusView::update_sync_status(status); + assert!(basic_status.contains("64%"), basic_status); +} diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs index 846179990..3d5680c57 100644 --- a/store/src/pmmr.rs +++ b/store/src/pmmr.rs @@ -140,6 +140,14 @@ impl Backend for PMMRBackend { } } + fn n_unpruned_leaves(&self) -> u64 { + if self.prunable { + self.leaf_set.len() as u64 + } else { + pmmr::n_leaves(self.unpruned_size()) + } + } + fn data_as_temp_file(&self) -> Result { self.data_file .as_temp_file() diff --git a/store/tests/pmmr.rs b/store/tests/pmmr.rs index cd72bbbc6..779f4a5eb 100644 --- a/store/tests/pmmr.rs +++ b/store/tests/pmmr.rs @@ -34,7 +34,7 @@ fn pmmr_append() { { let mut backend = store::pmmr::PMMRBackend::new( data_dir.to_string(), - true, + false, false, ProtocolVersion(1), None, @@ -53,6 +53,7 @@ fn pmmr_append() { // Note: 1-indexed PMMR API let pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size); + assert_eq!(pmmr.n_unpruned_leaves(), 4); assert_eq!(pmmr.get_data(1), Some(elems[0])); assert_eq!(pmmr.get_data(2), Some(elems[1])); @@ -88,6 +89,8 @@ fn pmmr_append() { // Note: 1-indexed PMMR API let pmmr: PMMR<'_, TestElem, _> = PMMR::at(&mut backend, mmr_size); + assert_eq!(pmmr.n_unpruned_leaves(), 9); + // First pair of leaves. assert_eq!(pmmr.get_data(1), Some(elems[0])); assert_eq!(pmmr.get_data(2), Some(elems[1])); @@ -132,6 +135,7 @@ fn pmmr_compact_leaf_sibling() { let mmr_size = load(0, &elems[..], &mut backend); backend.sync().unwrap(); + assert_eq!(backend.n_unpruned_leaves(), 19); // On far left of the MMR - // pos 1 and 2 are leaves (and siblings) // the parent is pos 3 @@ -159,6 +163,8 @@ fn pmmr_compact_leaf_sibling() { { let pmmr = PMMR::at(&mut backend, mmr_size); + assert_eq!(pmmr.n_unpruned_leaves(), 17); + // check that pos 1 is "removed" assert_eq!(pmmr.get_hash(1), None);