From bf815aa5cd5f3bbce968275a38e5b0753136fb75 Mon Sep 17 00:00:00 2001 From: Antioch Peverell Date: Wed, 5 Dec 2018 14:52:53 +0000 Subject: [PATCH] Readonly pmmr cleanup (#2083) * avoid locking txhashset by using a readonly PMMR * rustfmt --- chain/src/chain.rs | 18 +++--- chain/src/txhashset/txhashset.rs | 70 +++++++++------------ core/src/core/pmmr/pmmr.rs | 47 -------------- core/src/core/pmmr/readonly_pmmr.rs | 91 ++++++++++++++++++++++++++- core/src/core/pmmr/rewindable_pmmr.rs | 10 ++- core/tests/pmmr.rs | 24 ++++--- 6 files changed, 150 insertions(+), 110 deletions(-) diff --git a/chain/src/chain.rs b/chain/src/chain.rs index ef7cb086e..2cdd1b53d 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -583,7 +583,7 @@ impl Chain { Ok(()) } - /// Return a pre-built Merkle proof for the given commitment from the store. + /// Return a Merkle proof for the given commitment from the store. pub fn get_merkle_proof( &self, output: &OutputIdentifier, @@ -606,10 +606,9 @@ impl Chain { txhashset.merkle_proof(commit) } - /// Returns current txhashset roots + /// Returns current txhashset roots. pub fn get_txhashset_roots(&self) -> TxHashSetRoots { - let mut txhashset = self.txhashset.write(); - txhashset.roots() + self.txhashset.read().roots() } /// Provides a reading view into the current txhashset state as well as @@ -967,20 +966,17 @@ impl Chain { /// returns the last n nodes inserted into the output sum tree pub fn get_last_n_output(&self, distance: u64) -> Vec<(Hash, OutputIdentifier)> { - let mut txhashset = self.txhashset.write(); - txhashset.last_n_output(distance) + self.txhashset.read().last_n_output(distance) } /// as above, for rangeproofs pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, RangeProof)> { - let mut txhashset = self.txhashset.write(); - txhashset.last_n_rangeproof(distance) + self.txhashset.read().last_n_rangeproof(distance) } /// as above, for kernels pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> { - let mut txhashset = self.txhashset.write(); - txhashset.last_n_kernel(distance) + self.txhashset.read().last_n_kernel(distance) } /// outputs by insertion index @@ -989,7 +985,7 @@ impl Chain { start_index: u64, max: u64, ) -> Result<(u64, u64, Vec), Error> { - let mut txhashset = self.txhashset.write(); + let txhashset = self.txhashset.read(); let max_index = txhashset.highest_output_insertion_index(); let outputs = txhashset.outputs_by_insertion_index(start_index, max); let rangeproofs = txhashset.rangeproofs_by_insertion_index(start_index, max); diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 09d7c96da..ba123bc22 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -163,7 +163,7 @@ impl TxHashSet { pub fn is_unspent(&self, output_id: &OutputIdentifier) -> Result<(Hash, u64), Error> { match self.commit_index.get_output_pos(&output_id.commit) { Ok(pos) => { - let output_pmmr: ReadonlyPMMR = + let output_pmmr = ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); if let Some(hash) = output_pmmr.get_hash(pos) { if hash == output_id.hash_with_index(pos - 1) { @@ -184,34 +184,30 @@ impl TxHashSet { /// nodes at level 0 /// TODO: These need to return the actual data from the flat-files instead /// of hashes now - pub fn last_n_output(&mut self, distance: u64) -> Vec<(Hash, OutputIdentifier)> { - let output_pmmr: PMMR = - PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - output_pmmr.get_last_n_insertions(distance) + pub fn last_n_output(&self, distance: u64) -> Vec<(Hash, OutputIdentifier)> { + ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos) + .get_last_n_insertions(distance) } /// as above, for range proofs - pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<(Hash, RangeProof)> { - let rproof_pmmr: PMMR = - PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); - rproof_pmmr.get_last_n_insertions(distance) + pub fn last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, RangeProof)> { + ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos) + .get_last_n_insertions(distance) } /// as above, for kernels - pub fn last_n_kernel(&mut self, distance: u64) -> Vec<(Hash, TxKernelEntry)> { - let kernel_pmmr: PMMR = - PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); - kernel_pmmr.get_last_n_insertions(distance) + pub fn last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> { + ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos) + .get_last_n_insertions(distance) } /// Get the header at the specified height based on the current state of the txhashset. /// Derives the MMR pos from the height (insertion index) and retrieves the header hash. /// Looks the header up in the db by hash. - pub fn get_header_by_height(&mut self, height: u64) -> Result { + pub fn get_header_by_height(&self, height: u64) -> Result { let pos = pmmr::insertion_to_pmmr_index(height + 1); - - let header_pmmr: PMMR = - PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); + let header_pmmr = + ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); if let Some(hash) = header_pmmr.get_data(pos) { let header = self.commit_index.get_block_header(&hash)?; Ok(header) @@ -223,41 +219,39 @@ impl TxHashSet { /// returns outputs from the given insertion (leaf) index up to the /// specified limit. Also returns the last index actually populated pub fn outputs_by_insertion_index( - &mut self, + &self, start_index: u64, max_count: u64, ) -> (u64, Vec) { - let output_pmmr: PMMR = - PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - output_pmmr.elements_from_insertion_index(start_index, max_count) + ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos) + .elements_from_insertion_index(start_index, max_count) } /// highest output insertion index available - pub fn highest_output_insertion_index(&mut self) -> u64 { + pub fn highest_output_insertion_index(&self) -> u64 { pmmr::n_leaves(self.output_pmmr_h.last_pos) } /// As above, for rangeproofs pub fn rangeproofs_by_insertion_index( - &mut self, + &self, start_index: u64, max_count: u64, ) -> (u64, Vec) { - let rproof_pmmr: PMMR = - PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); - rproof_pmmr.elements_from_insertion_index(start_index, max_count) + ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos) + .elements_from_insertion_index(start_index, max_count) } /// Get MMR roots. - pub fn roots(&mut self) -> TxHashSetRoots { - let header_pmmr: PMMR = - PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); - let output_pmmr: PMMR = - PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - let rproof_pmmr: PMMR = - PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); - let kernel_pmmr: PMMR = - PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); + pub fn roots(&self) -> TxHashSetRoots { + let header_pmmr = + ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); + let output_pmmr = + ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); + let rproof_pmmr = + ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); + let kernel_pmmr = + ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); TxHashSetRoots { header_root: header_pmmr.root(), @@ -267,12 +261,10 @@ impl TxHashSet { } } - /// build a new merkle proof for the given position + /// build a new merkle proof for the given position. pub fn merkle_proof(&mut self, commit: Commitment) -> Result { let pos = self.commit_index.get_output_pos(&commit).unwrap(); - let output_pmmr: PMMR = - PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - output_pmmr.merkle_proof(pos) + PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos).merkle_proof(pos) } /// Compact the MMR data files and flush the rm logs diff --git a/core/src/core/pmmr/pmmr.rs b/core/src/core/pmmr/pmmr.rs index 439697dea..06be965b7 100644 --- a/core/src/core/pmmr/pmmr.rs +++ b/core/src/core/pmmr/pmmr.rs @@ -282,53 +282,6 @@ where } } - /// Helper function to get the last N nodes inserted, i.e. the last - /// n nodes along the bottom of the tree. - /// May return less than n items if the MMR has been pruned/compacted. - pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> { - let mut return_vec = vec![]; - let mut last_leaf = self.last_pos; - for _ in 0..n as u64 { - if last_leaf == 0 { - break; - } - last_leaf = bintree_rightmost(last_leaf); - - if let Some(hash) = self.backend.get_hash(last_leaf) { - if let Some(data) = self.backend.get_data(last_leaf) { - return_vec.push((hash, data)); - } - } - last_leaf -= 1; - } - return_vec - } - - /// Helper function which returns un-pruned nodes from the insertion index - /// forward - /// returns last insertion index returned along with data - pub fn elements_from_insertion_index( - &self, - mut index: u64, - max_count: u64, - ) -> (u64, Vec) { - let mut return_vec = vec![]; - if index == 0 { - index = 1; - } - let mut return_index = index; - let mut pmmr_index = insertion_to_pmmr_index(index); - while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos { - if let Some(t) = self.get_data(pmmr_index) { - return_vec.push(t); - return_index = index; - } - index += 1; - pmmr_index = insertion_to_pmmr_index(index); - } - (return_index, return_vec) - } - /// Walks all unpruned nodes in the MMR and revalidate all parent hashes pub fn validate(&self) -> Result<(), String> { // iterate on all parent nodes diff --git a/core/src/core/pmmr/readonly_pmmr.rs b/core/src/core/pmmr/readonly_pmmr.rs index e4557b3d5..a0f67433b 100644 --- a/core/src/core/pmmr/readonly_pmmr.rs +++ b/core/src/core/pmmr/readonly_pmmr.rs @@ -16,9 +16,10 @@ use std::marker; -use core::hash::Hash; +use core::hash::{Hash, ZERO_HASH}; +use core::pmmr::pmmr::{bintree_rightmost, insertion_to_pmmr_index, peaks}; use core::pmmr::{is_leaf, Backend}; -use ser::PMMRable; +use ser::{PMMRIndexHashable, PMMRable}; /// Readonly view of a PMMR. pub struct ReadonlyPMMR<'a, T, B> @@ -84,4 +85,90 @@ where self.backend.get_from_file(pos) } } + + /// Is the MMR empty? + pub fn is_empty(&self) -> bool { + self.last_pos == 0 + } + + /// Computes the root of the MMR. Find all the peaks in the current + /// tree and "bags" them to get a single peak. + pub fn root(&self) -> Hash { + if self.is_empty() { + return ZERO_HASH; + } + let mut res = None; + for peak in self.peaks().iter().rev() { + res = match res { + None => Some(*peak), + Some(rhash) => Some((*peak, rhash).hash_with_index(self.unpruned_size())), + } + } + res.expect("no root, invalid tree") + } + + /// Returns a vec of the peaks of this MMR. + pub fn peaks(&self) -> Vec { + let peaks_pos = peaks(self.last_pos); + peaks_pos + .into_iter() + .filter_map(|pi| { + // here we want to get from underlying hash file + // as the pos *may* have been "removed" + self.backend.get_from_file(pi) + }).collect() + } + + /// Total size of the tree, including intermediary nodes and ignoring any + /// pruning. + pub fn unpruned_size(&self) -> u64 { + self.last_pos + } + + /// Helper function which returns un-pruned nodes from the insertion index + /// forward + /// returns last insertion index returned along with data + pub fn elements_from_insertion_index( + &self, + mut index: u64, + max_count: u64, + ) -> (u64, Vec) { + let mut return_vec = vec![]; + if index == 0 { + index = 1; + } + let mut return_index = index; + let mut pmmr_index = insertion_to_pmmr_index(index); + while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos { + if let Some(t) = self.get_data(pmmr_index) { + return_vec.push(t); + return_index = index; + } + index += 1; + pmmr_index = insertion_to_pmmr_index(index); + } + (return_index, return_vec) + } + + /// Helper function to get the last N nodes inserted, i.e. the last + /// n nodes along the bottom of the tree. + /// May return less than n items if the MMR has been pruned/compacted. + pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> { + let mut return_vec = vec![]; + let mut last_leaf = self.last_pos; + for _ in 0..n as u64 { + if last_leaf == 0 { + break; + } + last_leaf = bintree_rightmost(last_leaf); + + if let Some(hash) = self.backend.get_hash(last_leaf) { + if let Some(data) = self.backend.get_data(last_leaf) { + return_vec.push((hash, data)); + } + } + last_leaf -= 1; + } + return_vec + } } diff --git a/core/src/core/pmmr/rewindable_pmmr.rs b/core/src/core/pmmr/rewindable_pmmr.rs index ca36101b1..c04e97423 100644 --- a/core/src/core/pmmr/rewindable_pmmr.rs +++ b/core/src/core/pmmr/rewindable_pmmr.rs @@ -17,7 +17,7 @@ use std::marker; -use core::hash::Hash; +use core::hash::{Hash, ZERO_HASH}; use core::pmmr::{bintree_postorder_height, is_leaf, peaks, Backend}; use ser::{PMMRIndexHashable, PMMRable}; @@ -88,9 +88,17 @@ where } } + /// Is the MMR empty? + pub fn is_empty(&self) -> bool { + self.last_pos == 0 + } + /// Computes the root of the MMR. Find all the peaks in the current /// tree and "bags" them to get a single peak. pub fn root(&self) -> Hash { + if self.is_empty() { + return ZERO_HASH; + } let mut res = None; for peak in self.peaks().iter().rev() { res = match res { diff --git a/core/tests/pmmr.rs b/core/tests/pmmr.rs index 431675dd8..282285985 100644 --- a/core/tests/pmmr.rs +++ b/core/tests/pmmr.rs @@ -378,26 +378,26 @@ fn pmmr_get_last_n_insertions() { let mut pmmr = PMMR::new(&mut ba); // test when empty - let res = pmmr.get_last_n_insertions(19); + let res = pmmr.readonly_pmmr().get_last_n_insertions(19); assert!(res.len() == 0); pmmr.push(&elems[0]).unwrap(); - let res = pmmr.get_last_n_insertions(19); + let res = pmmr.readonly_pmmr().get_last_n_insertions(19); assert!(res.len() == 1); pmmr.push(&elems[1]).unwrap(); - let res = pmmr.get_last_n_insertions(12); + let res = pmmr.readonly_pmmr().get_last_n_insertions(12); assert!(res.len() == 2); pmmr.push(&elems[2]).unwrap(); - let res = pmmr.get_last_n_insertions(2); + let res = pmmr.readonly_pmmr().get_last_n_insertions(2); assert!(res.len() == 2); pmmr.push(&elems[3]).unwrap(); - let res = pmmr.get_last_n_insertions(19); + let res = pmmr.readonly_pmmr().get_last_n_insertions(19); assert!(res.len() == 4); pmmr.push(&elems[5]).unwrap(); @@ -405,7 +405,7 @@ fn pmmr_get_last_n_insertions() { pmmr.push(&elems[7]).unwrap(); pmmr.push(&elems[8]).unwrap(); - let res = pmmr.get_last_n_insertions(7); + let res = pmmr.readonly_pmmr().get_last_n_insertions(7); assert!(res.len() == 7); } @@ -526,21 +526,23 @@ fn check_elements_from_insertion_index() { pmmr.push(&TestElem([0, 0, 0, x])).unwrap(); } // Normal case - let res = pmmr.elements_from_insertion_index(1, 100); + let res = pmmr.readonly_pmmr().elements_from_insertion_index(1, 100); assert_eq!(res.0, 100); assert_eq!(res.1.len(), 100); assert_eq!(res.1[0].0[3], 1); assert_eq!(res.1[99].0[3], 100); // middle of pack - let res = pmmr.elements_from_insertion_index(351, 70); + let res = pmmr.readonly_pmmr().elements_from_insertion_index(351, 70); assert_eq!(res.0, 420); assert_eq!(res.1.len(), 70); assert_eq!(res.1[0].0[3], 351); assert_eq!(res.1[69].0[3], 420); // past the end - let res = pmmr.elements_from_insertion_index(650, 1000); + let res = pmmr + .readonly_pmmr() + .elements_from_insertion_index(650, 1000); assert_eq!(res.0, 999); assert_eq!(res.1.len(), 350); assert_eq!(res.1[0].0[3], 650); @@ -552,7 +554,9 @@ fn check_elements_from_insertion_index() { pmmr.prune(pmmr::insertion_to_pmmr_index(800)).unwrap(); pmmr.prune(pmmr::insertion_to_pmmr_index(900)).unwrap(); pmmr.prune(pmmr::insertion_to_pmmr_index(998)).unwrap(); - let res = pmmr.elements_from_insertion_index(650, 1000); + let res = pmmr + .readonly_pmmr() + .elements_from_insertion_index(650, 1000); assert_eq!(res.0, 999); assert_eq!(res.1.len(), 345); assert_eq!(res.1[0].0[3], 652);