Readonly pmmr cleanup (#2083)

* avoid locking txhashset by using a readonly PMMR

* rustfmt

chain/src/chain.rs

@@ -583,7 +583,7 @@ impl Chain {
 		Ok(())
 	}
 
-	/// Return a pre-built Merkle proof for the given commitment from the store.
+	/// Return a Merkle proof for the given commitment from the store.
 	pub fn get_merkle_proof(
 		&self,
 		output: &OutputIdentifier,
@@ -606,10 +606,9 @@ impl Chain {
 		txhashset.merkle_proof(commit)
 	}
 
-	/// Returns current txhashset roots
+	/// Returns current txhashset roots.
 	pub fn get_txhashset_roots(&self) -> TxHashSetRoots {
-		let mut txhashset = self.txhashset.write();
+		self.txhashset.read().roots()
-		txhashset.roots()
 	}
 
 	/// Provides a reading view into the current txhashset state as well as
@@ -967,20 +966,17 @@ impl Chain {
 
 	/// returns the last n nodes inserted into the output sum tree
 	pub fn get_last_n_output(&self, distance: u64) -> Vec<(Hash, OutputIdentifier)> {
-		let mut txhashset = self.txhashset.write();
+		self.txhashset.read().last_n_output(distance)
-		txhashset.last_n_output(distance)
 	}
 
 	/// as above, for rangeproofs
 	pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, RangeProof)> {
-		let mut txhashset = self.txhashset.write();
+		self.txhashset.read().last_n_rangeproof(distance)
-		txhashset.last_n_rangeproof(distance)
 	}
 
 	/// as above, for kernels
 	pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
-		let mut txhashset = self.txhashset.write();
+		self.txhashset.read().last_n_kernel(distance)
-		txhashset.last_n_kernel(distance)
 	}
 
 	/// outputs by insertion index
@@ -989,7 +985,7 @@ impl Chain {
 		start_index: u64,
 		max: u64,
 	) -> Result<(u64, u64, Vec<Output>), Error> {
-		let mut txhashset = self.txhashset.write();
+		let txhashset = self.txhashset.read();
 		let max_index = txhashset.highest_output_insertion_index();
 		let outputs = txhashset.outputs_by_insertion_index(start_index, max);
 		let rangeproofs = txhashset.rangeproofs_by_insertion_index(start_index, max);

chain/src/txhashset/txhashset.rs

@@ -163,7 +163,7 @@ impl TxHashSet {
 	pub fn is_unspent(&self, output_id: &OutputIdentifier) -> Result<(Hash, u64), Error> {
 		match self.commit_index.get_output_pos(&output_id.commit) {
 			Ok(pos) => {
-				let output_pmmr: ReadonlyPMMR<Output, _> =
+				let output_pmmr =
 					ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
 				if let Some(hash) = output_pmmr.get_hash(pos) {
 					if hash == output_id.hash_with_index(pos - 1) {
@@ -184,34 +184,30 @@ impl TxHashSet {
 	/// nodes at level 0
 	/// TODO: These need to return the actual data from the flat-files instead
 	/// of hashes now
-	pub fn last_n_output(&mut self, distance: u64) -> Vec<(Hash, OutputIdentifier)> {
+	pub fn last_n_output(&self, distance: u64) -> Vec<(Hash, OutputIdentifier)> {
-		let output_pmmr: PMMR<Output, _> =
+		ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos)
-			PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
+			.get_last_n_insertions(distance)
-		output_pmmr.get_last_n_insertions(distance)
 	}
 
 	/// as above, for range proofs
-	pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<(Hash, RangeProof)> {
+	pub fn last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, RangeProof)> {
-		let rproof_pmmr: PMMR<RangeProof, _> =
+		ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos)
-			PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+			.get_last_n_insertions(distance)
-		rproof_pmmr.get_last_n_insertions(distance)
 	}
 
 	/// as above, for kernels
-	pub fn last_n_kernel(&mut self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
+	pub fn last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
-		let kernel_pmmr: PMMR<TxKernel, _> =
+		ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos)
-			PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+			.get_last_n_insertions(distance)
-		kernel_pmmr.get_last_n_insertions(distance)
 	}
 
 	/// Get the header at the specified height based on the current state of the txhashset.
 	/// Derives the MMR pos from the height (insertion index) and retrieves the header hash.
 	/// Looks the header up in the db by hash.
-	pub fn get_header_by_height(&mut self, height: u64) -> Result<BlockHeader, Error> {
+	pub fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> {
 		let pos = pmmr::insertion_to_pmmr_index(height + 1);
-
+		let header_pmmr =
-		let header_pmmr: PMMR<BlockHeader, _> =
+			ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
-			PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
 		if let Some(hash) = header_pmmr.get_data(pos) {
 			let header = self.commit_index.get_block_header(&hash)?;
 			Ok(header)
@@ -223,41 +219,39 @@ impl TxHashSet {
 	/// returns outputs from the given insertion (leaf) index up to the
 	/// specified limit. Also returns the last index actually populated
 	pub fn outputs_by_insertion_index(
-		&mut self,
+		&self,
 		start_index: u64,
 		max_count: u64,
 	) -> (u64, Vec<OutputIdentifier>) {
-		let output_pmmr: PMMR<Output, _> =
+		ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos)
-			PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
+			.elements_from_insertion_index(start_index, max_count)
-		output_pmmr.elements_from_insertion_index(start_index, max_count)
 	}
 
 	/// highest output insertion index available
-	pub fn highest_output_insertion_index(&mut self) -> u64 {
+	pub fn highest_output_insertion_index(&self) -> u64 {
 		pmmr::n_leaves(self.output_pmmr_h.last_pos)
 	}
 
 	/// As above, for rangeproofs
 	pub fn rangeproofs_by_insertion_index(
-		&mut self,
+		&self,
 		start_index: u64,
 		max_count: u64,
 	) -> (u64, Vec<RangeProof>) {
-		let rproof_pmmr: PMMR<RangeProof, _> =
+		ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos)
-			PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+			.elements_from_insertion_index(start_index, max_count)
-		rproof_pmmr.elements_from_insertion_index(start_index, max_count)
 	}
 
 	/// Get MMR roots.
-	pub fn roots(&mut self) -> TxHashSetRoots {
+	pub fn roots(&self) -> TxHashSetRoots {
-		let header_pmmr: PMMR<BlockHeader, _> =
+		let header_pmmr =
-			PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
+			ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
-		let output_pmmr: PMMR<Output, _> =
+		let output_pmmr =
-			PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
+			ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
-		let rproof_pmmr: PMMR<RangeProof, _> =
+		let rproof_pmmr =
-			PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+			ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
-		let kernel_pmmr: PMMR<TxKernel, _> =
+		let kernel_pmmr =
-			PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+			ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
 
 		TxHashSetRoots {
 			header_root: header_pmmr.root(),
@@ -267,12 +261,10 @@ impl TxHashSet {
 		}
 	}
 
-	/// build a new merkle proof for the given position
+	/// build a new merkle proof for the given position.
 	pub fn merkle_proof(&mut self, commit: Commitment) -> Result<MerkleProof, String> {
 		let pos = self.commit_index.get_output_pos(&commit).unwrap();
-		let output_pmmr: PMMR<Output, _> =
+		PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos).merkle_proof(pos)
-			PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
-		output_pmmr.merkle_proof(pos)
 	}
 
 	/// Compact the MMR data files and flush the rm logs

core/src/core/pmmr/pmmr.rs

@@ -282,53 +282,6 @@ where
 		}
 	}
 
-	/// Helper function to get the last N nodes inserted, i.e. the last
-	/// n nodes along the bottom of the tree.
-	/// May return less than n items if the MMR has been pruned/compacted.
-	pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> {
-		let mut return_vec = vec![];
-		let mut last_leaf = self.last_pos;
-		for _ in 0..n as u64 {
-			if last_leaf == 0 {
-				break;
-			}
-			last_leaf = bintree_rightmost(last_leaf);
-
-			if let Some(hash) = self.backend.get_hash(last_leaf) {
-				if let Some(data) = self.backend.get_data(last_leaf) {
-					return_vec.push((hash, data));
-				}
-			}
-			last_leaf -= 1;
-		}
-		return_vec
-	}
-
-	/// Helper function which returns un-pruned nodes from the insertion index
-	/// forward
-	/// returns last insertion index returned along with data
-	pub fn elements_from_insertion_index(
-		&self,
-		mut index: u64,
-		max_count: u64,
-	) -> (u64, Vec<T::E>) {
-		let mut return_vec = vec![];
-		if index == 0 {
-			index = 1;
-		}
-		let mut return_index = index;
-		let mut pmmr_index = insertion_to_pmmr_index(index);
-		while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos {
-			if let Some(t) = self.get_data(pmmr_index) {
-				return_vec.push(t);
-				return_index = index;
-			}
-			index += 1;
-			pmmr_index = insertion_to_pmmr_index(index);
-		}
-		(return_index, return_vec)
-	}
-
 	/// Walks all unpruned nodes in the MMR and revalidate all parent hashes
 	pub fn validate(&self) -> Result<(), String> {
 		// iterate on all parent nodes

core/src/core/pmmr/readonly_pmmr.rs

@@ -16,9 +16,10 @@
 
 use std::marker;
 
-use core::hash::Hash;
+use core::hash::{Hash, ZERO_HASH};
+use core::pmmr::pmmr::{bintree_rightmost, insertion_to_pmmr_index, peaks};
 use core::pmmr::{is_leaf, Backend};
-use ser::PMMRable;
+use ser::{PMMRIndexHashable, PMMRable};
 
 /// Readonly view of a PMMR.
 pub struct ReadonlyPMMR<'a, T, B>
@@ -84,4 +85,90 @@ where
 			self.backend.get_from_file(pos)
 		}
 	}
+
+	/// Is the MMR empty?
+	pub fn is_empty(&self) -> bool {
+		self.last_pos == 0
+	}
+
+	/// Computes the root of the MMR. Find all the peaks in the current
+	/// tree and "bags" them to get a single peak.
+	pub fn root(&self) -> Hash {
+		if self.is_empty() {
+			return ZERO_HASH;
+		}
+		let mut res = None;
+		for peak in self.peaks().iter().rev() {
+			res = match res {
+				None => Some(*peak),
+				Some(rhash) => Some((*peak, rhash).hash_with_index(self.unpruned_size())),
+			}
+		}
+		res.expect("no root, invalid tree")
+	}
+
+	/// Returns a vec of the peaks of this MMR.
+	pub fn peaks(&self) -> Vec<Hash> {
+		let peaks_pos = peaks(self.last_pos);
+		peaks_pos
+			.into_iter()
+			.filter_map(|pi| {
+				// here we want to get from underlying hash file
+				// as the pos *may* have been "removed"
+				self.backend.get_from_file(pi)
+			}).collect()
+	}
+
+	/// Total size of the tree, including intermediary nodes and ignoring any
+	/// pruning.
+	pub fn unpruned_size(&self) -> u64 {
+		self.last_pos
+	}
+
+	/// Helper function which returns un-pruned nodes from the insertion index
+	/// forward
+	/// returns last insertion index returned along with data
+	pub fn elements_from_insertion_index(
+		&self,
+		mut index: u64,
+		max_count: u64,
+	) -> (u64, Vec<T::E>) {
+		let mut return_vec = vec![];
+		if index == 0 {
+			index = 1;
+		}
+		let mut return_index = index;
+		let mut pmmr_index = insertion_to_pmmr_index(index);
+		while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos {
+			if let Some(t) = self.get_data(pmmr_index) {
+				return_vec.push(t);
+				return_index = index;
+			}
+			index += 1;
+			pmmr_index = insertion_to_pmmr_index(index);
+		}
+		(return_index, return_vec)
+	}
+
+	/// Helper function to get the last N nodes inserted, i.e. the last
+	/// n nodes along the bottom of the tree.
+	/// May return less than n items if the MMR has been pruned/compacted.
+	pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> {
+		let mut return_vec = vec![];
+		let mut last_leaf = self.last_pos;
+		for _ in 0..n as u64 {
+			if last_leaf == 0 {
+				break;
+			}
+			last_leaf = bintree_rightmost(last_leaf);
+
+			if let Some(hash) = self.backend.get_hash(last_leaf) {
+				if let Some(data) = self.backend.get_data(last_leaf) {
+					return_vec.push((hash, data));
+				}
+			}
+			last_leaf -= 1;
+		}
+		return_vec
+	}
 }

core/src/core/pmmr/rewindable_pmmr.rs

@@ -17,7 +17,7 @@
 
 use std::marker;
 
-use core::hash::Hash;
+use core::hash::{Hash, ZERO_HASH};
 use core::pmmr::{bintree_postorder_height, is_leaf, peaks, Backend};
 use ser::{PMMRIndexHashable, PMMRable};
 
@@ -88,9 +88,17 @@ where
 		}
 	}
 
+	/// Is the MMR empty?
+	pub fn is_empty(&self) -> bool {
+		self.last_pos == 0
+	}
+
 	/// Computes the root of the MMR. Find all the peaks in the current
 	/// tree and "bags" them to get a single peak.
 	pub fn root(&self) -> Hash {
+		if self.is_empty() {
+			return ZERO_HASH;
+		}
 		let mut res = None;
 		for peak in self.peaks().iter().rev() {
 			res = match res {

core/tests/pmmr.rs

@@ -378,26 +378,26 @@ fn pmmr_get_last_n_insertions() {
 	let mut pmmr = PMMR::new(&mut ba);
 
 	// test when empty
-	let res = pmmr.get_last_n_insertions(19);
+	let res = pmmr.readonly_pmmr().get_last_n_insertions(19);
 	assert!(res.len() == 0);
 
 	pmmr.push(&elems[0]).unwrap();
-	let res = pmmr.get_last_n_insertions(19);
+	let res = pmmr.readonly_pmmr().get_last_n_insertions(19);
 	assert!(res.len() == 1);
 
 	pmmr.push(&elems[1]).unwrap();
 
-	let res = pmmr.get_last_n_insertions(12);
+	let res = pmmr.readonly_pmmr().get_last_n_insertions(12);
 	assert!(res.len() == 2);
 
 	pmmr.push(&elems[2]).unwrap();
 
-	let res = pmmr.get_last_n_insertions(2);
+	let res = pmmr.readonly_pmmr().get_last_n_insertions(2);
 	assert!(res.len() == 2);
 
 	pmmr.push(&elems[3]).unwrap();
 
-	let res = pmmr.get_last_n_insertions(19);
+	let res = pmmr.readonly_pmmr().get_last_n_insertions(19);
 	assert!(res.len() == 4);
 
 	pmmr.push(&elems[5]).unwrap();
@@ -405,7 +405,7 @@ fn pmmr_get_last_n_insertions() {
 	pmmr.push(&elems[7]).unwrap();
 	pmmr.push(&elems[8]).unwrap();
 
-	let res = pmmr.get_last_n_insertions(7);
+	let res = pmmr.readonly_pmmr().get_last_n_insertions(7);
 	assert!(res.len() == 7);
 }
 
@@ -526,21 +526,23 @@ fn check_elements_from_insertion_index() {
 		pmmr.push(&TestElem([0, 0, 0, x])).unwrap();
 	}
 	// Normal case
-	let res = pmmr.elements_from_insertion_index(1, 100);
+	let res = pmmr.readonly_pmmr().elements_from_insertion_index(1, 100);
 	assert_eq!(res.0, 100);
 	assert_eq!(res.1.len(), 100);
 	assert_eq!(res.1[0].0[3], 1);
 	assert_eq!(res.1[99].0[3], 100);
 
 	// middle of pack
-	let res = pmmr.elements_from_insertion_index(351, 70);
+	let res = pmmr.readonly_pmmr().elements_from_insertion_index(351, 70);
 	assert_eq!(res.0, 420);
 	assert_eq!(res.1.len(), 70);
 	assert_eq!(res.1[0].0[3], 351);
 	assert_eq!(res.1[69].0[3], 420);
 
 	// past the end
-	let res = pmmr.elements_from_insertion_index(650, 1000);
+	let res = pmmr
+		.readonly_pmmr()
+		.elements_from_insertion_index(650, 1000);
 	assert_eq!(res.0, 999);
 	assert_eq!(res.1.len(), 350);
 	assert_eq!(res.1[0].0[3], 650);
@@ -552,7 +554,9 @@ fn check_elements_from_insertion_index() {
 	pmmr.prune(pmmr::insertion_to_pmmr_index(800)).unwrap();
 	pmmr.prune(pmmr::insertion_to_pmmr_index(900)).unwrap();
 	pmmr.prune(pmmr::insertion_to_pmmr_index(998)).unwrap();
-	let res = pmmr.elements_from_insertion_index(650, 1000);
+	let res = pmmr
+		.readonly_pmmr()
+		.elements_from_insertion_index(650, 1000);
 	assert_eq!(res.0, 999);
 	assert_eq!(res.1.len(), 345);
 	assert_eq!(res.1[0].0[3], 652);