Core/grin
0
0
Fork 0
mirror of https://github.com/mimblewimble/grin.git synced 2025-01-21 03:21:08 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Readonly pmmr cleanup (#2083)

Browse source 
* avoid locking txhashset by using a readonly PMMR

* rustfmt
This commit is contained in:
Antioch Peverell 2018-12-05 14:52:53 +00:00 committed by GitHub
parent 8d8f533b8e
commit bf815aa5cd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 150 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
chain/src/chain.rs
View file

@ -583,7 +583,7 @@ impl Chain {
Ok(())
}
/// Return a pre-built Merkle proof for the given commitment from the store.
/// Return a Merkle proof for the given commitment from the store.
pub fn get_merkle_proof(
&self,
output: &OutputIdentifier,
@ -606,10 +606,9 @@ impl Chain {
txhashset.merkle_proof(commit)
}
/// Returns current txhashset roots
/// Returns current txhashset roots.
pub fn get_txhashset_roots(&self) -> TxHashSetRoots {
let mut txhashset = self.txhashset.write();
txhashset.roots()
self.txhashset.read().roots()
}
/// Provides a reading view into the current txhashset state as well as
@ -967,20 +966,17 @@ impl Chain {
/// returns the last n nodes inserted into the output sum tree
pub fn get_last_n_output(&self, distance: u64) -> Vec<(Hash, OutputIdentifier)> {
let mut txhashset = self.txhashset.write();
txhashset.last_n_output(distance)
self.txhashset.read().last_n_output(distance)
}
/// as above, for rangeproofs
pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, RangeProof)> {
let mut txhashset = self.txhashset.write();
txhashset.last_n_rangeproof(distance)
self.txhashset.read().last_n_rangeproof(distance)
}
/// as above, for kernels
pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
let mut txhashset = self.txhashset.write();
txhashset.last_n_kernel(distance)
self.txhashset.read().last_n_kernel(distance)
}
/// outputs by insertion index
@ -989,7 +985,7 @@ impl Chain {
start_index: u64,
max: u64,
) -> Result<(u64, u64, Vec<Output>), Error> {
let mut txhashset = self.txhashset.write();
let txhashset = self.txhashset.read();
let max_index = txhashset.highest_output_insertion_index();
let outputs = txhashset.outputs_by_insertion_index(start_index, max);
let rangeproofs = txhashset.rangeproofs_by_insertion_index(start_index, max);

70
chain/src/txhashset/txhashset.rs
View file

@ -163,7 +163,7 @@ impl TxHashSet {
pub fn is_unspent(&self, output_id: &OutputIdentifier) -> Result<(Hash, u64), Error> {
match self.commit_index.get_output_pos(&output_id.commit) {
Ok(pos) => {
let output_pmmr: ReadonlyPMMR<Output, _> =
let output_pmmr =
ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
if let Some(hash) = output_pmmr.get_hash(pos) {
if hash == output_id.hash_with_index(pos - 1) {
@ -184,34 +184,30 @@ impl TxHashSet {
/// nodes at level 0
/// TODO: These need to return the actual data from the flat-files instead
/// of hashes now
pub fn last_n_output(&mut self, distance: u64) -> Vec<(Hash, OutputIdentifier)> {
let output_pmmr: PMMR<Output, _> =
PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
output_pmmr.get_last_n_insertions(distance)
pub fn last_n_output(&self, distance: u64) -> Vec<(Hash, OutputIdentifier)> {
ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos)
.get_last_n_insertions(distance)
}
/// as above, for range proofs
pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<(Hash, RangeProof)> {
let rproof_pmmr: PMMR<RangeProof, _> =
PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
rproof_pmmr.get_last_n_insertions(distance)
pub fn last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, RangeProof)> {
ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos)
.get_last_n_insertions(distance)
}
/// as above, for kernels
pub fn last_n_kernel(&mut self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
let kernel_pmmr: PMMR<TxKernel, _> =
PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
kernel_pmmr.get_last_n_insertions(distance)
pub fn last_n_kernel(&self, distance: u64) -> Vec<(Hash, TxKernelEntry)> {
ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos)
.get_last_n_insertions(distance)
}
/// Get the header at the specified height based on the current state of the txhashset.
/// Derives the MMR pos from the height (insertion index) and retrieves the header hash.
/// Looks the header up in the db by hash.
pub fn get_header_by_height(&mut self, height: u64) -> Result<BlockHeader, Error> {
pub fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> {
let pos = pmmr::insertion_to_pmmr_index(height + 1);
let header_pmmr: PMMR<BlockHeader, _> =
PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
let header_pmmr =
ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
if let Some(hash) = header_pmmr.get_data(pos) {
let header = self.commit_index.get_block_header(&hash)?;
Ok(header)
@ -223,41 +219,39 @@ impl TxHashSet {
/// returns outputs from the given insertion (leaf) index up to the
/// specified limit. Also returns the last index actually populated
pub fn outputs_by_insertion_index(
&mut self,
&self,
start_index: u64,
max_count: u64,
) -> (u64, Vec<OutputIdentifier>) {
let output_pmmr: PMMR<Output, _> =
PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
output_pmmr.elements_from_insertion_index(start_index, max_count)
ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos)
.elements_from_insertion_index(start_index, max_count)
}
/// highest output insertion index available
pub fn highest_output_insertion_index(&mut self) -> u64 {
pub fn highest_output_insertion_index(&self) -> u64 {
pmmr::n_leaves(self.output_pmmr_h.last_pos)
}
/// As above, for rangeproofs
pub fn rangeproofs_by_insertion_index(
&mut self,
&self,
start_index: u64,
max_count: u64,
) -> (u64, Vec<RangeProof>) {
let rproof_pmmr: PMMR<RangeProof, _> =
PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
rproof_pmmr.elements_from_insertion_index(start_index, max_count)
ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos)
.elements_from_insertion_index(start_index, max_count)
}
/// Get MMR roots.
pub fn roots(&mut self) -> TxHashSetRoots {
let header_pmmr: PMMR<BlockHeader, _> =
PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
let output_pmmr: PMMR<Output, _> =
PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
let rproof_pmmr: PMMR<RangeProof, _> =
PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
let kernel_pmmr: PMMR<TxKernel, _> =
PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
pub fn roots(&self) -> TxHashSetRoots {
let header_pmmr =
ReadonlyPMMR::at(&self.header_pmmr_h.backend, self.header_pmmr_h.last_pos);
let output_pmmr =
ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
let rproof_pmmr =
ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
let kernel_pmmr =
ReadonlyPMMR::at(&self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
TxHashSetRoots {
header_root: header_pmmr.root(),
@ -267,12 +261,10 @@ impl TxHashSet {
}
}
/// build a new merkle proof for the given position
/// build a new merkle proof for the given position.
pub fn merkle_proof(&mut self, commit: Commitment) -> Result<MerkleProof, String> {
let pos = self.commit_index.get_output_pos(&commit).unwrap();
let output_pmmr: PMMR<Output, _> =
PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
output_pmmr.merkle_proof(pos)
PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos).merkle_proof(pos)
}
/// Compact the MMR data files and flush the rm logs

47
core/src/core/pmmr/pmmr.rs
View file

@ -282,53 +282,6 @@ where
}
}
/// Helper function to get the last N nodes inserted, i.e. the last
/// n nodes along the bottom of the tree.
/// May return less than n items if the MMR has been pruned/compacted.
pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> {
let mut return_vec = vec![];
let mut last_leaf = self.last_pos;
for _ in 0..n as u64 {
if last_leaf == 0 {
break;
}
last_leaf = bintree_rightmost(last_leaf);
if let Some(hash) = self.backend.get_hash(last_leaf) {
if let Some(data) = self.backend.get_data(last_leaf) {
return_vec.push((hash, data));
}
}
last_leaf -= 1;
}
return_vec
}
/// Helper function which returns un-pruned nodes from the insertion index
/// forward
/// returns last insertion index returned along with data
pub fn elements_from_insertion_index(
&self,
mut index: u64,
max_count: u64,
) -> (u64, Vec<T::E>) {
let mut return_vec = vec![];
if index == 0 {
index = 1;
}
let mut return_index = index;
let mut pmmr_index = insertion_to_pmmr_index(index);
while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos {
if let Some(t) = self.get_data(pmmr_index) {
return_vec.push(t);
return_index = index;
}
index += 1;
pmmr_index = insertion_to_pmmr_index(index);
}
(return_index, return_vec)
}
/// Walks all unpruned nodes in the MMR and revalidate all parent hashes
pub fn validate(&self) -> Result<(), String> {
// iterate on all parent nodes

91
core/src/core/pmmr/readonly_pmmr.rs
View file

@ -16,9 +16,10 @@
use std::marker;
use core::hash::Hash;
use core::hash::{Hash, ZERO_HASH};
use core::pmmr::pmmr::{bintree_rightmost, insertion_to_pmmr_index, peaks};
use core::pmmr::{is_leaf, Backend};
use ser::PMMRable;
use ser::{PMMRIndexHashable, PMMRable};
/// Readonly view of a PMMR.
pub struct ReadonlyPMMR<'a, T, B>
@ -84,4 +85,90 @@ where
self.backend.get_from_file(pos)
}
}
/// Is the MMR empty?
pub fn is_empty(&self) -> bool {
self.last_pos == 0
}
/// Computes the root of the MMR. Find all the peaks in the current
/// tree and "bags" them to get a single peak.
pub fn root(&self) -> Hash {
if self.is_empty() {
return ZERO_HASH;
}
let mut res = None;
for peak in self.peaks().iter().rev() {
res = match res {
None => Some(*peak),
Some(rhash) => Some((*peak, rhash).hash_with_index(self.unpruned_size())),
}
}
res.expect("no root, invalid tree")
}
/// Returns a vec of the peaks of this MMR.
pub fn peaks(&self) -> Vec<Hash> {
let peaks_pos = peaks(self.last_pos);
peaks_pos
.into_iter()
.filter_map(|pi| {
// here we want to get from underlying hash file
// as the pos *may* have been "removed"
self.backend.get_from_file(pi)
}).collect()
}
/// Total size of the tree, including intermediary nodes and ignoring any
/// pruning.
pub fn unpruned_size(&self) -> u64 {
self.last_pos
}
/// Helper function which returns un-pruned nodes from the insertion index
/// forward
/// returns last insertion index returned along with data
pub fn elements_from_insertion_index(
&self,
mut index: u64,
max_count: u64,
) -> (u64, Vec<T::E>) {
let mut return_vec = vec![];
if index == 0 {
index = 1;
}
let mut return_index = index;
let mut pmmr_index = insertion_to_pmmr_index(index);
while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos {
if let Some(t) = self.get_data(pmmr_index) {
return_vec.push(t);
return_index = index;
}
index += 1;
pmmr_index = insertion_to_pmmr_index(index);
}
(return_index, return_vec)
}
/// Helper function to get the last N nodes inserted, i.e. the last
/// n nodes along the bottom of the tree.
/// May return less than n items if the MMR has been pruned/compacted.
pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> {
let mut return_vec = vec![];
let mut last_leaf = self.last_pos;
for _ in 0..n as u64 {
if last_leaf == 0 {
break;
}
last_leaf = bintree_rightmost(last_leaf);
if let Some(hash) = self.backend.get_hash(last_leaf) {
if let Some(data) = self.backend.get_data(last_leaf) {
return_vec.push((hash, data));
}
}
last_leaf -= 1;
}
return_vec
}
}

10
core/src/core/pmmr/rewindable_pmmr.rs
View file

@ -17,7 +17,7 @@
use std::marker;
use core::hash::Hash;
use core::hash::{Hash, ZERO_HASH};
use core::pmmr::{bintree_postorder_height, is_leaf, peaks, Backend};
use ser::{PMMRIndexHashable, PMMRable};
@ -88,9 +88,17 @@ where
}
}
/// Is the MMR empty?
pub fn is_empty(&self) -> bool {
self.last_pos == 0
}
/// Computes the root of the MMR. Find all the peaks in the current
/// tree and "bags" them to get a single peak.
pub fn root(&self) -> Hash {
if self.is_empty() {
return ZERO_HASH;
}
let mut res = None;
for peak in self.peaks().iter().rev() {
res = match res {

24
core/tests/pmmr.rs
View file

@ -378,26 +378,26 @@ fn pmmr_get_last_n_insertions() {
let mut pmmr = PMMR::new(&mut ba);
// test when empty
let res = pmmr.get_last_n_insertions(19);
let res = pmmr.readonly_pmmr().get_last_n_insertions(19);
assert!(res.len() == 0);
pmmr.push(&elems[0]).unwrap();
let res = pmmr.get_last_n_insertions(19);
let res = pmmr.readonly_pmmr().get_last_n_insertions(19);
assert!(res.len() == 1);
pmmr.push(&elems[1]).unwrap();
let res = pmmr.get_last_n_insertions(12);
let res = pmmr.readonly_pmmr().get_last_n_insertions(12);
assert!(res.len() == 2);
pmmr.push(&elems[2]).unwrap();
let res = pmmr.get_last_n_insertions(2);
let res = pmmr.readonly_pmmr().get_last_n_insertions(2);
assert!(res.len() == 2);
pmmr.push(&elems[3]).unwrap();
let res = pmmr.get_last_n_insertions(19);
let res = pmmr.readonly_pmmr().get_last_n_insertions(19);
assert!(res.len() == 4);
pmmr.push(&elems[5]).unwrap();
@ -405,7 +405,7 @@ fn pmmr_get_last_n_insertions() {
pmmr.push(&elems[7]).unwrap();
pmmr.push(&elems[8]).unwrap();
let res = pmmr.get_last_n_insertions(7);
let res = pmmr.readonly_pmmr().get_last_n_insertions(7);
assert!(res.len() == 7);
}
@ -526,21 +526,23 @@ fn check_elements_from_insertion_index() {
pmmr.push(&TestElem([0, 0, 0, x])).unwrap();
}
// Normal case
let res = pmmr.elements_from_insertion_index(1, 100);
let res = pmmr.readonly_pmmr().elements_from_insertion_index(1, 100);
assert_eq!(res.0, 100);
assert_eq!(res.1.len(), 100);
assert_eq!(res.1[0].0[3], 1);
assert_eq!(res.1[99].0[3], 100);
// middle of pack
let res = pmmr.elements_from_insertion_index(351, 70);
let res = pmmr.readonly_pmmr().elements_from_insertion_index(351, 70);
assert_eq!(res.0, 420);
assert_eq!(res.1.len(), 70);
assert_eq!(res.1[0].0[3], 351);
assert_eq!(res.1[69].0[3], 420);
// past the end
let res = pmmr.elements_from_insertion_index(650, 1000);
let res = pmmr
.readonly_pmmr()
.elements_from_insertion_index(650, 1000);
assert_eq!(res.0, 999);
assert_eq!(res.1.len(), 350);
assert_eq!(res.1[0].0[3], 650);
@ -552,7 +554,9 @@ fn check_elements_from_insertion_index() {
pmmr.prune(pmmr::insertion_to_pmmr_index(800)).unwrap();
pmmr.prune(pmmr::insertion_to_pmmr_index(900)).unwrap();
pmmr.prune(pmmr::insertion_to_pmmr_index(998)).unwrap();
let res = pmmr.elements_from_insertion_index(650, 1000);
let res = pmmr
.readonly_pmmr()
.elements_from_insertion_index(650, 1000);
assert_eq!(res.0, 999);
assert_eq!(res.1.len(), 345);
assert_eq!(res.1[0].0[3], 652);