diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 9462e7fd9..6bc5b2997 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -27,6 +27,7 @@ use lru_cache::LruCache; use core::core::hash::{Hash, Hashed, ZERO_HASH}; use core::core::merkle_proof::MerkleProof; +use core::core::pmmr; use core::core::verifier_cache::VerifierCache; use core::core::{ Block, BlockHeader, BlockSums, Output, OutputIdentifier, Transaction, TxKernelEntry, @@ -1196,9 +1197,16 @@ fn setup_head( // If we have no header MMR then rebuild as necessary. // Supports old nodes with no header MMR. txhashset::header_extending(txhashset, &mut batch, |extension| { - if extension.size() == 0 { + let pos = pmmr::insertion_to_pmmr_index(head.height + 1); + let needs_rebuild = match extension.get_header_hash(pos) { + None => true, + Some(hash) => hash != head.last_block_h, + }; + + if needs_rebuild { extension.rebuild(&head, &genesis.header)?; } + Ok(()) })?; diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index f99f63d02..422d444e2 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -28,7 +28,7 @@ use util::secp::pedersen::{Commitment, RangeProof}; use core::core::committed::Committed; use core::core::hash::{Hash, Hashed}; use core::core::merkle_proof::MerkleProof; -use core::core::pmmr::{self, ReadonlyPMMR, RewindablePMMR, DBPMMR, PMMR}; +use core::core::pmmr::{self, ReadonlyPMMR, RewindablePMMR, PMMR}; use core::core::{ Block, BlockHeader, Input, Output, OutputFeatures, OutputIdentifier, TxKernel, TxKernelEntry, }; @@ -37,7 +37,7 @@ use core::ser::{PMMRIndexHashable, PMMRable}; use error::{Error, ErrorKind}; use grin_store; -use grin_store::pmmr::{HashOnlyMMRBackend, PMMRBackend, PMMR_FILES}; +use grin_store::pmmr::{PMMRBackend, PMMR_FILES}; use grin_store::types::prune_noop; use store::{Batch, ChainStore}; use txhashset::{RewindableKernelView, UTXOView}; @@ -56,21 +56,6 @@ const KERNEL_SUBDIR: &'static str = "kernel"; const TXHASHSET_ZIP: &'static str = "txhashset_snapshot"; -struct HashOnlyMMRHandle { - backend: HashOnlyMMRBackend, - last_pos: u64, -} - -impl HashOnlyMMRHandle { - fn new(root_dir: &str, sub_dir: &str, file_name: &str) -> Result { - let path = Path::new(root_dir).join(sub_dir).join(file_name); - fs::create_dir_all(path.clone())?; - let backend = HashOnlyMMRBackend::new(path.to_str().unwrap())?; - let last_pos = backend.unpruned_size(); - Ok(HashOnlyMMRHandle { backend, last_pos }) - } -} - struct PMMRHandle { backend: PMMRBackend, last_pos: u64, @@ -107,8 +92,7 @@ pub struct TxHashSet { /// output, rangeproof and kernel MMRs during an extension or a /// readonly_extension. /// It can also be rewound and applied separately via a header_extension. - /// Note: the header MMR is backed by the database maintains just the hash file. - header_pmmr_h: HashOnlyMMRHandle, + header_pmmr_h: PMMRHandle, /// Header MMR to support exploratory sync_head. /// The header_head and sync_head chains can diverge so we need to maintain @@ -116,8 +100,7 @@ pub struct TxHashSet { /// /// Note: this is rewound and applied separately to the other MMRs /// via a "sync_extension". - /// Note: the sync MMR is backed by the database and maintains just the hash file. - sync_pmmr_h: HashOnlyMMRHandle, + sync_pmmr_h: PMMRHandle, output_pmmr_h: PMMRHandle, rproof_pmmr_h: PMMRHandle, @@ -135,12 +118,20 @@ impl TxHashSet { header: Option<&BlockHeader>, ) -> Result { Ok(TxHashSet { - header_pmmr_h: HashOnlyMMRHandle::new( + header_pmmr_h: PMMRHandle::new( &root_dir, HEADERHASHSET_SUBDIR, HEADER_HEAD_SUBDIR, + false, + None, + )?, + sync_pmmr_h: PMMRHandle::new( + &root_dir, + HEADERHASHSET_SUBDIR, + SYNC_HEAD_SUBDIR, + false, + None, )?, - sync_pmmr_h: HashOnlyMMRHandle::new(&root_dir, HEADERHASHSET_SUBDIR, SYNC_HEAD_SUBDIR)?, output_pmmr_h: PMMRHandle::new( &root_dir, TXHASHSET_SUBDIR, @@ -243,8 +234,8 @@ impl TxHashSet { /// Get MMR roots. pub fn roots(&mut self) -> TxHashSetRoots { - let header_pmmr: DBPMMR = - DBPMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); + let header_pmmr: PMMR = + PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); let output_pmmr: PMMR = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); let rproof_pmmr: PMMR = @@ -491,7 +482,7 @@ where let child_batch = batch.child()?; { trace!("Starting new txhashset sync_head extension."); - let pmmr = DBPMMR::at(&mut trees.sync_pmmr_h.backend, trees.sync_pmmr_h.last_pos); + let pmmr = PMMR::at(&mut trees.sync_pmmr_h.backend, trees.sync_pmmr_h.last_pos); let mut extension = HeaderExtension::new(pmmr, &child_batch, header); res = inner(&mut extension); @@ -550,7 +541,7 @@ where let child_batch = batch.child()?; { trace!("Starting new txhashset header extension."); - let pmmr = DBPMMR::at( + let pmmr = PMMR::at( &mut trees.header_pmmr_h.backend, trees.header_pmmr_h.last_pos, ); @@ -591,7 +582,7 @@ where pub struct HeaderExtension<'a> { header: BlockHeader, - pmmr: DBPMMR<'a, BlockHeader, HashOnlyMMRBackend>, + pmmr: PMMR<'a, BlockHeader, PMMRBackend>, /// Rollback flag. rollback: bool, @@ -604,7 +595,7 @@ pub struct HeaderExtension<'a> { impl<'a> HeaderExtension<'a> { fn new( - pmmr: DBPMMR<'a, BlockHeader, HashOnlyMMRBackend>, + pmmr: PMMR<'a, BlockHeader, PMMRBackend>, batch: &'a Batch, header: BlockHeader, ) -> HeaderExtension<'a> { @@ -616,6 +607,11 @@ impl<'a> HeaderExtension<'a> { } } + /// Get the header hash for the specified pos from the underlying MMR backend. + pub fn get_header_hash(&self, pos: u64) -> Option { + self.pmmr.get_data(pos) + } + /// Force the rollback of this extension, no matter the result. pub fn force_rollback(&mut self) { self.rollback = true; @@ -625,7 +621,9 @@ impl<'a> HeaderExtension<'a> { /// This may be either the header MMR or the sync MMR depending on the /// extension. pub fn apply_header(&mut self, header: &BlockHeader) -> Result { - self.pmmr.push(header).map_err(&ErrorKind::TxHashSetErr)?; + self.pmmr + .push(header.clone()) + .map_err(&ErrorKind::TxHashSetErr)?; self.header = header.clone(); Ok(self.root()) } @@ -641,7 +639,7 @@ impl<'a> HeaderExtension<'a> { let header_pos = pmmr::insertion_to_pmmr_index(header.height + 1); self.pmmr - .rewind(header_pos) + .rewind(header_pos, &Bitmap::create()) .map_err(&ErrorKind::TxHashSetErr)?; // Update our header to reflect the one we rewound to. @@ -655,7 +653,9 @@ impl<'a> HeaderExtension<'a> { /// including the genesis block header. pub fn truncate(&mut self) -> Result<(), Error> { debug!("Truncating header extension."); - self.pmmr.rewind(0).map_err(&ErrorKind::TxHashSetErr)?; + self.pmmr + .rewind(0, &Bitmap::create()) + .map_err(&ErrorKind::TxHashSetErr)?; Ok(()) } @@ -732,7 +732,7 @@ impl<'a> HeaderExtension<'a> { pub struct Extension<'a> { header: BlockHeader, - header_pmmr: DBPMMR<'a, BlockHeader, HashOnlyMMRBackend>, + header_pmmr: PMMR<'a, BlockHeader, PMMRBackend>, output_pmmr: PMMR<'a, OutputIdentifier, PMMRBackend>, rproof_pmmr: PMMR<'a, RangeProof, PMMRBackend>, kernel_pmmr: PMMR<'a, TxKernelEntry, PMMRBackend>, @@ -780,7 +780,7 @@ impl<'a> Extension<'a> { fn new(trees: &'a mut TxHashSet, batch: &'a Batch, header: BlockHeader) -> Extension<'a> { Extension { header, - header_pmmr: DBPMMR::at( + header_pmmr: PMMR::at( &mut trees.header_pmmr_h.backend, trees.header_pmmr_h.last_pos, ), @@ -960,7 +960,7 @@ impl<'a> Extension<'a> { fn apply_header(&mut self, header: &BlockHeader) -> Result<(), Error> { self.header_pmmr - .push(header) + .push(header.clone()) .map_err(&ErrorKind::TxHashSetErr)?; Ok(()) } @@ -1041,7 +1041,7 @@ impl<'a> Extension<'a> { ); self.header_pmmr - .rewind(header_pos) + .rewind(header_pos, &Bitmap::create()) .map_err(&ErrorKind::TxHashSetErr)?; self.output_pmmr .rewind(output_pos, rewind_rm_pos) diff --git a/core/src/core/block.rs b/core/src/core/block.rs index b8863df0b..609770dd7 100644 --- a/core/src/core/block.rs +++ b/core/src/core/block.rs @@ -34,7 +34,7 @@ use core::{ use global; use keychain::{self, BlindingFactor}; use pow::{Difficulty, Proof, ProofOfWork}; -use ser::{self, HashOnlyPMMRable, Readable, Reader, Writeable, Writer}; +use ser::{self, PMMRable, Readable, Reader, Writeable, Writer}; use util::{secp, static_secp_instance}; /// Errors thrown by Block validation @@ -159,7 +159,13 @@ impl Default for BlockHeader { } } -impl HashOnlyPMMRable for BlockHeader {} +impl PMMRable for BlockHeader { + type E = Hash; + + fn as_elmt(self) -> Self::E { + self.hash() + } +} /// Serialization of a block header impl Writeable for BlockHeader { diff --git a/core/src/core/pmmr/backend.rs b/core/src/core/pmmr/backend.rs index 69a098e62..8946168cd 100644 --- a/core/src/core/pmmr/backend.rs +++ b/core/src/core/pmmr/backend.rs @@ -18,18 +18,6 @@ use core::hash::Hash; use core::BlockHeader; use ser::PMMRable; -/// Simple "hash only" backend (used for header MMR, headers stored in the db). -pub trait HashOnlyBackend { - /// Append a vec of hashes to the backend. - fn append(&mut self, data: Vec) -> Result<(), String>; - - /// Rewind the backend to the specified position. - fn rewind(&mut self, position: u64) -> Result<(), String>; - - /// Get the hash at the specified position. - fn get_hash(&self, position: u64) -> Option; -} - /// Storage backend for the MMR, just needs to be indexed by order of insertion. /// The PMMR itself does not need the Backend to be accurate on the existence /// of an element (i.e. remove could be a no-op) but layers above can @@ -52,7 +40,7 @@ pub trait Backend { fn get_hash(&self, position: u64) -> Option; /// Get underlying data by insertion position. - fn get_data(&self, position: u64) -> Option; + fn get_data(&self, position: u64) -> Option; /// Get a Hash by original insertion position /// (ignoring the remove log). @@ -60,7 +48,7 @@ pub trait Backend { /// Get a Data Element by original insertion position /// (ignoring the remove log). - fn get_data_from_file(&self, position: u64) -> Option; + fn get_data_from_file(&self, position: u64) -> Option; /// Remove Hash by insertion position. An index is also provided so the /// underlying backend can implement some rollback of positions up to a diff --git a/core/src/core/pmmr/db_pmmr.rs b/core/src/core/pmmr/db_pmmr.rs deleted file mode 100644 index a4a89bc2e..000000000 --- a/core/src/core/pmmr/db_pmmr.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2018 The Grin Developers -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Database backed MMR. - -use std::marker; - -use core::hash::{Hash, ZERO_HASH}; -use core::pmmr::{bintree_postorder_height, is_leaf, peak_map_height, peaks, HashOnlyBackend}; -use ser::{HashOnlyPMMRable, PMMRIndexHashable}; - -/// Database backed MMR. -pub struct DBPMMR<'a, T, B> -where - T: HashOnlyPMMRable, - B: 'a + HashOnlyBackend, -{ - /// The last position in the PMMR - last_pos: u64, - /// The backend for this readonly PMMR - backend: &'a mut B, - // only needed to parameterise Backend - _marker: marker::PhantomData, -} - -impl<'a, T, B> DBPMMR<'a, T, B> -where - T: HashOnlyPMMRable, - B: 'a + HashOnlyBackend, -{ - /// Build a new db backed MMR. - pub fn new(backend: &'a mut B) -> DBPMMR { - DBPMMR { - backend, - last_pos: 0, - _marker: marker::PhantomData, - } - } - - /// Build a new db backed MMR initialized to - /// last_pos with the provided db backend. - pub fn at(backend: &'a mut B, last_pos: u64) -> DBPMMR { - DBPMMR { - backend, - last_pos, - _marker: marker::PhantomData, - } - } - - /// Is the MMR empty? - pub fn is_empty(&self) -> bool { - self.last_pos == 0 - } - - /// Total size of the tree, including intermediary nodes and ignoring any - /// pruning. - pub fn unpruned_size(&self) -> u64 { - self.last_pos - } - - /// Rewind the MMR to the specified position. - pub fn rewind(&mut self, position: u64) -> Result<(), String> { - // Identify which actual position we should rewind to as the provided - // position is a leaf. We traverse the MMR to include any parent(s) that - // need to be included for the MMR to be valid. - let mut pos = position; - while bintree_postorder_height(pos + 1) > 0 { - pos += 1; - } - self.backend.rewind(pos)?; - self.last_pos = pos; - Ok(()) - } - - /// Get the hash element at provided position in the MMR. - pub fn get_hash(&self, pos: u64) -> Option { - if pos > self.last_pos { - // If we are beyond the rhs of the MMR return None. - None - } else if is_leaf(pos) { - // If we are a leaf then get data from the backend. - self.backend.get_hash(pos) - } else { - // If we are not a leaf then return None as only leaves have data. - None - } - } - - /// Push a new element into the MMR. Computes new related peaks at - /// the same time if applicable. - pub fn push(&mut self, elmt: &T) -> Result { - let elmt_pos = self.last_pos + 1; - let mut current_hash = elmt.hash_with_index(elmt_pos - 1); - - let mut to_append = vec![current_hash]; - let mut pos = elmt_pos; - - let (peak_map, height) = peak_map_height(pos - 1); - if height != 0 { - return Err(format!("bad mmr size {}", pos - 1)); - } - // hash with all immediately preceding peaks, as indicated by peak map - let mut peak = 1; - while (peak_map & peak) != 0 { - let left_sibling = pos + 1 - 2 * peak; - let left_hash = self - .backend - .get_hash(left_sibling) - .ok_or("missing left sibling in tree, should not have been pruned")?; - peak *= 2; - pos += 1; - current_hash = (left_hash, current_hash).hash_with_index(pos - 1); - to_append.push(current_hash); - } - - // append all the new nodes and update the MMR index - self.backend.append(to_append)?; - self.last_pos = pos; - Ok(elmt_pos) - } - - /// Return the vec of peak hashes for this MMR. - pub fn peaks(&self) -> Vec { - let peaks_pos = peaks(self.last_pos); - peaks_pos - .into_iter() - .filter_map(|pi| self.backend.get_hash(pi)) - .collect() - } - - /// Return the overall root hash for this MMR. - pub fn root(&self) -> Hash { - if self.is_empty() { - return ZERO_HASH; - } - let mut res = None; - for peak in self.peaks().iter().rev() { - res = match res { - None => Some(*peak), - Some(rhash) => Some((*peak, rhash).hash_with_index(self.unpruned_size())), - } - } - res.expect("no root, invalid tree") - } - - /// Validate all the hashes in the MMR. - /// For every parent node we check hashes of the children produce the parent hash - /// by hashing them together. - pub fn validate(&self) -> Result<(), String> { - // iterate on all parent nodes - for n in 1..(self.last_pos + 1) { - let height = bintree_postorder_height(n); - if height > 0 { - if let Some(hash) = self.get_hash(n) { - let left_pos = n - (1 << height); - let right_pos = n - 1; - if let Some(left_child_hs) = self.get_hash(left_pos) { - if let Some(right_child_hs) = self.get_hash(right_pos) { - // hash the two child nodes together with parent_pos and compare - if (left_child_hs, right_child_hs).hash_with_index(n - 1) != hash { - return Err(format!( - "Invalid MMR, hash of parent at {} does \ - not match children.", - n - )); - } - } - } - } - } - } - Ok(()) - } -} diff --git a/core/src/core/pmmr/mod.rs b/core/src/core/pmmr/mod.rs index a15cbfce9..22f468f84 100644 --- a/core/src/core/pmmr/mod.rs +++ b/core/src/core/pmmr/mod.rs @@ -37,13 +37,11 @@ //! either be a simple Vec or a database. mod backend; -mod db_pmmr; mod pmmr; mod readonly_pmmr; mod rewindable_pmmr; pub use self::backend::*; -pub use self::db_pmmr::*; pub use self::pmmr::*; pub use self::readonly_pmmr::*; pub use self::rewindable_pmmr::*; diff --git a/core/src/core/pmmr/pmmr.rs b/core/src/core/pmmr/pmmr.rs index 0850bc517..8ef9ec687 100644 --- a/core/src/core/pmmr/pmmr.rs +++ b/core/src/core/pmmr/pmmr.rs @@ -175,7 +175,7 @@ where let elmt_pos = self.last_pos + 1; let mut current_hash = elmt.hash_with_index(elmt_pos - 1); - let mut to_append = vec![current_hash]; + let mut hashes = vec![current_hash]; let mut pos = elmt_pos; let (peak_map, height) = peak_map_height(pos - 1); @@ -193,11 +193,11 @@ where peak *= 2; pos += 1; current_hash = (left_hash, current_hash).hash_with_index(pos - 1); - to_append.push(current_hash); + hashes.push(current_hash); } // append all the new nodes and update the MMR index - self.backend.append(elmt, to_append)?; + self.backend.append(elmt, hashes)?; self.last_pos = pos; Ok(elmt_pos) } @@ -259,7 +259,7 @@ where } /// Get the data element at provided position in the MMR. - pub fn get_data(&self, pos: u64) -> Option { + pub fn get_data(&self, pos: u64) -> Option { if pos > self.last_pos { // If we are beyond the rhs of the MMR return None. None @@ -285,7 +285,7 @@ where /// Helper function to get the last N nodes inserted, i.e. the last /// n nodes along the bottom of the tree. /// May return less than n items if the MMR has been pruned/compacted. - pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T)> { + pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, T::E)> { let mut return_vec = vec![]; let mut last_leaf = self.last_pos; for _ in 0..n as u64 { @@ -307,7 +307,11 @@ where /// Helper function which returns un-pruned nodes from the insertion index /// forward /// returns last insertion index returned along with data - pub fn elements_from_insertion_index(&self, mut index: u64, max_count: u64) -> (u64, Vec) { + pub fn elements_from_insertion_index( + &self, + mut index: u64, + max_count: u64, + ) -> (u64, Vec) { let mut return_vec = vec![]; if index == 0 { index = 1; diff --git a/core/src/core/pmmr/readonly_pmmr.rs b/core/src/core/pmmr/readonly_pmmr.rs index 8fdeaf346..a9a3a7562 100644 --- a/core/src/core/pmmr/readonly_pmmr.rs +++ b/core/src/core/pmmr/readonly_pmmr.rs @@ -58,7 +58,7 @@ where } /// Get the data element at provided position in the MMR. - pub fn get_data(&self, pos: u64) -> Option { + pub fn get_data(&self, pos: u64) -> Option { if pos > self.last_pos { // If we are beyond the rhs of the MMR return None. None diff --git a/core/src/core/pmmr/rewindable_pmmr.rs b/core/src/core/pmmr/rewindable_pmmr.rs index 2f8854383..ca36101b1 100644 --- a/core/src/core/pmmr/rewindable_pmmr.rs +++ b/core/src/core/pmmr/rewindable_pmmr.rs @@ -75,7 +75,7 @@ where } /// Get the data element at provided position in the MMR. - pub fn get_data(&self, pos: u64) -> Option { + pub fn get_data(&self, pos: u64) -> Option { if pos > self.last_pos { // If we are beyond the rhs of the MMR return None. None diff --git a/core/src/core/transaction.rs b/core/src/core/transaction.rs index c7a41d4e1..6c6bd630e 100644 --- a/core/src/core/transaction.rs +++ b/core/src/core/transaction.rs @@ -310,7 +310,13 @@ impl FixedLength for TxKernelEntry { + secp::constants::AGG_SIGNATURE_SIZE; } -impl PMMRable for TxKernelEntry {} +impl PMMRable for TxKernelEntry { + type E = Self; + + fn as_elmt(self) -> Self::E { + self + } +} /// TransactionBody is a common abstraction for transaction and block #[derive(Serialize, Deserialize, Debug, Clone)] @@ -1237,7 +1243,13 @@ impl FixedLength for OutputIdentifier { const LEN: usize = 1 + secp::constants::PEDERSEN_COMMITMENT_SIZE; } -impl PMMRable for OutputIdentifier {} +impl PMMRable for OutputIdentifier { + type E = Self; + + fn as_elmt(self) -> Self::E { + self + } +} impl Writeable for OutputIdentifier { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { diff --git a/core/src/ser.rs b/core/src/ser.rs index b6f9396b7..ac7ed599d 100644 --- a/core/src/ser.rs +++ b/core/src/ser.rs @@ -512,7 +512,13 @@ impl FixedLength for RangeProof { + MAX_PROOF_SIZE; } -impl PMMRable for RangeProof {} +impl PMMRable for RangeProof { + type E = Self; + + fn as_elmt(self) -> Self::E { + self + } +} impl Readable for Signature { fn read(reader: &mut Reader) -> Result { @@ -682,11 +688,15 @@ pub trait FixedLength { const LEN: usize; } -/// Trait for types that can be added to a "hash only" PMMR (block headers for example). -pub trait HashOnlyPMMRable: Writeable + Clone + Debug {} - /// Trait for types that can be added to a PMMR. -pub trait PMMRable: FixedLength + Readable + Writeable + Clone + Debug {} +pub trait PMMRable: Writeable + Clone + Debug { + /// The type of element actually stored in the MMR data file. + /// This allows us to store Hash elements in the header MMR for variable size BlockHeaders. + type E: FixedLength + Readable + Writeable; + + /// Convert the pmmrable into the element to be stored in the MMR data file. + fn as_elmt(self) -> Self::E; +} /// Generic trait to ensure PMMR elements can be hashed with an index pub trait PMMRIndexHashable { diff --git a/core/tests/vec_backend/mod.rs b/core/tests/vec_backend/mod.rs index 06a36f833..9c22c6896 100644 --- a/core/tests/vec_backend/mod.rs +++ b/core/tests/vec_backend/mod.rs @@ -29,7 +29,13 @@ impl FixedLength for TestElem { const LEN: usize = 16; } -impl PMMRable for TestElem {} +impl PMMRable for TestElem { + type E = Self; + + fn as_elmt(self) -> Self::E { + self + } +} impl Writeable for TestElem { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { @@ -77,7 +83,7 @@ impl Backend for VecBackend { } } - fn get_data(&self, position: u64) -> Option { + fn get_data(&self, position: u64) -> Option { if self.remove_list.contains(&position) { None } else { @@ -90,10 +96,10 @@ impl Backend for VecBackend { Some(hash.clone()) } - fn get_data_from_file(&self, position: u64) -> Option { + fn get_data_from_file(&self, position: u64) -> Option { let idx = pmmr::n_leaves(position); - let data = &self.data[(idx - 1) as usize]; - Some(data.clone()) + let data = self.data[(idx - 1) as usize].clone(); + Some(data.as_elmt()) } fn remove(&mut self, position: u64) -> Result<(), String> { diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs index 35f078229..a0874c1a7 100644 --- a/store/src/pmmr.rs +++ b/store/src/pmmr.rs @@ -18,9 +18,9 @@ use std::{fs, io, marker}; use croaring::Bitmap; use core::core::hash::{Hash, Hashed}; -use core::core::pmmr::{self, family, Backend, HashOnlyBackend}; +use core::core::pmmr::{self, family, Backend}; use core::core::BlockHeader; -use core::ser::{self, PMMRable}; +use core::ser::{self, FixedLength, PMMRable}; use leaf_set::LeafSet; use prune_list::PruneList; use types::{prune_noop, AppendOnlyFile, HashFile}; @@ -69,7 +69,8 @@ impl Backend for PMMRBackend { let position = self.hash_file.size_unsync() + shift + 1; self.leaf_set.add(position); } - self.data_file.append(&mut ser::ser_vec(&data).unwrap()); + self.data_file + .append(&mut ser::ser_vec(&data.as_elmt()).unwrap()); for h in &hashes { self.hash_file .append(h) @@ -86,7 +87,7 @@ impl Backend for PMMRBackend { self.hash_file.read(position - shift) } - fn get_data_from_file(&self, position: u64) -> Option { + fn get_data_from_file(&self, position: u64) -> Option { if self.is_compacted(position) { return None; } @@ -94,8 +95,8 @@ impl Backend for PMMRBackend { let pos = pmmr::n_leaves(position) - 1; // Must be on disk, doing a read at the correct position - let file_offset = ((pos - shift) as usize) * T::LEN; - let data = self.data_file.read(file_offset, T::LEN); + let file_offset = ((pos - shift) as usize) * T::E::LEN; + let data = self.data_file.read(file_offset, T::E::LEN); match ser::deserialize(&mut &data[..]) { Ok(h) => Some(h), Err(e) => { @@ -120,7 +121,7 @@ impl Backend for PMMRBackend { /// Get the data at pos. /// Return None if it has been removed or if pos is not a leaf node. - fn get_data(&self, pos: u64) -> Option<(T)> { + fn get_data(&self, pos: u64) -> Option<(T::E)> { if !pmmr::is_leaf(pos) { return None; } @@ -146,7 +147,7 @@ impl Backend for PMMRBackend { // Rewind the data file accounting for pruned/compacted pos let leaf_shift = self.prune_list.get_leaf_shift(position); let flatfile_pos = pmmr::n_leaves(position); - let file_pos = (flatfile_pos - leaf_shift) * T::LEN as u64; + let file_pos = (flatfile_pos - leaf_shift) * T::E::LEN as u64; self.data_file.rewind(file_pos); Ok(()) @@ -240,7 +241,7 @@ impl PMMRBackend { /// Number of elements in the underlying stored data. Extremely dependent on /// pruning and compaction. pub fn data_size(&self) -> u64 { - self.data_file.size() / T::LEN as u64 + self.data_file.size() / T::E::LEN as u64 } /// Size of the underlying hashed data. Extremely dependent on pruning @@ -332,13 +333,13 @@ impl PMMRBackend { let off_to_rm = map_vec!(leaf_pos_to_rm, |&pos| { let flat_pos = pmmr::n_leaves(pos); let shift = self.prune_list.get_leaf_shift(pos); - (flat_pos - 1 - shift) * T::LEN as u64 + (flat_pos - 1 - shift) * T::E::LEN as u64 }); self.data_file.save_prune( tmp_prune_file_data.clone(), &off_to_rm, - T::LEN as u64, + T::E::LEN as u64, prune_cb, )?; } @@ -405,64 +406,6 @@ impl PMMRBackend { } } -/// Simple MMR Backend for hashes only (data maintained in the db). -pub struct HashOnlyMMRBackend { - /// The hash file underlying this MMR backend. - hash_file: HashFile, -} - -impl HashOnlyBackend for HashOnlyMMRBackend { - fn append(&mut self, hashes: Vec) -> Result<(), String> { - for h in &hashes { - self.hash_file - .append(h) - .map_err(|e| format!("Failed to append to backend, {:?}", e))?; - } - Ok(()) - } - - fn rewind(&mut self, position: u64) -> Result<(), String> { - self.hash_file - .rewind(position) - .map_err(|e| format!("Failed to rewind backend, {:?}", e))?; - Ok(()) - } - - fn get_hash(&self, position: u64) -> Option { - self.hash_file.read(position) - } -} - -impl HashOnlyMMRBackend { - /// Instantiates a new PMMR backend. - /// Use the provided dir to store its files. - pub fn new(data_dir: &str) -> io::Result { - let hash_file = HashFile::open(&format!("{}/{}", data_dir, PMMR_HASH_FILE))?; - Ok(HashOnlyMMRBackend { hash_file }) - } - - /// The unpruned size of this MMR backend. - pub fn unpruned_size(&self) -> u64 { - self.hash_file.size() - } - - /// Discard any pending changes to this MMR backend. - pub fn discard(&mut self) { - self.hash_file.discard(); - } - - /// Sync pending changes to the backend file on disk. - pub fn sync(&mut self) -> io::Result<()> { - if let Err(e) = self.hash_file.flush() { - return Err(io::Error::new( - io::ErrorKind::Interrupted, - format!("Could not write to hash storage, disk full? {:?}", e), - )); - } - Ok(()) - } -} - /// Filter remove list to exclude roots. /// We want to keep roots around so we have hashes for Merkle proofs. fn removed_excl_roots(removed: &Bitmap) -> Bitmap { diff --git a/store/tests/pmmr.rs b/store/tests/pmmr.rs index c2a118124..2d7043b6f 100644 --- a/store/tests/pmmr.rs +++ b/store/tests/pmmr.rs @@ -758,7 +758,13 @@ impl FixedLength for TestElem { const LEN: usize = 4; } -impl PMMRable for TestElem {} +impl PMMRable for TestElem { + type E = Self; + + fn as_elmt(self) -> Self::E { + self + } +} impl Writeable for TestElem { fn write(&self, writer: &mut W) -> Result<(), Error> {