diff --git a/api/src/handlers.rs b/api/src/handlers.rs index e79794064..acefd9ad5 100644 --- a/api/src/handlers.rs +++ b/api/src/handlers.rs @@ -206,13 +206,13 @@ impl Handler for UtxoHandler { } // Sum tree handler. Retrieve the roots: -// GET /v1/sumtrees/roots +// GET /v1/pmmrtrees/roots // // Last inserted nodes:: -// GET /v1/sumtrees/lastutxos (gets last 10) -// GET /v1/sumtrees/lastutxos?n=5 -// GET /v1/sumtrees/lastrangeproofs -// GET /v1/sumtrees/lastkernels +// GET /v1/pmmrtrees/lastutxos (gets last 10) +// GET /v1/pmmrtrees/lastutxos?n=5 +// GET /v1/pmmrtrees/lastrangeproofs +// GET /v1/pmmrtrees/lastkernels struct SumTreeHandler { chain: Weak, } @@ -224,18 +224,18 @@ impl SumTreeHandler { } // gets last n utxos inserted in to the tree - fn get_last_n_utxo(&self, distance: u64) -> Vec { - SumTreeNode::get_last_n_utxo(w(&self.chain), distance) + fn get_last_n_utxo(&self, distance: u64) -> Vec { + PmmrTreeNode::get_last_n_utxo(w(&self.chain), distance) } // gets last n utxos inserted in to the tree - fn get_last_n_rangeproof(&self, distance: u64) -> Vec { - SumTreeNode::get_last_n_rangeproof(w(&self.chain), distance) + fn get_last_n_rangeproof(&self, distance: u64) -> Vec { + PmmrTreeNode::get_last_n_rangeproof(w(&self.chain), distance) } // gets last n utxos inserted in to the tree - fn get_last_n_kernel(&self, distance: u64) -> Vec { - SumTreeNode::get_last_n_kernel(w(&self.chain), distance) + fn get_last_n_kernel(&self, distance: u64) -> Vec { + PmmrTreeNode::get_last_n_kernel(w(&self.chain), distance) } } @@ -620,10 +620,10 @@ pub fn start_rest_apis( "get chain".to_string(), "get chain/utxos".to_string(), "get status".to_string(), - "get sumtrees/roots".to_string(), - "get sumtrees/lastutxos?n=10".to_string(), - "get sumtrees/lastrangeproofs".to_string(), - "get sumtrees/lastkernels".to_string(), + "get pmmrtrees/roots".to_string(), + "get pmmrtrees/lastutxos?n=10".to_string(), + "get pmmrtrees/lastrangeproofs".to_string(), + "get pmmrtrees/lastkernels".to_string(), "get pool".to_string(), "post pool/push".to_string(), "post peers/a.b.c.d:p/ban".to_string(), @@ -641,7 +641,7 @@ pub fn start_rest_apis( chain_tip: get "/chain" => chain_tip_handler, chain_utxos: get "/chain/utxos/*" => utxo_handler, status: get "/status" => status_handler, - sumtree_roots: get "/sumtrees/*" => sumtree_handler, + sumtree_roots: get "/pmmrtrees/*" => sumtree_handler, pool_info: get "/pool" => pool_info_handler, pool_push: post "/pool/push" => pool_push_handler, peers_all: get "/peers/all" => peers_all_handler, diff --git a/api/src/types.rs b/api/src/types.rs index eecac01ea..59dc7b8a7 100644 --- a/api/src/types.rs +++ b/api/src/types.rs @@ -16,7 +16,6 @@ use std::sync::Arc; use core::{core, ser}; use core::core::hash::Hashed; -use core::core::SumCommit; use core::core::SwitchCommitHash; use chain; use p2p; @@ -89,8 +88,6 @@ impl Status { pub struct SumTrees { /// UTXO Root Hash pub utxo_root_hash: String, - // UTXO Root Sum - pub utxo_root_sum: String, // Rangeproof root hash pub range_proof_root_hash: String, // Kernel set root hash @@ -101,10 +98,9 @@ impl SumTrees { pub fn from_head(head: Arc) -> SumTrees { let roots = head.get_sumtree_roots(); SumTrees { - utxo_root_hash: roots.0.hash.to_hex(), - utxo_root_sum: roots.0.sum.to_hex(), - range_proof_root_hash: roots.1.hash.to_hex(), - kernel_root_hash: roots.2.hash.to_hex(), + utxo_root_hash: roots.0.to_hex(), + range_proof_root_hash: roots.1.to_hex(), + kernel_root_hash: roots.2.to_hex(), } } } @@ -112,45 +108,40 @@ impl SumTrees { /// Wrapper around a list of sumtree nodes, so it can be /// presented properly via json #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct SumTreeNode { +pub struct PmmrTreeNode { // The hash pub hash: String, - // SumCommit (features|commitment), optional (only for utxos) - pub sum: Option, } -impl SumTreeNode { - pub fn get_last_n_utxo(chain: Arc, distance: u64) -> Vec { +impl PmmrTreeNode { + pub fn get_last_n_utxo(chain: Arc, distance: u64) -> Vec { let mut return_vec = Vec::new(); let last_n = chain.get_last_n_utxo(distance); for x in last_n { - return_vec.push(SumTreeNode { - hash: util::to_hex(x.hash.to_vec()), - sum: Some(x.sum), + return_vec.push(PmmrTreeNode { + hash: util::to_hex(x.0.to_vec()), }); } return_vec } - pub fn get_last_n_rangeproof(head: Arc, distance: u64) -> Vec { + pub fn get_last_n_rangeproof(head: Arc, distance: u64) -> Vec { let mut return_vec = Vec::new(); let last_n = head.get_last_n_rangeproof(distance); for elem in last_n { - return_vec.push(SumTreeNode { - hash: util::to_hex(elem.hash.to_vec()), - sum: None, + return_vec.push(PmmrTreeNode { + hash: util::to_hex(elem.0.to_vec()), }); } return_vec } - pub fn get_last_n_kernel(head: Arc, distance: u64) -> Vec { + pub fn get_last_n_kernel(head: Arc, distance: u64) -> Vec { let mut return_vec = Vec::new(); let last_n = head.get_last_n_kernel(distance); for elem in last_n { - return_vec.push(SumTreeNode { - hash: util::to_hex(elem.hash.to_vec()), - sum: None, + return_vec.push(PmmrTreeNode { + hash: util::to_hex(elem.0.to_vec()), }); } return_vec diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 5ad68dafd..f4077b54a 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -20,21 +20,18 @@ use std::fs::File; use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; -use util::secp::pedersen::RangeProof; - -use core::core::{Input, OutputIdentifier, SumCommit}; -use core::core::hash::Hashed; -use core::core::pmmr::{HashSum, NoSum}; +use core::core::{Input, OutputIdentifier, OutputStoreable, TxKernel}; +use core::core::hash::{Hash, Hashed}; use core::global; -use core::core::{Block, BlockHeader, TxKernel}; +use core::core::{Block, BlockHeader}; use core::core::target::Difficulty; -use core::core::hash::Hash; use grin_store::Error::NotFoundErr; use pipe; use store; use sumtree; use types::*; +use util::secp::pedersen::RangeProof; use util::LOGGER; @@ -428,9 +425,9 @@ impl Chain { Ok(extension.roots()) })?; - b.header.utxo_root = roots.0.hash; - b.header.range_proof_root = roots.1.hash; - b.header.kernel_root = roots.2.hash; + b.header.utxo_root = roots.utxo_root; + b.header.range_proof_root = roots.rproof_root; + b.header.kernel_root = roots.kernel_root; Ok(()) } @@ -438,9 +435,9 @@ impl Chain { pub fn get_sumtree_roots( &self, ) -> ( - HashSum, - HashSum>, - HashSum>, + Hash, + Hash, + Hash, ) { let mut sumtrees = self.sumtrees.write().unwrap(); sumtrees.roots() @@ -507,7 +504,7 @@ impl Chain { { let mut head = self.head.lock().unwrap(); *head = Tip::from_block(&header); - self.store.save_body_head(&head)?; + let _ = self.store.save_body_head(&head); self.store.save_header_height(&header)?; } @@ -517,19 +514,19 @@ impl Chain { } /// returns the last n nodes inserted into the utxo sum tree - pub fn get_last_n_utxo(&self, distance: u64) -> Vec> { + pub fn get_last_n_utxo(&self, distance: u64) -> Vec<(Hash, Option)> { let mut sumtrees = self.sumtrees.write().unwrap(); sumtrees.last_n_utxo(distance) } /// as above, for rangeproofs - pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec>> { + pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, Option)> { let mut sumtrees = self.sumtrees.write().unwrap(); sumtrees.last_n_rangeproof(distance) } /// as above, for kernels - pub fn get_last_n_kernel(&self, distance: u64) -> Vec>> { + pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, Option)> { let mut sumtrees = self.sumtrees.write().unwrap(); sumtrees.last_n_kernel(distance) } diff --git a/chain/src/pipe.rs b/chain/src/pipe.rs index d8b3aead2..58c3cc5c5 100644 --- a/chain/src/pipe.rs +++ b/chain/src/pipe.rs @@ -305,28 +305,28 @@ fn validate_block( // apply the new block to the MMR trees and check the new root hashes ext.apply_block(&b)?; - let (utxo_root, rproof_root, kernel_root) = ext.roots(); - if utxo_root.hash != b.header.utxo_root || rproof_root.hash != b.header.range_proof_root - || kernel_root.hash != b.header.kernel_root + let roots = ext.roots(); + if roots.utxo_root != b.header.utxo_root || roots.rproof_root != b.header.range_proof_root + || roots.kernel_root != b.header.kernel_root { ext.dump(false); debug!( LOGGER, "validate_block: utxo roots - {:?}, {:?}", - utxo_root.hash, + roots.utxo_root, b.header.utxo_root, ); debug!( LOGGER, "validate_block: rproof roots - {:?}, {:?}", - rproof_root.hash, + roots.rproof_root, b.header.range_proof_root, ); debug!( LOGGER, "validate_block: kernel roots - {:?}, {:?}", - kernel_root.hash, + roots.kernel_root, b.header.kernel_root, ); diff --git a/chain/src/sumtree.rs b/chain/src/sumtree.rs index 3abdceba6..3a86c51be 100644 --- a/chain/src/sumtree.rs +++ b/chain/src/sumtree.rs @@ -1,4 +1,3 @@ - // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,26 +25,26 @@ use util::static_secp_instance; use util::secp::pedersen::{RangeProof, Commitment}; use core::consensus::reward; -use core::core::{Block, BlockHeader, SumCommit, Input, Output, OutputIdentifier, OutputFeatures, TxKernel}; -use core::core::pmmr::{self, HashSum, NoSum, Summable, PMMR}; -use core::core::hash::Hashed; -use core::ser; +use core::core::{Block, BlockHeader, Input, Output, OutputIdentifier, + OutputFeatures, OutputStoreable, TxKernel}; +use core::core::pmmr::{self, PMMR}; +use core::core::hash::{Hash, Hashed}; +use core::ser::{self, PMMRable}; + use grin_store; -use grin_store::sumtree::{PMMRBackend, AppendOnlyFile}; -use types::ChainStore; -use types::Error; +use grin_store::pmmr::PMMRBackend; +use types::{ChainStore, SumTreeRoots, Error}; use util::{LOGGER, zip}; const SUMTREES_SUBDIR: &'static str = "sumtrees"; const UTXO_SUBDIR: &'static str = "utxo"; const RANGE_PROOF_SUBDIR: &'static str = "rangeproof"; const KERNEL_SUBDIR: &'static str = "kernel"; -const KERNEL_FILE: &'static str = "kernel_full_data.bin"; const SUMTREES_ZIP: &'static str = "sumtrees_snapshot.zip"; struct PMMRHandle where - T: Summable + Clone, + T: PMMRable, { backend: PMMRBackend, last_pos: u64, @@ -53,7 +52,7 @@ where impl PMMRHandle where - T: Summable + Clone, + T: PMMRable, { fn new(root_dir: String, file_name: &str) -> Result, Error> { let path = Path::new(&root_dir).join(SUMTREES_SUBDIR).join(file_name); @@ -70,20 +69,17 @@ where /// An easy to manipulate structure holding the 3 sum trees necessary to /// validate blocks and capturing the UTXO set, the range proofs and the /// kernels. Also handles the index of Commitments to positions in the -/// output and range proof sum trees. +/// output and range proof pmmr trees. /// /// Note that the index is never authoritative, only the trees are /// guaranteed to indicate whether an output is spent or not. The index /// may have commitments that have already been spent, even with /// pruning enabled. -/// -/// In addition of the sumtrees, this maintains the full list of kernel -/// data so it can be easily packaged for sync or validation. + pub struct SumTrees { - output_pmmr_h: PMMRHandle, - rproof_pmmr_h: PMMRHandle>, - kernel_pmmr_h: PMMRHandle>, - kernel_file: AppendOnlyFile, + utxo_pmmr_h: PMMRHandle, + rproof_pmmr_h: PMMRHandle, + kernel_pmmr_h: PMMRHandle, // chain store used as index of commitments to MMR positions commit_index: Arc, @@ -92,16 +88,20 @@ pub struct SumTrees { impl SumTrees { /// Open an existing or new set of backends for the SumTrees pub fn open(root_dir: String, commit_index: Arc) -> Result { - let mut kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect(); + + let utxo_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, UTXO_SUBDIR].iter().collect(); + fs::create_dir_all(utxo_file_path.clone())?; + + let rproof_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, RANGE_PROOF_SUBDIR].iter().collect(); + fs::create_dir_all(rproof_file_path.clone())?; + + let kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect(); fs::create_dir_all(kernel_file_path.clone())?; - kernel_file_path.push(KERNEL_FILE); - let kernel_file = AppendOnlyFile::open(kernel_file_path.to_str().unwrap().to_owned())?; Ok(SumTrees { - output_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?, + utxo_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?, rproof_pmmr_h: PMMRHandle::new(root_dir.clone(), RANGE_PROOF_SUBDIR)?, kernel_pmmr_h: PMMRHandle::new(root_dir.clone(), KERNEL_SUBDIR)?, - kernel_file: kernel_file, commit_index: commit_index, }) } @@ -109,19 +109,19 @@ impl SumTrees { /// Check is an output is unspent. /// We look in the index to find the output MMR pos. /// Then we check the entry in the output MMR and confirm the hash matches. - pub fn is_unspent(&mut self, output: &OutputIdentifier) -> Result<(), Error> { - match self.commit_index.get_output_pos(&output.commit) { + pub fn is_unspent(&mut self, output_id: &OutputIdentifier) -> Result<(), Error> { + match self.commit_index.get_output_pos(&output_id.commit) { Ok(pos) => { - let output_pmmr = PMMR::at( - &mut self.output_pmmr_h.backend, - self.output_pmmr_h.last_pos, + let output_pmmr:PMMR = PMMR::at( + &mut self.utxo_pmmr_h.backend, + self.utxo_pmmr_h.last_pos, ); - if let Some(HashSum { hash, sum: _ }) = output_pmmr.get(pos) { - let sum_commit = output.as_sum_commit(); - let hash_sum = HashSum::from_summable(pos, &sum_commit); - if hash == hash_sum.hash { + if let Some((hash, _)) = output_pmmr.get(pos, false) { + println!("Getting output ID hash"); + if hash == output_id.hash() { Ok(()) } else { + println!("MISMATCH BECAUSE THE BLOODY THING MISMATCHES"); Err(Error::SumTreeErr(format!("sumtree hash mismatch"))) } } else { @@ -164,20 +164,21 @@ impl SumTrees { /// returns the last N nodes inserted into the tree (i.e. the 'bottom' /// nodes at level 0 - pub fn last_n_utxo(&mut self, distance: u64) -> Vec> { - let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - output_pmmr.get_last_n_insertions(distance) + /// TODO: These need to return the actual data from the flat-files instead of hashes now + pub fn last_n_utxo(&mut self, distance: u64) -> Vec<(Hash, Option)> { + let utxo_pmmr:PMMR = PMMR::at(&mut self.utxo_pmmr_h.backend, self.utxo_pmmr_h.last_pos); + utxo_pmmr.get_last_n_insertions(distance) } /// as above, for range proofs - pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec>> { - let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); + pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<(Hash, Option)> { + let rproof_pmmr:PMMR = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); rproof_pmmr.get_last_n_insertions(distance) } /// as above, for kernels - pub fn last_n_kernel(&mut self, distance: u64) -> Vec>> { - let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); + pub fn last_n_kernel(&mut self, distance: u64) -> Vec<(Hash, Option)> { + let kernel_pmmr:PMMR = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); kernel_pmmr.get_last_n_insertions(distance) } @@ -186,17 +187,19 @@ impl SumTrees { indexes_at(block, self.commit_index.deref()) } + /// Get sum tree roots + /// TODO: Return data instead of hashes pub fn roots( &mut self, ) -> ( - HashSum, - HashSum>, - HashSum>, + Hash, + Hash, + Hash, ) { - let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos); - let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); - let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); + let output_pmmr:PMMR = PMMR::at(&mut self.utxo_pmmr_h.backend, self.utxo_pmmr_h.last_pos); + let rproof_pmmr:PMMR = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos); + let kernel_pmmr:PMMR = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos); (output_pmmr.root(), rproof_pmmr.root(), kernel_pmmr.root()) } } @@ -231,26 +234,23 @@ where match res { Err(e) => { debug!(LOGGER, "Error returned, discarding sumtree extension."); - trees.output_pmmr_h.backend.discard(); + trees.utxo_pmmr_h.backend.discard(); trees.rproof_pmmr_h.backend.discard(); trees.kernel_pmmr_h.backend.discard(); - trees.kernel_file.discard(); Err(e) } Ok(r) => { if rollback { debug!(LOGGER, "Rollbacking sumtree extension."); - trees.output_pmmr_h.backend.discard(); + trees.utxo_pmmr_h.backend.discard(); trees.rproof_pmmr_h.backend.discard(); trees.kernel_pmmr_h.backend.discard(); - trees.kernel_file.discard(); } else { debug!(LOGGER, "Committing sumtree extension."); - trees.output_pmmr_h.backend.sync()?; + trees.utxo_pmmr_h.backend.sync()?; trees.rproof_pmmr_h.backend.sync()?; trees.kernel_pmmr_h.backend.sync()?; - trees.kernel_file.flush()?; - trees.output_pmmr_h.last_pos = sizes.0; + trees.utxo_pmmr_h.last_pos = sizes.0; trees.rproof_pmmr_h.last_pos = sizes.1; trees.kernel_pmmr_h.last_pos = sizes.2; } @@ -265,11 +265,10 @@ where /// reversible manner within a unit of work provided by the `extending` /// function. pub struct Extension<'a> { - output_pmmr: PMMR<'a, SumCommit, PMMRBackend>, - rproof_pmmr: PMMR<'a, NoSum, PMMRBackend>>, - kernel_pmmr: PMMR<'a, NoSum, PMMRBackend>>, + utxo_pmmr: PMMR<'a, OutputStoreable, PMMRBackend>, + rproof_pmmr: PMMR<'a, RangeProof, PMMRBackend>, + kernel_pmmr: PMMR<'a, TxKernel, PMMRBackend>, - kernel_file: &'a mut AppendOnlyFile, commit_index: Arc, new_output_commits: HashMap, new_kernel_excesses: HashMap, @@ -284,9 +283,9 @@ impl<'a> Extension<'a> { ) -> Extension<'a> { Extension { - output_pmmr: PMMR::at( - &mut trees.output_pmmr_h.backend, - trees.output_pmmr_h.last_pos, + utxo_pmmr: PMMR::at( + &mut trees.utxo_pmmr_h.backend, + trees.utxo_pmmr_h.last_pos, ), rproof_pmmr: PMMR::at( &mut trees.rproof_pmmr_h.backend, @@ -296,10 +295,10 @@ impl<'a> Extension<'a> { &mut trees.kernel_pmmr_h.backend, trees.kernel_pmmr_h.last_pos, ), - kernel_file: &mut trees.kernel_file, commit_index: commit_index, new_output_commits: HashMap::new(), new_kernel_excesses: HashMap::new(), + rollback: false, } } @@ -354,15 +353,14 @@ impl<'a> Extension<'a> { fn apply_input(&mut self, input: &Input, height: u64) -> Result<(), Error> { let commit = input.commitment(); let pos_res = self.get_output_pos(&commit); + let output_id_hash = OutputIdentifier::from_input(input).hash(); if let Ok(pos) = pos_res { - if let Some(HashSum { hash, sum: _ }) = self.output_pmmr.get(pos) { - let sum_commit = SumCommit::from_input(&input); - - // check hash from pmmr matches hash from input + if let Some((read_hash, read_elem)) = self.utxo_pmmr.get(pos, true) { + // check hash from pmmr matches hash from input (or corresponding output) // if not then the input is not being honest about // what it is attempting to spend... - let hash_sum = HashSum::from_summable(pos, &sum_commit); - if hash != hash_sum.hash { + if output_id_hash != read_hash || + output_id_hash != read_elem.expect("no output at position").hash() { return Err(Error::SumTreeErr(format!("output pmmr hash mismatch"))); } @@ -379,9 +377,10 @@ impl<'a> Extension<'a> { } } - // Now prune the output_pmmr and rproof_pmmr. + // Now prune the utxo_pmmr, rproof_pmmr and their storage. // Input is not valid if we cannot prune successfully (to spend an unspent output). - match self.output_pmmr.prune(pos, height as u32) { + // TODO: rm log, skip list for utxo and range proofs + match self.utxo_pmmr.prune(pos, height as u32) { Ok(true) => { self.rproof_pmmr .prune(pos, height as u32) @@ -398,7 +397,6 @@ impl<'a> Extension<'a> { fn apply_output(&mut self, out: &Output) -> Result<(), Error> { let commit = out.commitment(); - let sum_commit = SumCommit::from_output(out); if let Ok(pos) = self.get_output_pos(&commit) { // we need to check whether the commitment is in the current MMR view @@ -406,27 +404,24 @@ impl<'a> Extension<'a> { // (non-historical node will have a much smaller one) // note that this doesn't show the commitment *never* existed, just // that this is not an existing unspent commitment right now - if let Some(c) = self.output_pmmr.get(pos) { - let hash_sum = HashSum::from_summable(pos, &sum_commit); - + if let Some((hash, _)) = self.utxo_pmmr.get(pos, false) { // processing a new fork so we may get a position on the old // fork that exists but matches a different node // filtering that case out - if c.hash == hash_sum.hash { + if hash == OutputStoreable::from_output(out).hash() { return Err(Error::DuplicateCommitment(commit)); } } } - // push new outputs commitments in their MMR and save them in the index - let pos = self.output_pmmr - .push(sum_commit) + // push new outputs in their MMR and save them in the index + let pos = self.utxo_pmmr + .push(OutputStoreable::from_output(out)) .map_err(&Error::SumTreeErr)?; - self.new_output_commits.insert(out.commitment(), pos); - // push range proofs in their MMR + // push range proofs in their MMR and file self.rproof_pmmr - .push(NoSum(out.proof)) + .push(out.proof) .map_err(&Error::SumTreeErr)?; Ok(()) } @@ -434,9 +429,8 @@ impl<'a> Extension<'a> { fn apply_kernel(&mut self, kernel: &TxKernel) -> Result<(), Error> { if let Ok(pos) = self.get_kernel_pos(&kernel.excess) { // same as outputs - if let Some(k) = self.kernel_pmmr.get(pos) { - let hashsum = HashSum::from_summable(pos, &NoSum(kernel)); - if k.hash == hashsum.hash { + if let Some((h,_)) = self.kernel_pmmr.get(pos, false) { + if h == kernel.hash() { return Err(Error::DuplicateKernel(kernel.excess.clone())); } } @@ -444,10 +438,9 @@ impl<'a> Extension<'a> { // push kernels in their MMR and file let pos = self.kernel_pmmr - .push(NoSum(kernel.clone())) + .push(kernel.clone()) .map_err(&Error::SumTreeErr)?; self.new_kernel_excesses.insert(kernel.excess, pos); - self.kernel_file.append(&mut ser::ser_vec(&kernel).unwrap()); Ok(()) } @@ -465,13 +458,6 @@ impl<'a> Extension<'a> { // rewind each MMR let (out_pos_rew, kern_pos_rew) = indexes_at(block, self.commit_index.deref())?; self.rewind_pos(block.header.height, out_pos_rew, kern_pos_rew)?; - - // rewind the kernel file store, the position is the number of kernels - // multiplied by their size - // the number of kernels is the number of leaves in the MMR - let pos = pmmr::n_leaves(kern_pos_rew); - self.kernel_file.rewind(pos * (TxKernel::size() as u64)); - Ok(()) } @@ -485,7 +471,7 @@ impl<'a> Extension<'a> { kern_pos_rew, ); - self.output_pmmr + self.utxo_pmmr .rewind(out_pos_rew, height as u32) .map_err(&Error::SumTreeErr)?; self.rproof_pmmr @@ -495,7 +481,6 @@ impl<'a> Extension<'a> { .rewind(kern_pos_rew, height as u32) .map_err(&Error::SumTreeErr)?; - self.dump(true); Ok(()) } @@ -515,26 +500,23 @@ impl<'a> Extension<'a> { } } + /// Current root hashes and sums (if applicable) for the UTXO, range proof /// and kernel sum trees. pub fn roots( &self, - ) -> ( - HashSum, - HashSum>, - HashSum>, - ) { - ( - self.output_pmmr.root(), - self.rproof_pmmr.root(), - self.kernel_pmmr.root(), - ) + ) -> SumTreeRoots { + SumTreeRoots { + utxo_root: self.utxo_pmmr.root(), + rproof_root: self.rproof_pmmr.root(), + kernel_root: self.kernel_pmmr.root(), + } } /// Validate the current sumtree state against a block header pub fn validate(&self, header: &BlockHeader) -> Result<(), Error> { // validate all hashes and sums within the trees - if let Err(e) = self.output_pmmr.validate() { + if let Err(e) = self.utxo_pmmr.validate() { return Err(Error::InvalidSumtree(e)); } if let Err(e) = self.rproof_pmmr.validate() { @@ -545,9 +527,9 @@ impl<'a> Extension<'a> { } // validate the tree roots against the block header - let (utxo_root, rproof_root, kernel_root) = self.roots(); - if utxo_root.hash != header.utxo_root || rproof_root.hash != header.range_proof_root - || kernel_root.hash != header.kernel_root + let roots = self.roots(); + if roots.utxo_root != header.utxo_root || roots.rproof_root != header.range_proof_root + || roots.kernel_root != header.kernel_root { return Err(Error::InvalidRoot); } @@ -574,11 +556,11 @@ impl<'a> Extension<'a> { /// by iterating over the whole MMR data. This is a costly operation /// performed only when we receive a full new chain state. pub fn rebuild_index(&self) -> Result<(), Error> { - for n in 1..self.output_pmmr.unpruned_size()+1 { + for n in 1..self.utxo_pmmr.unpruned_size()+1 { // non-pruned leaves only if pmmr::bintree_postorder_height(n) == 0 { - if let Some(hs) = self.output_pmmr.get(n) { - self.commit_index.save_output_pos(&hs.sum.commit, n)?; + if let Some((_, out)) = self.utxo_pmmr.get(n, true) { + self.commit_index.save_output_pos(&out.expect("not a leaf node").commit, n)?; } } } @@ -594,7 +576,7 @@ impl<'a> Extension<'a> { /// version only prints the UTXO tree. pub fn dump(&self, short: bool) { debug!(LOGGER, "-- outputs --"); - self.output_pmmr.dump(short); + self.utxo_pmmr.dump(short); if !short { debug!(LOGGER, "-- range proofs --"); self.rproof_pmmr.dump(short); @@ -606,7 +588,7 @@ impl<'a> Extension<'a> { // Sizes of the sum trees, used by `extending` on rollback. fn sizes(&self) -> (u64, u64, u64) { ( - self.output_pmmr.unpruned_size(), + self.utxo_pmmr.unpruned_size(), self.rproof_pmmr.unpruned_size(), self.kernel_pmmr.unpruned_size(), ) @@ -619,7 +601,7 @@ impl<'a> Extension<'a> { let mmr_sz = self.kernel_pmmr.unpruned_size(); let count = pmmr::n_leaves(mmr_sz); - let mut kernel_file = File::open(self.kernel_file.path())?; + let mut kernel_file = File::open(self.kernel_pmmr.data_file_path())?; let first: TxKernel = ser::deserialize(&mut kernel_file)?; first.verify()?; let mut sum_kernel = first.excess; @@ -651,14 +633,14 @@ impl<'a> Extension<'a> { let mut sum_utxo = None; let mut utxo_count = 0; let secp = static_secp_instance(); - for n in 1..self.output_pmmr.unpruned_size()+1 { + for n in 1..self.utxo_pmmr.unpruned_size()+1 { if pmmr::bintree_postorder_height(n) == 0 { - if let Some(hs) = self.output_pmmr.get(n) { + if let Some((_,output)) = self.utxo_pmmr.get(n, true) { if n == 1 { - sum_utxo = Some(hs.sum.commit); + sum_utxo = Some(output.expect("not a leaf node").commit); } else { let secp = secp.lock().unwrap(); - sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), hs.sum.commit], vec![])?); + sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), output.expect("not a leaf node").commit], vec![])?); } utxo_count += 1; } @@ -669,6 +651,42 @@ impl<'a> Extension<'a> { } } +/*fn store_element(file_store: &mut FlatFileStore, data: T) + -> Result<(), String> +where + T: ser::Readable + ser::Writeable + Clone +{ + file_store.append(vec![data]) +} + +fn read_element_at_pmmr_index(file_store: &FlatFileStore, pos: u64) -> Option +where + T: ser::Readable + ser::Writeable + Clone +{ + let leaf_index = pmmr::leaf_index(pos); + // flat files are zero indexed + file_store.get(leaf_index - 1) +} + +fn _remove_element_at_pmmr_index(file_store: &mut FlatFileStore, pos: u64) + -> Result<(), String> +where + T: ser::Readable + ser::Writeable + Clone +{ + let leaf_index = pmmr::leaf_index(pos); + // flat files are zero indexed + file_store.remove(vec![leaf_index - 1]) +} + +fn rewind_to_pmmr_index(file_store: &mut FlatFileStore, pos: u64) -> Result<(), String> +where + T: ser::Readable + ser::Writeable + Clone +{ + let leaf_index = pmmr::leaf_index(pos); + // flat files are zero indexed + file_store.rewind(leaf_index - 1) +}*/ + /// Output and kernel MMR indexes at the end of the provided block fn indexes_at(block: &Block, commit_index: &ChainStore) -> Result<(u64, u64), Error> { let out_idx = match block.outputs.last() { diff --git a/chain/src/types.rs b/chain/src/types.rs index 43ab71fbd..a16a64512 100644 --- a/chain/src/types.rs +++ b/chain/src/types.rs @@ -40,6 +40,17 @@ bitflags! { } } +/// A helper to hold the roots of the sumtrees in order to keep them +/// readable +pub struct SumTreeRoots { + /// UTXO root + pub utxo_root: Hash, + /// Range Proof root + pub rproof_root: Hash, + /// Kernel root + pub kernel_root: Hash, +} + /// Errors #[derive(Debug)] pub enum Error { @@ -81,6 +92,8 @@ pub enum Error { InvalidSumtree(String), /// Internal issue when trying to save or load data from store StoreErr(grin_store::Error, String), + /// Internal issue when trying to save or load data from append only files + FileReadErr(String), /// Error serializing or deserializing a type SerErr(ser::Error), /// Error with the sumtrees diff --git a/chain/tests/mine_simple_chain.rs b/chain/tests/mine_simple_chain.rs index 8fff8b7a6..6cb4ce579 100644 --- a/chain/tests/mine_simple_chain.rs +++ b/chain/tests/mine_simple_chain.rs @@ -253,6 +253,8 @@ fn spend_in_fork() { fork_head = b.header.clone(); chain.process_block(b, chain::Options::SKIP_POW).unwrap(); + println!("First block"); + // now mine three further blocks for n in 3..6 { let b = prepare_block(&kc, &fork_head, &chain, n); @@ -263,6 +265,8 @@ fn spend_in_fork() { let lock_height = 1 + global::coinbase_maturity(); assert_eq!(lock_height, 4); + println!("3 Further Blocks: should have 4 blocks or 264 bytes in file "); + let tx1 = build::transaction( vec![ build::coinbase_input(consensus::REWARD, block_hash, kc.derive_key_id(2).unwrap()), @@ -272,10 +276,14 @@ fn spend_in_fork() { &kc, ).unwrap(); + println!("Built coinbase input and output"); + let next = prepare_block_tx(&kc, &fork_head, &chain, 7, vec![&tx1]); let prev_main = next.header.clone(); chain.process_block(next.clone(), chain::Options::SKIP_POW).unwrap(); + println!("tx 1 processed, should have 6 outputs or 396 bytes in file, first skipped"); + let tx2 = build::transaction( vec![ build::input(consensus::REWARD - 20000, next.hash(), kc.derive_key_id(30).unwrap()), @@ -289,6 +297,9 @@ fn spend_in_fork() { let prev_main = next.header.clone(); chain.process_block(next, chain::Options::SKIP_POW).unwrap(); + println!("tx 2 processed"); + /*panic!("Stop");*/ + // mine 2 forked blocks from the first let fork = prepare_fork_block_tx(&kc, &fork_head, &chain, 6, vec![&tx1]); let prev_fork = fork.header.clone(); diff --git a/core/src/core/hash.rs b/core/src/core/hash.rs index 7b3131ec2..27d1918ff 100644 --- a/core/src/core/hash.rs +++ b/core/src/core/hash.rs @@ -20,13 +20,13 @@ use std::cmp::min; use std::{fmt, ops}; use std::convert::AsRef; +use std::ops::Add; use blake2::blake2b::Blake2b; use consensus; use ser::{self, AsFixedBytes, Error, Readable, Reader, Writeable, Writer}; use util; -use util::LOGGER; /// A hash consisting of all zeroes, used as a sentinel. No known preimage. pub const ZERO_HASH: Hash = Hash([0; 32]); @@ -147,6 +147,13 @@ impl Writeable for Hash { } } +impl Add for Hash { + type Output = Hash; + fn add(self, other: Hash) -> Hash { + self.hash_with(other) + } +} + /// Serializer that outputs a hash of the serialized object pub struct HashWriter { state: Blake2b, @@ -205,7 +212,6 @@ impl Hashed for W { fn hash_with(&self, other: T) -> Hash { let mut hasher = HashWriter::default(); ser::Writeable::write(self, &mut hasher).unwrap(); - trace!(LOGGER, "Hashing with additional data"); ser::Writeable::write(&other, &mut hasher).unwrap(); let mut ret = [0; 32]; hasher.finalize(&mut ret); diff --git a/core/src/core/pmmr.rs b/core/src/core/pmmr.rs index b7fe85bb8..1a4b29214 100644 --- a/core/src/core/pmmr.rs +++ b/core/src/core/pmmr.rs @@ -29,169 +29,30 @@ //! position of siblings, parents, etc. As all those functions only rely on //! binary operations, they're extremely fast. For more information, see the //! doc on bintree_jump_left_sibling. -//! 2. The implementation of a prunable MMR sum tree using the above. Each leaf -//! is required to be Summable and Hashed. Tree roots can be trivially and +//! 2. The implementation of a prunable MMR tree using the above. Each leaf +//! is required to be Writeable (which implements Hashed). Tree roots can be trivially and //! efficiently calculated without materializing the full tree. The underlying -//! (Hash, Sum) pais are stored in a Backend implementation that can either be +//! Hashes are stored in a Backend implementation that can either be //! a simple Vec or a database. use std::clone::Clone; +use std::ops::Deref; use std::marker::PhantomData; -use std::ops::{self, Deref}; use core::hash::{Hash, Hashed}; -use ser::{self, Readable, Reader, Writeable, Writer}; +use ser::PMMRable; use util::LOGGER; -/// Trait for an element of the tree that has a well-defined sum and hash that -/// the tree can sum over -pub trait Summable { - /// The type of the sum - type Sum: Clone + ops::Add + Readable + Writeable + PartialEq; - - /// Obtain the sum of the element - fn sum(&self) -> Self::Sum; - - /// Length of the Sum type when serialized. Can be used as a hint by - /// underlying storages. - fn sum_len() -> usize; -} - -/// An empty sum that takes no space, to store elements that do not need summing -/// but can still leverage the hierarchical hashing. -#[derive(Copy, Clone, Debug)] -pub struct NullSum; -impl ops::Add for NullSum { - type Output = NullSum; - fn add(self, _: NullSum) -> NullSum { - NullSum - } -} - -impl Readable for NullSum { - fn read(_: &mut Reader) -> Result { - Ok(NullSum) - } -} - -impl Writeable for NullSum { - fn write(&self, _: &mut W) -> Result<(), ser::Error> { - Ok(()) - } -} - -impl PartialEq for NullSum { - fn eq(&self, _other: &NullSum) -> bool { - true - } -} - -/// Wrapper for a type that allows it to be inserted in a tree without summing -#[derive(Clone, Debug)] -pub struct NoSum(pub T); -impl Summable for NoSum { - type Sum = NullSum; - fn sum(&self) -> NullSum { - NullSum - } - fn sum_len() -> usize { - return 0; - } -} -impl Writeable for NoSum -where - T: Writeable, -{ - fn write(&self, writer: &mut W) -> Result<(), ser::Error> { - self.0.write(writer) - } -} - -/// A utility type to handle (Hash, Sum) pairs more conveniently. The addition -/// of two HashSums is the (Hash(h1|h2), h1 + h2) HashSum. -#[derive(Debug, Clone, Eq)] -pub struct HashSum -where - T: Summable, -{ - /// The hash - pub hash: Hash, - /// The sum - pub sum: T::Sum, -} - -impl HashSum -where - T: Summable + Hashed, -{ - /// Create a hash sum from a summable - pub fn from_summable(idx: u64, elmt: &T) -> HashSum { - let hash = elmt.hash(); - let sum = elmt.sum(); - let node_hash = (idx, &sum, hash).hash(); - HashSum { - hash: node_hash, - sum: sum, - } - } -} - -impl PartialEq for HashSum -where - T: Summable, -{ - fn eq(&self, other: &HashSum) -> bool { - self.hash == other.hash && self.sum == other.sum - } -} - -impl Readable for HashSum -where - T: Summable, -{ - fn read(r: &mut Reader) -> Result, ser::Error> { - Ok(HashSum { - hash: Hash::read(r)?, - sum: T::Sum::read(r)?, - }) - } -} - -impl Writeable for HashSum -where - T: Summable, -{ - fn write(&self, w: &mut W) -> Result<(), ser::Error> { - self.hash.write(w)?; - self.sum.write(w) - } -} - -impl ops::Add for HashSum -where - T: Summable, -{ - type Output = HashSum; - fn add(self, other: HashSum) -> HashSum { - HashSum { - hash: (self.hash, other.hash).hash(), - sum: self.sum + other.sum, - } - } -} - /// Storage backend for the MMR, just needs to be indexed by order of insertion. /// The PMMR itself does not need the Backend to be accurate on the existence /// of an element (i.e. remove could be a no-op) but layers above can /// depend on an accurate Backend to check existence. -pub trait Backend -where - T: Summable, -{ - /// Append the provided HashSums to the backend storage. The position of the - /// first element of the Vec in the MMR is provided to help the - /// implementation. - fn append(&mut self, position: u64, data: Vec>) -> Result<(), String>; +pub trait Backend where + T:PMMRable { + /// Append the provided Hashes to the backend storage, and optionally an associated + /// data element to flatfile storage (for leaf nodes only). The position of the + /// first element of the Vec in the MMR is provided to help the implementation. + fn append(&mut self, position: u64, data: Vec<(Hash, Option)>) -> Result<(), String>; /// Rewind the backend state to a previous position, as if all append /// operations after that had been canceled. Expects a position in the PMMR @@ -199,14 +60,20 @@ where /// occurred (see remove). fn rewind(&mut self, position: u64, index: u32) -> Result<(), String>; - /// Get a HashSum by insertion position - fn get(&self, position: u64) -> Option>; + /// Get a Hash/Element by insertion position. If include_data is true, will + /// also return the associated data element + fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option)>; - /// Remove HashSums by insertion position. An index is also provided so the + /// Remove Hashes/Data by insertion position. An index is also provided so the /// underlying backend can implement some rollback of positions up to a /// given index (practically the index is a the height of a block that /// triggered removal). fn remove(&mut self, positions: Vec, index: u32) -> Result<(), String>; + + /// Returns the data file path.. this is a bit of a hack now that doesn't + /// sit well with the design, but TxKernels have to be summed and the + /// fastest way to to be able to allow direct access to the file + fn get_data_file_path(&self) -> String; } /// Prunable Merkle Mountain Range implementation. All positions within the tree @@ -218,18 +85,18 @@ where /// we are in the sequence of nodes making up the MMR. pub struct PMMR<'a, T, B> where - T: Summable, + T: PMMRable, B: 'a + Backend, { last_pos: u64, backend: &'a mut B, // only needed for parameterizing Backend - summable: PhantomData, + writeable: PhantomData, } impl<'a, T, B> PMMR<'a, T, B> where - T: Summable + Hashed + Clone, + T: PMMRable, B: 'a + Backend, { /// Build a new prunable Merkle Mountain Range using the provided backend. @@ -237,7 +104,7 @@ where PMMR { last_pos: 0, backend: backend, - summable: PhantomData, + writeable: PhantomData, } } @@ -248,48 +115,51 @@ where PMMR { last_pos: last_pos, backend: backend, - summable: PhantomData, + writeable: PhantomData, } } /// Computes the root of the MMR. Find all the peaks in the current /// tree and "bags" them to get a single peak. - pub fn root(&self) -> HashSum { + pub fn root(&self) -> Hash { let peaks_pos = peaks(self.last_pos); - let peaks: Vec>> = map_vec!(peaks_pos, |&pi| self.backend.get(pi)); + let peaks: Vec)>> = peaks_pos.into_iter() + .map(|pi| self.backend.get(pi, false)) + .collect(); + let mut ret = None; for peak in peaks { ret = match (ret, peak) { (None, x) => x, - (Some(hsum), None) => Some(hsum), - (Some(lhsum), Some(rhsum)) => Some(lhsum + rhsum), + (Some(hash), None) => Some(hash), + (Some(lhash), Some(rhash)) => Some((lhash.0.hash_with(rhash.0), None)), } } - ret.expect("no root, invalid tree") + ret.expect("no root, invalid tree").0 } - /// Push a new Summable element in the MMR. Computes new related peaks at + /// Push a new element into the MMR. Computes new related peaks at /// the same time if applicable. pub fn push(&mut self, elmt: T) -> Result { let elmt_pos = self.last_pos + 1; - let mut current_hashsum = HashSum::from_summable(elmt_pos, &elmt); - let mut to_append = vec![current_hashsum.clone()]; + let mut current_hash = elmt.hash(); + let mut to_append = vec![(current_hash, Some(elmt))]; let mut height = 0; let mut pos = elmt_pos; // we look ahead one position in the MMR, if the expected node has a higher - // height it means we have to build a higher peak by summing with a previous + // height it means we have to build a higher peak by hashing with a previous // sibling. we do it iteratively in case the new peak itself allows the // creation of another parent. while bintree_postorder_height(pos + 1) > height { let left_sibling = bintree_jump_left_sibling(pos); - let left_hashsum = self.backend.get(left_sibling).expect( + let left_elem = self.backend.get(left_sibling, false).expect( "missing left sibling in tree, should not have been pruned", ); - current_hashsum = left_hashsum + current_hashsum; + current_hash = left_elem.0 + current_hash; - to_append.push(current_hashsum.clone()); + to_append.push((current_hash.clone(), None)); height += 1; pos += 1; } @@ -322,7 +192,7 @@ where /// to keep an index of elements to positions in the tree. Prunes parent /// nodes as well when they become childless. pub fn prune(&mut self, position: u64, index: u32) -> Result { - if let None = self.backend.get(position) { + if let None = self.backend.get(position, false) { return Ok(false); } let prunable_height = bintree_postorder_height(position); @@ -345,7 +215,7 @@ where // if we have a pruned sibling, we can continue up the tree // otherwise we're done - if let None = self.backend.get(sibling) { + if let None = self.backend.get(sibling, false) { current = parent; } else { break; @@ -356,26 +226,27 @@ where Ok(true) } - /// Helper function to get the HashSum of a node at a given position from + /// Helper function to get a node at a given position from /// the backend. - pub fn get(&self, position: u64) -> Option> { + pub fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option)> { if position > self.last_pos { None } else { - self.backend.get(position) + self.backend.get(position, include_data) } } + /// Helper function to get the last N nodes inserted, i.e. the last /// n nodes along the bottom of the tree - pub fn get_last_n_insertions(&self, n: u64) -> Vec> { + pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, Option)> { let mut return_vec = Vec::new(); let mut last_leaf = self.last_pos; let size = self.unpruned_size(); // Special case that causes issues in bintree functions, // just return if size == 1 { - return_vec.push(self.backend.get(last_leaf).unwrap()); + return_vec.push(self.backend.get(last_leaf, true).unwrap()); return return_vec; } // if size is even, we're already at the bottom, otherwise @@ -390,7 +261,7 @@ where if bintree_postorder_height(last_leaf) > 0 { last_leaf = bintree_rightmost(last_leaf); } - return_vec.push(self.backend.get(last_leaf).unwrap()); + return_vec.push(self.backend.get(last_leaf, true).unwrap()); last_leaf = bintree_jump_left_sibling(last_leaf); } @@ -398,21 +269,20 @@ where } /// Walks all unpruned nodes in the MMR and revalidate all parent hashes - /// and sums. pub fn validate(&self) -> Result<(), String> { // iterate on all parent nodes for n in 1..(self.last_pos + 1) { if bintree_postorder_height(n) > 0 { - if let Some(hs) = self.get(n) { + if let Some(hs) = self.get(n, false) { // take the left and right children, if they exist let left_pos = bintree_move_down_left(n).unwrap(); let right_pos = bintree_jump_right_sibling(left_pos); - if let Some(left_child_hs) = self.get(left_pos) { - if let Some(right_child_hs) = self.get(right_pos) { - // sum and compare - if left_child_hs + right_child_hs != hs { - return Err(format!("Invalid MMR, hashsum of parent at {} does \ + if let Some(left_child_hs) = self.get(left_pos, false) { + if let Some(right_child_hs) = self.get(right_pos, false) { + // add hashes and compare + if left_child_hs.0+right_child_hs.0 != hs.0 { + return Err(format!("Invalid MMR, hash of parent at {} does \ not match children.", n)); } } @@ -429,6 +299,11 @@ where self.last_pos } + /// Return the path of the data file (needed to sum kernels efficiently) + pub fn data_file_path(&self) -> String { + self.backend.get_data_file_path() + } + /// Debugging utility to print information about the MMRs. Short version /// only prints the last 8 nodes. pub fn dump(&self, short: bool) { @@ -445,40 +320,36 @@ where break; } idx.push_str(&format!("{:>8} ", m + 1)); - let ohs = self.get(m + 1); + let ohs = self.get(m + 1, false); match ohs { - Some(hs) => hashes.push_str(&format!("{} ", hs.hash)), + Some(hs) => hashes.push_str(&format!("{} ", hs.0)), None => hashes.push_str(&format!("{:>8} ", "??")), } } - debug!(LOGGER, "{}", idx); - debug!(LOGGER, "{}", hashes); + trace!(LOGGER, "{}", idx); + trace!(LOGGER, "{}", hashes); } } } /// Simple MMR backend implementation based on a Vector. Pruning does not /// compact the Vector itself but still frees the reference to the -/// underlying HashSum. +/// underlying Hash. #[derive(Clone)] pub struct VecBackend -where - T: Summable + Clone, -{ + where T:PMMRable { /// Backend elements - pub elems: Vec>>, + pub elems: Vec)>>, } -impl Backend for VecBackend -where - T: Summable + Clone, -{ +impl Backend for VecBackend + where T: PMMRable { #[allow(unused_variables)] - fn append(&mut self, position: u64, data: Vec>) -> Result<(), String> { + fn append(&mut self, position: u64, data: Vec<(Hash, Option)>) -> Result<(), String> { self.elems.append(&mut map_vec!(data, |d| Some(d.clone()))); Ok(()) } - fn get(&self, position: u64) -> Option> { + fn get(&self, position: u64, _include_data:bool) -> Option<(Hash, Option)> { self.elems[(position - 1) as usize].clone() } #[allow(unused_variables)] @@ -493,18 +364,19 @@ where self.elems = self.elems[0..(position as usize) + 1].to_vec(); Ok(()) } + fn get_data_file_path(&self) -> String { + "".to_string() + } } -impl VecBackend -where - T: Summable + Clone, -{ +impl VecBackend + where T:PMMRable { /// Instantiates a new VecBackend pub fn new() -> VecBackend { VecBackend { elems: vec![] } } - /// Current number of HashSum elements in the underlying Vec. + /// Current number of elements in the underlying Vec. pub fn used_size(&self) -> usize { let mut usz = self.elems.len(); for elem in self.elems.deref() { @@ -568,6 +440,28 @@ impl PruneList { } } + /// As above, but only returning the number of leaf nodes to skip for a + /// given leaf. Helpful if, for instance, data for each leaf is being stored + /// separately in a continuous flat-file + pub fn get_leaf_shift(&self, pos: u64) -> Option { + + // get the position where the node at pos would fit in the pruned list, if + // it's already pruned, nothing to skip + match self.pruned_pos(pos) { + None => None, + Some(idx) => { + // skip by the number of leaf nodes pruned in the preceeding subtrees + // which just 2^height + Some( + self.pruned_nodes[0..(idx as usize)] + .iter() + .map(|n| 1 << bintree_postorder_height(*n)) + .sum(), + ) + } + } + } + /// Push the node at the provided position in the prune list. Compacts the /// list if pruning the additional node means a parent can get pruned as /// well. @@ -591,7 +485,7 @@ impl PruneList { } /// Gets the position a new pruned node should take in the prune list. - /// If the node has already bee pruned, either directly or through one of + /// If the node has already been pruned, either directly or through one of /// its parents contained in the prune list, returns None. pub fn pruned_pos(&self, pos: u64) -> Option { match self.pruned_nodes.binary_search(&pos) { @@ -833,7 +727,20 @@ fn most_significant_pos(num: u64) -> u64 { #[cfg(test)] mod test { use super::*; - use core::hash::Hashed; + use ser::{Writeable, Readable, Error}; + use core::{Writer, Reader}; + use core::hash::{Hash}; + + #[test] + fn test_leaf_index(){ + assert_eq!(n_leaves(1),1); + assert_eq!(n_leaves(2),2); + assert_eq!(n_leaves(4),3); + assert_eq!(n_leaves(5),4); + assert_eq!(n_leaves(8),5); + assert_eq!(n_leaves(9),6); + + } #[test] fn some_all_ones() { @@ -890,23 +797,17 @@ mod test { assert_eq!(peaks(42), vec![31, 38, 41, 42]); } - #[derive(Copy, Clone, Debug, PartialEq, Eq)] + #[derive(Copy, Clone, Debug, PartialEq, Eq)] struct TestElem([u32; 4]); - impl Summable for TestElem { - type Sum = u64; - fn sum(&self) -> u64 { - // sums are not allowed to overflow, so we use this simple - // non-injective "sum" function that will still be homomorphic - self.0[0] as u64 * 0x1000 + self.0[1] as u64 * 0x100 + self.0[2] as u64 * 0x10 + - self.0[3] as u64 - } - fn sum_len() -> usize { - 8 + + impl PMMRable for TestElem { + fn len() -> usize { + 16 } } impl Writeable for TestElem { - fn write(&self, writer: &mut W) -> Result<(), ser::Error> { + fn write(&self, writer: &mut W) -> Result<(), Error> { try!(writer.write_u32(self.0[0])); try!(writer.write_u32(self.0[1])); try!(writer.write_u32(self.0[2])); @@ -914,6 +815,19 @@ mod test { } } + impl Readable for TestElem { + fn read(reader: &mut Reader) -> Result { + Ok(TestElem ( + [ + reader.read_u32()?, + reader.read_u32()?, + reader.read_u32()?, + reader.read_u32()?, + ] + )) + } + } + #[test] #[allow(unused_variables)] fn pmmr_push_root() { @@ -934,72 +848,67 @@ mod test { // one element pmmr.push(elems[0]).unwrap(); - let hash = Hashed::hash(&elems[0]); - let sum = elems[0].sum(); - let node_hash = (1 as u64, &sum, hash).hash(); + let node_hash = elems[0].hash(); assert_eq!( pmmr.root(), - HashSum { - hash: node_hash, - sum: sum, - } + node_hash, ); assert_eq!(pmmr.unpruned_size(), 1); + pmmr.dump(false); // two elements pmmr.push(elems[1]).unwrap(); - let sum2 = HashSum::from_summable(1, &elems[0]) + - HashSum::from_summable(2, &elems[1]); + let sum2 = elems[0].hash() + elems[1].hash(); + pmmr.dump(false); assert_eq!(pmmr.root(), sum2); assert_eq!(pmmr.unpruned_size(), 3); // three elements pmmr.push(elems[2]).unwrap(); - let sum3 = sum2.clone() + HashSum::from_summable(4, &elems[2]); + let sum3 = sum2 + elems[2].hash(); + pmmr.dump(false); assert_eq!(pmmr.root(), sum3); assert_eq!(pmmr.unpruned_size(), 4); // four elements pmmr.push(elems[3]).unwrap(); - let sum4 = sum2 + - (HashSum::from_summable(4, &elems[2]) + - HashSum::from_summable(5, &elems[3])); + let sum_one = elems[2].hash() + elems[3].hash(); + let sum4 = sum2 + sum_one; + pmmr.dump(false); assert_eq!(pmmr.root(), sum4); assert_eq!(pmmr.unpruned_size(), 7); // five elements pmmr.push(elems[4]).unwrap(); - let sum5 = sum4.clone() + HashSum::from_summable(8, &elems[4]); - assert_eq!(pmmr.root(), sum5); + let sum3 = sum4 + elems[4].hash(); + pmmr.dump(false); + assert_eq!(pmmr.root(), sum3); assert_eq!(pmmr.unpruned_size(), 8); // six elements pmmr.push(elems[5]).unwrap(); - let sum6 = sum4.clone() + - (HashSum::from_summable(8, &elems[4]) + - HashSum::from_summable(9, &elems[5])); + let sum6 = sum4 + + (elems[4].hash() + elems[5].hash()); assert_eq!(pmmr.root(), sum6.clone()); assert_eq!(pmmr.unpruned_size(), 10); // seven elements pmmr.push(elems[6]).unwrap(); - let sum7 = sum6 + HashSum::from_summable(11, &elems[6]); + let sum7 = sum6 + elems[6].hash(); assert_eq!(pmmr.root(), sum7); assert_eq!(pmmr.unpruned_size(), 11); // eight elements pmmr.push(elems[7]).unwrap(); let sum8 = sum4 + - ((HashSum::from_summable(8, &elems[4]) + - HashSum::from_summable(9, &elems[5])) + - (HashSum::from_summable(11, &elems[6]) + - HashSum::from_summable(12, &elems[7]))); + ((elems[4].hash() + elems[5].hash()) + + (elems[6].hash() + elems[7].hash())); assert_eq!(pmmr.root(), sum8); assert_eq!(pmmr.unpruned_size(), 15); // nine elements pmmr.push(elems[8]).unwrap(); - let sum9 = sum8 + HashSum::from_summable(16, &elems[8]); + let sum9 = sum8 + elems[8].hash(); assert_eq!(pmmr.root(), sum9); assert_eq!(pmmr.unpruned_size(), 16); } @@ -1015,8 +924,9 @@ mod test { TestElem([0, 0, 0, 6]), TestElem([0, 0, 0, 7]), TestElem([0, 0, 0, 8]), - TestElem([0, 0, 0, 9]), + TestElem([1, 0, 0, 0]), ]; + let mut ba = VecBackend::new(); let mut pmmr = PMMR::new(&mut ba); @@ -1026,23 +936,23 @@ mod test { pmmr.push(elems[0]).unwrap(); let res = pmmr.get_last_n_insertions(19); - assert!(res.len() == 1 && res[0].sum == 1); + assert!(res.len() == 1); pmmr.push(elems[1]).unwrap(); let res = pmmr.get_last_n_insertions(12); - assert!(res[0].sum == 2 && res[1].sum == 1); + assert!(res.len() == 2); pmmr.push(elems[2]).unwrap(); let res = pmmr.get_last_n_insertions(2); - assert!(res[0].sum == 3 && res[1].sum == 2); + assert!(res.len() == 2); pmmr.push(elems[3]).unwrap(); let res = pmmr.get_last_n_insertions(19); assert!( - res[0].sum == 4 && res[1].sum == 3 && res[2].sum == 2 && res[3].sum == 1 && res.len() == 4 + res.len() == 4 ); pmmr.push(elems[5]).unwrap(); @@ -1052,7 +962,7 @@ mod test { let res = pmmr.get_last_n_insertions(7); assert!( - res[0].sum == 9 && res[1].sum == 8 && res[2].sum == 7 && res[3].sum == 6 && res.len() == 7 + res.len() == 7 ); } @@ -1071,7 +981,7 @@ mod test { TestElem([1, 0, 0, 0]), ]; - let orig_root: HashSum; + let orig_root: Hash; let sz: u64; let mut ba = VecBackend::new(); { @@ -1085,7 +995,7 @@ mod test { // pruning a leaf with no parent should do nothing { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); pmmr.prune(16, 0).unwrap(); assert_eq!(orig_root, pmmr.root()); } @@ -1093,14 +1003,14 @@ mod test { // pruning leaves with no shared parent just removes 1 element { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); pmmr.prune(2, 0).unwrap(); assert_eq!(orig_root, pmmr.root()); } assert_eq!(ba.used_size(), 15); { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); pmmr.prune(4, 0).unwrap(); assert_eq!(orig_root, pmmr.root()); } @@ -1108,7 +1018,7 @@ mod test { // pruning a non-leaf node has no effect { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); pmmr.prune(3, 0).unwrap_err(); assert_eq!(orig_root, pmmr.root()); } @@ -1116,7 +1026,7 @@ mod test { // pruning sibling removes subtree { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); pmmr.prune(5, 0).unwrap(); assert_eq!(orig_root, pmmr.root()); } @@ -1124,7 +1034,7 @@ mod test { // pruning all leaves under level >1 removes all subtree { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); pmmr.prune(1, 0).unwrap(); assert_eq!(orig_root, pmmr.root()); } @@ -1132,7 +1042,7 @@ mod test { // pruning everything should only leave us the peaks { - let mut pmmr = PMMR::at(&mut ba, sz); + let mut pmmr:PMMR = PMMR::at(&mut ba, sz); for n in 1..16 { let _ = pmmr.prune(n, 0); } diff --git a/core/src/core/transaction.rs b/core/src/core/transaction.rs index 0c24f1854..5e42f718d 100644 --- a/core/src/core/transaction.rs +++ b/core/src/core/transaction.rs @@ -19,16 +19,14 @@ use util::{static_secp_instance, kernel_sig_msg}; use util::secp::pedersen::{Commitment, RangeProof}; use std::cmp::{min, max}; use std::cmp::Ordering; -use std::ops; use consensus; use consensus::VerifySortOrder; use core::Committed; use core::hash::{Hash, Hashed, ZERO_HASH}; -use core::pmmr::Summable; -use keychain; use keychain::{Identifier, Keychain, BlindingFactor}; -use ser::{self, read_and_verify_sorted, Readable, Reader, Writeable, WriteableSorted, Writer}; +use keychain; +use ser::{self, read_and_verify_sorted, PMMRable, Readable, Reader, Writeable, WriteableSorted, Writer}; use util; /// The size of the blake2 hash of a switch commitment (256 bits) @@ -217,9 +215,10 @@ impl TxKernel { ..self } } +} - /// Size in bytes of a kernel, necessary for binary storage - pub fn size() -> usize { +impl PMMRable for TxKernel { + fn len() -> usize { 17 + // features plus fee and lock_height secp::constants::PEDERSEN_COMMITMENT_SIZE + secp::constants::AGG_SIGNATURE_SIZE @@ -671,17 +670,13 @@ impl SwitchCommitHash { /// provides future-proofing against quantum-based attacks, as well as providing /// wallet implementations with a way to identify their outputs for wallet /// reconstruction. -/// -/// The hash of an output only covers its features, commitment, -/// and switch commitment. The range proof is expected to have its own hash -/// and is stored and committed to separately. #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub struct Output { /// Options for an output's structure or use pub features: OutputFeatures, /// The homomorphic commitment representing the output amount pub commit: Commitment, - /// The switch commitment hash, a 160 bit length blake2 hash of blind*J + /// The switch commitment hash, a 256 bit length blake2 hash of blind*J pub switch_commit_hash: SwitchCommitHash, /// A proof that the commitment is in the right range pub proof: RangeProof, @@ -704,9 +699,13 @@ impl Writeable for Output { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { writer.write_u8(self.features.bits())?; writer.write_fixed_bytes(&self.commit)?; - writer.write_fixed_bytes(&self.switch_commit_hash)?; - - // The hash of an output doesn't include the range proof + // Hash of an output doesn't cover the switch commit, it should + // be wound into the range proof separately + if writer.serialization_mode() != ser::SerializationMode::Hash { + writer.write_fixed_bytes(&self.switch_commit_hash)?; + } + // The hash of an output doesn't include the range proof, which + // is commit to separately if writer.serialization_mode() == ser::SerializationMode::Full { writer.write_bytes(&self.proof)? } @@ -818,21 +817,6 @@ impl OutputIdentifier { util::to_hex(self.commit.0.to_vec()), ) } - - /// Convert an output_indentifier to a sum_commit representation - /// so we can use it to query the the output MMR - pub fn as_sum_commit(&self) -> SumCommit { - SumCommit { - features: self.features, - commit: self.commit, - switch_commit_hash: SwitchCommitHash::zero(), - } - } - - /// Convert a sum_commit back to an output_identifier. - pub fn from_sum_commit(sum_commit: &SumCommit) -> OutputIdentifier { - OutputIdentifier::new(sum_commit.features, &sum_commit.commit) - } } impl Writeable for OutputIdentifier { @@ -855,140 +839,73 @@ impl Readable for OutputIdentifier { } } -/// Wrapper to Output commitments to provide the Summable trait. +/// Yet another output version to read/write from disk. Ends up being far too awkward +/// to use the write serialisation property to do this #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct SumCommit { +pub struct OutputStoreable { /// Output features (coinbase vs. regular transaction output) /// We need to include this when hashing to ensure coinbase maturity can be enforced. pub features: OutputFeatures, /// Output commitment pub commit: Commitment, - /// The corresponding switch commit hash + /// Switch commit hash pub switch_commit_hash: SwitchCommitHash, } -impl SumCommit { - /// Build a new sum_commit. - pub fn new( - features: OutputFeatures, - commit: &Commitment, - switch_commit_hash: &SwitchCommitHash, - ) -> SumCommit { - SumCommit { - features: features.clone(), - commit: commit.clone(), - switch_commit_hash: switch_commit_hash.clone(), - } - } - - /// Build a new sum_commit from an existing output. - pub fn from_output(output: &Output) -> SumCommit { - SumCommit { +impl OutputStoreable { + /// Build a StoreableOutput from an existing output. + pub fn from_output(output: &Output) -> OutputStoreable { + OutputStoreable { features: output.features, commit: output.commit, switch_commit_hash: output.switch_commit_hash, } } - /// Build a new sum_commit from an existing input. - pub fn from_input(input: &Input) -> SumCommit { - SumCommit { - features: input.features, - commit: input.commit, - switch_commit_hash: SwitchCommitHash::zero(), + /// Return a regular output + pub fn to_output(self) -> Output { + Output{ + features: self.features, + commit: self.commit, + switch_commit_hash: self.switch_commit_hash, + proof: RangeProof{ + proof:[0; secp::constants::MAX_PROOF_SIZE], + plen: 0, + }, } } - - /// Hex string representation of a sum_commit. - pub fn to_hex(&self) -> String { - format!( - "{:b}{}{}", - self.features.bits(), - util::to_hex(self.commit.0.to_vec()), - self.switch_commit_hash.to_hex(), - ) - } } -/// Outputs get summed through their commitments. -impl Summable for SumCommit { - type Sum = SumCommit; - - fn sum(&self) -> SumCommit { - SumCommit { - commit: self.commit.clone(), - features: self.features.clone(), - switch_commit_hash: self.switch_commit_hash.clone(), - } - } - - fn sum_len() -> usize { - secp::constants::PEDERSEN_COMMITMENT_SIZE + SWITCH_COMMIT_HASH_SIZE + 1 +impl PMMRable for OutputStoreable { + fn len() -> usize { + 1 + secp::constants::PEDERSEN_COMMITMENT_SIZE + SWITCH_COMMIT_HASH_SIZE } } -impl Writeable for SumCommit { +impl Writeable for OutputStoreable { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { writer.write_u8(self.features.bits())?; self.commit.write(writer)?; - if writer.serialization_mode() == ser::SerializationMode::Full { + if writer.serialization_mode() != ser::SerializationMode::Hash { self.switch_commit_hash.write(writer)?; } Ok(()) } } -impl Readable for SumCommit { - fn read(reader: &mut Reader) -> Result { +impl Readable for OutputStoreable { + fn read(reader: &mut Reader) -> Result { let features = OutputFeatures::from_bits(reader.read_u8()?).ok_or( ser::Error::CorruptedData, )?; - Ok(SumCommit { - features: features, + Ok(OutputStoreable { commit: Commitment::read(reader)?, switch_commit_hash: SwitchCommitHash::read(reader)?, + features: features, }) } } -impl ops::Add for SumCommit { - type Output = SumCommit; - - fn add(self, other: SumCommit) -> SumCommit { - // Build a new commitment by summing the two commitments. - let secp = static_secp_instance(); - let sum = match secp.lock().unwrap().commit_sum( - vec![ - self.commit.clone(), - other.commit.clone(), - ], - vec![], - ) { - Ok(s) => s, - Err(_) => Commitment::from_vec(vec![1; 33]), - }; - - // Now build a new switch_commit_hash by concatenating the two switch_commit_hash value - // and hashing the result. - let mut bytes = self.switch_commit_hash.0.to_vec(); - bytes.extend(other.switch_commit_hash.0.iter().cloned()); - let key = SwitchCommitHashKey::zero(); - let hash = blake2b(SWITCH_COMMIT_HASH_SIZE, &key.0, &bytes); - let hash = hash.as_bytes(); - let mut h = [0; SWITCH_COMMIT_HASH_SIZE]; - for i in 0..SWITCH_COMMIT_HASH_SIZE { - h[i] = hash[i]; - } - let switch_commit_hash_sum = SwitchCommitHash(h); - - SumCommit { - features: self.features | other.features, - commit: sum, - switch_commit_hash: switch_commit_hash_sum, - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/core/src/ser.rs b/core/src/ser.rs index 5f3c662c2..0d7f0bb8c 100644 --- a/core/src/ser.rs +++ b/core/src/ser.rs @@ -37,6 +37,8 @@ use util::secp::constants::{ SECRET_KEY_SIZE, }; +const BULLET_PROOF_SIZE:usize = 674; + /// Possible errors deriving from serializing or deserializing. #[derive(Debug)] pub enum Error { @@ -119,6 +121,9 @@ pub enum SerializationMode { Hash, /// Serialize everything that a signer of the object should know SigHash, + /// Serialize for local storage, for instance in the case where + /// an output doesn't wish to store its range proof + Storage, } /// Implementations defined how different numbers and binary structures are @@ -255,6 +260,7 @@ pub fn ser_vec(thing: &W) -> Result, Error> { Ok(vec) } +/// Utility to read from a binary source struct BinReader<'a> { source: &'a mut Read, } @@ -364,7 +370,7 @@ impl Writeable for RangeProof { impl Readable for RangeProof { fn read(reader: &mut Reader) -> Result { - let p = try!(reader.read_limited_vec(MAX_PROOF_SIZE)); + let p = try!(reader.read_limited_vec(BULLET_PROOF_SIZE)); let mut a = [0; MAX_PROOF_SIZE]; for i in 0..p.len() { a[i] = p[i]; @@ -376,6 +382,12 @@ impl Readable for RangeProof { } } +impl PMMRable for RangeProof { + fn len() -> usize { + BULLET_PROOF_SIZE + } +} + impl Readable for Signature { fn read(reader: &mut Reader) -> Result { let a = try!(reader.read_fixed_bytes(AGG_SIGNATURE_SIZE)); @@ -542,6 +554,12 @@ impl Writeable for [u8; 4] { } } +/// Trait for types that can serialize and report their size +pub trait PMMRable: Readable + Writeable + Hashed + Clone { + /// Length in bytes + fn len() -> usize; +} + /// Useful marker trait on types that can be sized byte slices pub trait AsFixedBytes: Sized + AsRef<[u8]> { /// The length in bytes diff --git a/grin/tests/api.rs b/grin/tests/api.rs index 022a7af09..24dac9543 100644 --- a/grin/tests/api.rs +++ b/grin/tests/api.rs @@ -283,38 +283,38 @@ fn get_utxos_by_height(base_addr: &String, api_server_port: u16, start_height: u // Sumtree handler functions fn get_sumtree_roots(base_addr: &String, api_server_port: u16) -> Result { - let url = format!("http://{}:{}/v1/sumtrees/roots", base_addr, api_server_port); + let url = format!("http://{}:{}/v1/pmmrtrees/roots", base_addr, api_server_port); api::client::get::(url.as_str()).map_err(|e| Error::API(e)) } -fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result, Error> { +fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result, Error> { let url: String; if n == 0 { - url = format!("http://{}:{}/v1/sumtrees/lastutxos", base_addr, api_server_port); + url = format!("http://{}:{}/v1/pmmrtrees/lastutxos", base_addr, api_server_port); } else { - url = format!("http://{}:{}/v1/sumtrees/lastutxos?n={}", base_addr, api_server_port, n); + url = format!("http://{}:{}/v1/pmmrtrees/lastutxos?n={}", base_addr, api_server_port, n); } - api::client::get::>(url.as_str()).map_err(|e| Error::API(e)) + api::client::get::>(url.as_str()).map_err(|e| Error::API(e)) } -fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result, Error> { +fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result, Error> { let url: String; if n == 0 { - url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs", base_addr, api_server_port); + url = format!("http://{}:{}/v1/pmmrtrees/lastrangeproofs", base_addr, api_server_port); } else { - url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs?n={}", base_addr, api_server_port, n); + url = format!("http://{}:{}/v1/pmmrtrees/lastrangeproofs?n={}", base_addr, api_server_port, n); } - api::client::get::>(url.as_str()).map_err(|e| Error::API(e)) + api::client::get::>(url.as_str()).map_err(|e| Error::API(e)) } -fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result, Error> { +fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result, Error> { let url: String; if n == 0 { - url = format!("http://{}:{}/v1/sumtrees/lastkernels", base_addr, api_server_port); + url = format!("http://{}:{}/v1/pmmrtrees/lastkernels", base_addr, api_server_port); } else { - url = format!("http://{}:{}/v1/sumtrees/lastkernels?n={}", base_addr, api_server_port, n); + url = format!("http://{}:{}/v1/pmmrtrees/lastkernels?n={}", base_addr, api_server_port, n); } - api::client::get::>(url.as_str()).map_err(|e| Error::API(e)) + api::client::get::>(url.as_str()).map_err(|e| Error::API(e)) } // Helper function to get a vec of commitment output ids from a vec of block outputs diff --git a/keychain/src/keychain.rs b/keychain/src/keychain.rs index 27e9f210b..39a5e622e 100644 --- a/keychain/src/keychain.rs +++ b/keychain/src/keychain.rs @@ -29,9 +29,9 @@ use blind::{BlindSum, BlindingFactor}; use extkey::{self, Identifier}; #[cfg(feature = "use-bullet-proofs")] -const USE_BULLET_PROOFS:bool = true; +pub const USE_BULLET_PROOFS:bool = true; #[cfg(not(feature = "use-bullet-proofs"))] -const USE_BULLET_PROOFS:bool = false; +pub const USE_BULLET_PROOFS:bool = false; #[derive(PartialEq, Eq, Clone, Debug)] pub enum Error { diff --git a/store/src/lib.rs b/store/src/lib.rs index a3abdce49..745982d48 100644 --- a/store/src/lib.rs +++ b/store/src/lib.rs @@ -30,7 +30,8 @@ extern crate rocksdb; #[macro_use] extern crate slog; -pub mod sumtree; +pub mod pmmr; +pub mod types; const SEP: u8 = ':' as u8; diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs new file mode 100644 index 000000000..df4e0213a --- /dev/null +++ b/store/src/pmmr.rs @@ -0,0 +1,318 @@ +// Copyright 2017 The Grin Developers +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation of the persistent Backend for the prunable MMR tree. + +use std::fs::{self}; +use std::io::{self}; +use std::marker::PhantomData; + +use core::core::pmmr::{self, Backend}; +use core::ser::{self, PMMRable}; +use core::core::hash::Hash; +use util::LOGGER; +use types::{AppendOnlyFile, RemoveLog, read_ordered_vec, write_vec}; + +const PMMR_HASH_FILE: &'static str = "pmmr_hash.bin"; +const PMMR_DATA_FILE: &'static str = "pmmr_data.bin"; +const PMMR_RM_LOG_FILE: &'static str = "pmmr_rm_log.bin"; +const PMMR_PRUNED_FILE: &'static str = "pmmr_pruned.bin"; + +/// Maximum number of nodes in the remove log before it gets flushed +pub const RM_LOG_MAX_NODES: usize = 10000; + +/// PMMR persistent backend implementation. Relies on multiple facilities to +/// handle writing, reading and pruning. +/// +/// * A main storage file appends Hash instances as they come. This +/// AppendOnlyFile is also backed by a mmap for reads. +/// * An in-memory backend buffers the latest batch of writes to ensure the +/// PMMR can always read recent values even if they haven't been flushed to +/// disk yet. +/// * A remove log tracks the positions that need to be pruned from the +/// main storage file. +pub struct PMMRBackend +where + T: PMMRable, +{ + data_dir: String, + hash_file: AppendOnlyFile, + data_file: AppendOnlyFile, + rm_log: RemoveLog, + pruned_nodes: pmmr::PruneList, + phantom: PhantomData, +} + +impl Backend for PMMRBackend +where + T: PMMRable, +{ + /// Append the provided Hashes to the backend storage. + #[allow(unused_variables)] + fn append(&mut self, position: u64, data: Vec<(Hash, Option)>) -> Result<(), String> { + for d in data { + self.hash_file.append(&mut ser::ser_vec(&d.0).unwrap()); + if let Some(elem) = d.1 { + self.data_file.append(&mut ser::ser_vec(&elem).unwrap()); + } + } + Ok(()) + } + + /// Get a Hash by insertion position + fn get(&self, position: u64, include_data:bool) -> Option<(Hash, Option)> { + // Check if this position has been pruned in the remove log or the + // pruned list + if self.rm_log.includes(position) { + return None; + } + let shift = self.pruned_nodes.get_shift(position); + if let None = shift { + return None; + } + + // Read PMMR + // The MMR starts at 1, our binary backend starts at 0 + let pos = position - 1; + + // Must be on disk, doing a read at the correct position + let hash_record_len = 32; + let file_offset = ((pos - shift.unwrap()) as usize) * hash_record_len; + let data = self.hash_file.read(file_offset, hash_record_len); + let hash_val = match ser::deserialize(&mut &data[..]) { + Ok(h) => h, + Err(e) => { + error!( + LOGGER, + "Corrupted storage, could not read an entry from hash store: {:?}", + e + ); + return None; + } + }; + + if !include_data { + return Some(((hash_val), None)); + } + + // Optionally read flatfile storage to get data element + let flatfile_pos = pmmr::n_leaves(position) + - 1 - self.pruned_nodes.get_leaf_shift(position).unwrap(); + let record_len = T::len(); + let file_offset = flatfile_pos as usize * T::len(); + let data = self.data_file.read(file_offset, record_len); + let data = match ser::deserialize(&mut &data[..]) { + Ok(elem) => Some(elem), + Err(e) => { + error!( + LOGGER, + "Corrupted storage, could not read an entry from backend flatfile store: {:?}", + e + ); + None + } + }; + + Some((hash_val, data)) + } + + fn rewind(&mut self, position: u64, index: u32) -> Result<(), String> { + self.rm_log + .rewind(index) + .map_err(|e| format!("Could not truncate remove log: {}", e))?; + + let shift = self.pruned_nodes.get_shift(position).unwrap_or(0); + let record_len = 32; + let file_pos = (position - shift) * (record_len as u64); + self.hash_file.rewind(file_pos); + + //Data file + let flatfile_pos = pmmr::n_leaves(position) - 1; + let file_pos = (flatfile_pos as usize + 1) * T::len(); + self.data_file.rewind(file_pos as u64); + Ok(()) + } + + /// Remove Hashes by insertion position + fn remove(&mut self, positions: Vec, index: u32) -> Result<(), String> { + self.rm_log.append(positions, index).map_err(|e| { + format!("Could not write to log storage, disk full? {:?}", e) + }) + } + + /// Return data file path + fn get_data_file_path(&self) -> String { + self.data_file.path() + } +} + +impl PMMRBackend +where + T: PMMRable, +{ + /// Instantiates a new PMMR backend that will use the provided directly to + /// store its files. + pub fn new(data_dir: String) -> io::Result> { + let hash_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_HASH_FILE))?; + let rm_log = RemoveLog::open(format!("{}/{}", data_dir, PMMR_RM_LOG_FILE))?; + let prune_list = read_ordered_vec(format!("{}/{}", data_dir, PMMR_PRUNED_FILE), 8)?; + let data_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_DATA_FILE))?; + + Ok(PMMRBackend { + data_dir: data_dir, + hash_file: hash_file, + data_file: data_file, + rm_log: rm_log, + pruned_nodes: pmmr::PruneList { + pruned_nodes: prune_list, + }, + phantom: PhantomData, + }) + } + + /// Total size of the PMMR stored by this backend. Only produces the fully + /// sync'd size. + pub fn unpruned_size(&self) -> io::Result { + let total_shift = self.pruned_nodes.get_shift(::std::u64::MAX).unwrap(); + let record_len = 32; + let sz = self.hash_file.size()?; + Ok(sz / record_len + total_shift) + } + + /// Syncs all files to disk. A call to sync is required to ensure all the + /// data has been successfully written to disk. + pub fn sync(&mut self) -> io::Result<()> { + if let Err(e) = self.hash_file.flush() { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + format!("Could not write to log hash storage, disk full? {:?}", e), + )); + } + if let Err(e) = self.data_file.flush() { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + format!("Could not write to log data storage, disk full? {:?}", e), + )); + } + self.rm_log.flush()?; + Ok(()) + } + + /// Discard the current, non synced state of the backend. + pub fn discard(&mut self) { + self.hash_file.discard(); + self.rm_log.discard(); + self.data_file.discard(); + } + + /// Return the data file path + pub fn data_file_path(&self) -> String { + self.get_data_file_path() + } + + /// Checks the length of the remove log to see if it should get compacted. + /// If so, the remove log is flushed into the pruned list, which itself gets + /// saved, and the main hashsum data file is rewritten, cutting the removed + /// data. + /// + /// If a max_len strictly greater than 0 is provided, the value will be used + /// to decide whether the remove log has reached its maximum length, + /// otherwise the RM_LOG_MAX_NODES default value is used. + /// + /// TODO whatever is calling this should also clean up the commit to + /// position index in db + pub fn check_compact(&mut self, max_len: usize) -> io::Result<()> { + if !(max_len > 0 && self.rm_log.len() > max_len + || max_len == 0 && self.rm_log.len() > RM_LOG_MAX_NODES) + { + return Ok(()); + } + + // 0. validate none of the nodes in the rm log are in the prune list (to + // avoid accidental double compaction) + for pos in &self.rm_log.removed[..] { + if let None = self.pruned_nodes.pruned_pos(pos.0) { + // TODO we likely can recover from this by directly jumping to 3 + error!( + LOGGER, + "The remove log contains nodes that are already in the pruned \ + list, a previous compaction likely failed." + ); + return Ok(()); + } + } + + // 1. save hashsum file to a compact copy, skipping data that's in the + // remove list + let tmp_prune_file_hash = format!("{}/{}.hashprune", self.data_dir, PMMR_HASH_FILE); + let record_len = 32; + let to_rm = self.rm_log + .removed + .iter() + .map(|&(pos, _)| { + let shift = self.pruned_nodes.get_shift(pos); + (pos - 1 - shift.unwrap()) * record_len + }) + .collect(); + self.hash_file + .save_prune(tmp_prune_file_hash.clone(), to_rm, record_len)?; + + // 2. And the same with the data file + let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir, PMMR_DATA_FILE); + let record_len = T::len() as u64; + let to_rm = self.rm_log + .removed.clone() + .into_iter() + .filter(|&(pos, _)| pmmr::bintree_postorder_height(pos) == 0) + .map(|(pos, _)| { + let shift = self.pruned_nodes.get_leaf_shift(pos).unwrap(); + let pos = pmmr::n_leaves(pos as u64); + (pos - 1 - shift) * record_len + }) + .collect(); + + self.data_file + .save_prune(tmp_prune_file_data.clone(), to_rm, record_len)?; + + // 3. update the prune list and save it in place + for &(rm_pos, _) in &self.rm_log.removed[..] { + self.pruned_nodes.add(rm_pos); + } + write_vec( + format!("{}/{}", self.data_dir, PMMR_PRUNED_FILE), + &self.pruned_nodes.pruned_nodes, + )?; + + // 4. move the compact copy of hashes to the hashsum file and re-open it + fs::rename( + tmp_prune_file_hash.clone(), + format!("{}/{}", self.data_dir, PMMR_HASH_FILE), + )?; + self.hash_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_HASH_FILE))?; + + // 5. and the same with the data file + fs::rename( + tmp_prune_file_data.clone(), + format!("{}/{}", self.data_dir, PMMR_DATA_FILE), + )?; + self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE))?; + + // 6. truncate the rm log + self.rm_log.rewind(0)?; + self.rm_log.flush()?; + + Ok(()) + } +} + + diff --git a/store/src/sumtree.rs b/store/src/types.rs similarity index 54% rename from store/src/sumtree.rs rename to store/src/types.rs index 2f9368105..f2bbff371 100644 --- a/store/src/sumtree.rs +++ b/store/src/types.rs @@ -1,4 +1,4 @@ -// Copyright 2017 The Grin Developers +// Copyright 2018 The Grin Developers // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -11,33 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Implementation of the persistent Backend for the prunable MMR sum-tree. - +//! Common storage-related types use memmap; use std::cmp; use std::fs::{self, File, OpenOptions}; use std::io::{self, BufRead, BufReader, ErrorKind, Write}; -use std::marker::PhantomData; use std::os::unix::io::AsRawFd; -use std::path::Path; use std::io::Read; +use std::path::Path; #[cfg(any(target_os = "linux"))] use libc::{ftruncate64, off64_t}; #[cfg(not(any(target_os = "linux", target_os = "android")))] use libc::{ftruncate as ftruncate64, off_t as off64_t}; -use core::core::pmmr::{self, Backend, HashSum, Summable}; use core::ser; -use util::LOGGER; - -const PMMR_DATA_FILE: &'static str = "pmmr_dat.bin"; -const PMMR_RM_LOG_FILE: &'static str = "pmmr_rm_log.bin"; -const PMMR_PRUNED_FILE: &'static str = "pmmr_pruned.bin"; - -/// Maximum number of nodes in the remove log before it gets flushed -pub const RM_LOG_MAX_NODES: usize = 10000; /// Wrapper for a file that can be read at any position (random read) but for /// which writes are append only. Reads are backed by a memory map (mmap(2)), @@ -54,6 +43,7 @@ pub struct AppendOnlyFile { buffer_start: usize, buffer: Vec, buffer_start_bak: usize, + unflushed_data_size: usize, } impl AppendOnlyFile { @@ -71,6 +61,7 @@ impl AppendOnlyFile { buffer_start: 0, buffer: vec![], buffer_start_bak: 0, + unflushed_data_size: 0, }; if let Ok(sz) = aof.size() { if sz > 0 { @@ -84,6 +75,7 @@ impl AppendOnlyFile { /// Append data to the file. Until the append-only file is synced, data is /// only written to memory. pub fn append(&mut self, buf: &mut Vec) { + self.unflushed_data_size += buf.len(); self.buffer.append(buf); } @@ -110,6 +102,7 @@ impl AppendOnlyFile { self.file.sync_data()?; self.buffer = vec![]; self.mmap = Some(unsafe { memmap::Mmap::map(&self.file)? }); + self.unflushed_data_size = 0; Ok(()) } @@ -121,11 +114,12 @@ impl AppendOnlyFile { self.buffer_start_bak = 0; } self.buffer = vec![]; + self.unflushed_data_size = 0; } /// Read length bytes of data at offset from the file. Leverages the memory /// map. - fn read(&self, offset: usize, length: usize) -> Vec { + pub fn read(&self, offset: usize, length: usize) -> Vec { if offset >= self.buffer_start { let offset = offset - self.buffer_start; return self.buffer[offset..(offset+length)].to_vec(); @@ -138,7 +132,7 @@ impl AppendOnlyFile { } /// Truncates the underlying file to the provided offset - fn truncate(&self, offs: usize) -> io::Result<()> { + pub fn truncate(&self, offs: usize) -> io::Result<()> { let fd = self.file.as_raw_fd(); let res = unsafe { ftruncate64(fd, offs as off64_t) }; if res == -1 { @@ -150,7 +144,7 @@ impl AppendOnlyFile { /// Saves a copy of the current file content, skipping data at the provided /// prune indices. The prune Vec must be ordered. - fn save_prune(&self, target: String, prune_offs: Vec, prune_len: u64) -> io::Result<()> { + pub fn save_prune(&self, target: String, prune_offs: Vec, prune_len: u64) -> io::Result<()> { let mut reader = File::open(self.path.clone())?; let mut writer = File::create(target)?; @@ -188,10 +182,15 @@ impl AppendOnlyFile { } /// Current size of the file in bytes. - fn size(&self) -> io::Result { + pub fn size(&self) -> io::Result { fs::metadata(&self.path).map(|md| md.len()) } + /// Current size of file in bytes + size of unsaved data + pub fn size_with_unsaved(&self) -> u64 { + self.size().unwrap() + self.unflushed_data_size as u64 + } + /// Path of the underlying file pub fn path(&self) -> String { self.path.clone() @@ -203,10 +202,10 @@ impl AppendOnlyFile { /// checking of whether a piece of data has been marked for deletion. When the /// log becomes too long, the MMR backend will actually remove chunks from the /// MMR data file and truncate the remove log. -struct RemoveLog { +pub struct RemoveLog { path: String, - // Ordered vector of MMR positions that should get eventually removed. - removed: Vec<(u64, u32)>, + /// Ordered vector of MMR positions that should get eventually removed. + pub removed: Vec<(u64, u32)>, // Holds positions temporarily until flush is called. removed_tmp: Vec<(u64, u32)>, // Holds truncated removed temporarily until discarded or committed @@ -216,7 +215,7 @@ struct RemoveLog { impl RemoveLog { /// Open the remove log file. The content of the file will be read in memory /// for fast checking. - fn open(path: String) -> io::Result { + pub fn open(path: String) -> io::Result { let removed = read_ordered_vec(path.clone(), 12)?; Ok(RemoveLog { path: path, @@ -227,7 +226,7 @@ impl RemoveLog { } /// Truncate and empties the remove log. - fn rewind(&mut self, last_offs: u32) -> io::Result<()> { + pub fn rewind(&mut self, last_offs: u32) -> io::Result<()> { // simplifying assumption: we always remove older than what's in tmp self.removed_tmp = vec![]; @@ -251,7 +250,7 @@ impl RemoveLog { /// Append a set of new positions to the remove log. Both adds those /// positions the ordered in-memory set and to the file. - fn append(&mut self, elmts: Vec, index: u32) -> io::Result<()> { + pub fn append(&mut self, elmts: Vec, index: u32) -> io::Result<()> { for elmt in elmts { match self.removed_tmp.binary_search(&(elmt, index)) { Ok(_) => continue, @@ -264,7 +263,7 @@ impl RemoveLog { } /// Flush the positions to remove to file. - fn flush(&mut self) -> io::Result<()> { + pub fn flush(&mut self) -> io::Result<()> { let mut file = File::create(self.path.clone())?; for elmt in &self.removed_tmp { match self.removed.binary_search(&elmt) { @@ -283,7 +282,7 @@ impl RemoveLog { } /// Discard pending changes - fn discard(&mut self) { + pub fn discard(&mut self) { if self.removed_bak.len() > 0 { self.removed = self.removed_bak.clone(); self.removed_bak = vec![]; @@ -292,12 +291,30 @@ impl RemoveLog { } /// Whether the remove log currently includes the provided position. - fn includes(&self, elmt: u64) -> bool { + pub fn includes(&self, elmt: u64) -> bool { include_tuple(&self.removed, elmt) || include_tuple(&self.removed_tmp, elmt) } + /// How many removed positions exist before this particular position + pub fn get_shift(&self, elmt: u64) -> usize { + let mut complete_list = self.removed.clone(); + for e in &self.removed_tmp { + match self.removed.binary_search(&e) { + Ok(_) => continue, + Err(idx) => { + complete_list.insert(idx, *e); + } + } + } + let pos = match complete_list.binary_search(&(elmt,0)){ + Ok(idx) => idx+1, + Err(idx) => idx, + }; + complete_list.split_at(pos).0.len() + } + /// Number of positions stored in the remove log. - fn len(&self) -> usize { + pub fn len(&self) -> usize { self.removed.len() } } @@ -311,216 +328,8 @@ fn include_tuple(v: &Vec<(u64, u32)>, e: u64) -> bool { false } -/// PMMR persistent backend implementation. Relies on multiple facilities to -/// handle writing, reading and pruning. -/// -/// * A main storage file appends HashSum instances as they come. This -/// AppendOnlyFile is also backed by a mmap for reads. -/// * An in-memory backend buffers the latest batch of writes to ensure the -/// PMMR can always read recent values even if they haven't been flushed to -/// disk yet. -/// * A remove log tracks the positions that need to be pruned from the -/// main storage file. -pub struct PMMRBackend -where - T: Summable + Clone, -{ - data_dir: String, - hashsum_file: AppendOnlyFile, - remove_log: RemoveLog, - pruned_nodes: pmmr::PruneList, - phantom: PhantomData, -} - -impl Backend for PMMRBackend -where - T: Summable + Clone, -{ - /// Append the provided HashSums to the backend storage. - #[allow(unused_variables)] - fn append(&mut self, position: u64, data: Vec>) -> Result<(), String> { - for d in data { - self.hashsum_file.append(&mut ser::ser_vec(&d).unwrap()); - } - Ok(()) - } - - /// Get a HashSum by insertion position - fn get(&self, position: u64) -> Option> { - // Check if this position has been pruned in the remove log or the - // pruned list - if self.remove_log.includes(position) { - return None; - } - let shift = self.pruned_nodes.get_shift(position); - if let None = shift { - return None; - } - - // The MMR starts at 1, our binary backend starts at 0 - let pos = position - 1; - - // Must be on disk, doing a read at the correct position - let record_len = 32 + T::sum_len(); - let file_offset = ((pos - shift.unwrap()) as usize) * record_len; - let data = self.hashsum_file.read(file_offset, record_len); - match ser::deserialize(&mut &data[..]) { - Ok(hashsum) => Some(hashsum), - Err(e) => { - error!( - LOGGER, - "Corrupted storage, could not read an entry from sum tree store: {:?}", - e - ); - None - } - } - } - - fn rewind(&mut self, position: u64, index: u32) -> Result<(), String> { - self.remove_log - .rewind(index) - .map_err(|e| format!("Could not truncate remove log: {}", e))?; - - let shift = self.pruned_nodes.get_shift(position).unwrap_or(0); - let record_len = 32 + T::sum_len(); - let file_pos = (position - shift) * (record_len as u64); - self.hashsum_file.rewind(file_pos); - Ok(()) - } - - /// Remove HashSums by insertion position - fn remove(&mut self, positions: Vec, index: u32) -> Result<(), String> { - self.remove_log.append(positions, index).map_err(|e| { - format!("Could not write to log storage, disk full? {:?}", e) - }) - } -} - -impl PMMRBackend -where - T: Summable + Clone, -{ - /// Instantiates a new PMMR backend that will use the provided directly to - /// store its files. - pub fn new(data_dir: String) -> io::Result> { - let hs_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_DATA_FILE))?; - let rm_log = RemoveLog::open(format!("{}/{}", data_dir, PMMR_RM_LOG_FILE))?; - let prune_list = read_ordered_vec(format!("{}/{}", data_dir, PMMR_PRUNED_FILE), 8)?; - - Ok(PMMRBackend { - data_dir: data_dir, - hashsum_file: hs_file, - remove_log: rm_log, - pruned_nodes: pmmr::PruneList { - pruned_nodes: prune_list, - }, - phantom: PhantomData, - }) - } - - /// Total size of the PMMR stored by this backend. Only produces the fully - /// sync'd size. - pub fn unpruned_size(&self) -> io::Result { - let total_shift = self.pruned_nodes.get_shift(::std::u64::MAX).unwrap(); - let record_len = 32 + T::sum_len() as u64; - let sz = self.hashsum_file.size()?; - Ok(sz / record_len + total_shift) - } - - /// Syncs all files to disk. A call to sync is required to ensure all the - /// data has been successfully written to disk. - pub fn sync(&mut self) -> io::Result<()> { - if let Err(e) = self.hashsum_file.flush() { - return Err(io::Error::new( - io::ErrorKind::Interrupted, - format!("Could not write to log storage, disk full? {:?}", e), - )); - } - - self.remove_log.flush()?; - Ok(()) - } - - /// Discard the current, non synced state of the backend. - pub fn discard(&mut self) { - self.hashsum_file.discard(); - self.remove_log.discard(); - } - - /// Checks the length of the remove log to see if it should get compacted. - /// If so, the remove log is flushed into the pruned list, which itself gets - /// saved, and the main hashsum data file is rewritten, cutting the removed - /// data. - /// - /// If a max_len strictly greater than 0 is provided, the value will be used - /// to decide whether the remove log has reached its maximum length, - /// otherwise the RM_LOG_MAX_NODES default value is used. - /// - /// TODO whatever is calling this should also clean up the commit to - /// position index in db - pub fn check_compact(&mut self, max_len: usize) -> io::Result<()> { - if !(max_len > 0 && self.remove_log.len() > max_len - || max_len == 0 && self.remove_log.len() > RM_LOG_MAX_NODES) - { - return Ok(()); - } - - // 0. validate none of the nodes in the rm log are in the prune list (to - // avoid accidental double compaction) - for pos in &self.remove_log.removed[..] { - if let None = self.pruned_nodes.pruned_pos(pos.0) { - // TODO we likely can recover from this by directly jumping to 3 - error!( - LOGGER, - "The remove log contains nodes that are already in the pruned \ - list, a previous compaction likely failed." - ); - return Ok(()); - } - } - - // 1. save hashsum file to a compact copy, skipping data that's in the - // remove list - let tmp_prune_file = format!("{}/{}.prune", self.data_dir, PMMR_DATA_FILE); - let record_len = (32 + T::sum_len()) as u64; - let to_rm = self.remove_log - .removed - .iter() - .map(|&(pos, _)| { - let shift = self.pruned_nodes.get_shift(pos); - (pos - 1 - shift.unwrap()) * record_len - }) - .collect(); - self.hashsum_file - .save_prune(tmp_prune_file.clone(), to_rm, record_len)?; - - // 2. update the prune list and save it in place - for &(rm_pos, _) in &self.remove_log.removed[..] { - self.pruned_nodes.add(rm_pos); - } - write_vec( - format!("{}/{}", self.data_dir, PMMR_PRUNED_FILE), - &self.pruned_nodes.pruned_nodes, - )?; - - // 3. move the compact copy to the hashsum file and re-open it - fs::rename( - tmp_prune_file.clone(), - format!("{}/{}", self.data_dir, PMMR_DATA_FILE), - )?; - self.hashsum_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE))?; - - // 4. truncate the rm log - self.remove_log.rewind(0)?; - self.remove_log.flush()?; - - Ok(()) - } -} - -// Read an ordered vector of scalars from a file. -fn read_ordered_vec(path: String, elmt_len: usize) -> io::Result> +/// Read an ordered vector of scalars from a file. +pub fn read_ordered_vec(path: String, elmt_len: usize) -> io::Result> where T: ser::Readable + cmp::Ord, { @@ -557,7 +366,8 @@ where Ok(ovec) } -fn write_vec(path: String, v: &Vec) -> io::Result<()> +/// Writes an ordered vector to a file +pub fn write_vec(path: String, v: &Vec) -> io::Result<()> where T: ser::Writeable, { diff --git a/store/tests/sumtree.rs b/store/tests/pmmr.rs similarity index 62% rename from store/tests/sumtree.rs rename to store/tests/pmmr.rs index 7b4139cce..00c4becff 100644 --- a/store/tests/sumtree.rs +++ b/store/tests/pmmr.rs @@ -20,13 +20,13 @@ extern crate time; use std::fs; use core::ser::*; -use core::core::pmmr::{Backend, HashSum, Summable, PMMR}; -use core::core::hash::Hashed; +use core::core::pmmr::{PMMR, Backend}; +use core::core::hash::{Hash, Hashed}; #[test] -fn sumtree_append() { +fn pmmr_append() { let (data_dir, elems) = setup("append"); - let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap(); + let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap(); // adding first set of 4 elements and sync let mut mmr_size = load(0, &elems[0..4], &mut backend); @@ -37,31 +37,22 @@ fn sumtree_append() { backend.sync().unwrap(); // check the resulting backend store and the computation of the root - let hash = Hashed::hash(&elems[0].clone()); - let sum = elems[0].sum(); - let node_hash = (1 as u64, &sum, hash).hash(); + let node_hash = elems[0].hash(); assert_eq!( - backend.get(1), - Some(HashSum { - hash: node_hash, - sum: sum, - }) + backend.get(1, false).expect("").0, + node_hash ); - let sum2 = HashSum::from_summable(1, &elems[0]) - + HashSum::from_summable(2, &elems[1]); + let sum2 = elems[0].hash() + elems[1].hash(); let sum4 = sum2 - + (HashSum::from_summable(4, &elems[2]) - + HashSum::from_summable(5, &elems[3])); + + (elems[2].hash() + elems[3].hash()); let sum8 = sum4 - + ((HashSum::from_summable(8, &elems[4]) - + HashSum::from_summable(9, &elems[5])) - + (HashSum::from_summable(11, &elems[6]) - + HashSum::from_summable(12, &elems[7]))); - let sum9 = sum8 + HashSum::from_summable(16, &elems[8]); + + ((elems[4].hash() + elems[5].hash()) + + (elems[6].hash() + elems[7].hash())); + let sum9 = sum8 + elems[8].hash(); { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); assert_eq!(pmmr.root(), sum9); } @@ -69,75 +60,81 @@ fn sumtree_append() { } #[test] -fn sumtree_prune_compact() { +fn pmmr_prune_compact() { let (data_dir, elems) = setup("prune_compact"); // setup the mmr store with all elements - let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap(); + let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap(); let mmr_size = load(0, &elems[..], &mut backend); backend.sync().unwrap(); // save the root - let root: HashSum; + let root: Hash; { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); root = pmmr.root(); } // pruning some choice nodes { - let mut pmmr = PMMR::at(&mut backend, mmr_size); + let mut pmmr:PMMR = PMMR::at(&mut backend, mmr_size); pmmr.prune(1, 1).unwrap(); pmmr.prune(4, 1).unwrap(); pmmr.prune(5, 1).unwrap(); } backend.sync().unwrap(); - // check the root + // check the root and stored data { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); assert_eq!(root, pmmr.root()); + // check we can still retrieve same element from leaf index 2 + assert_eq!(pmmr.get(2, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 2])); } // compact backend.check_compact(2).unwrap(); - // recheck the root + // recheck the root and stored data { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); assert_eq!(root, pmmr.root()); + assert_eq!(pmmr.get(2, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 2])); + assert_eq!(pmmr.get(11, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 7])); } teardown(data_dir); } #[test] -fn sumtree_reload() { +fn pmmr_reload() { let (data_dir, elems) = setup("reload"); // set everything up with a first backend let mmr_size: u64; - let root: HashSum; + let root: Hash; { - let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap(); + let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap(); mmr_size = load(0, &elems[..], &mut backend); backend.sync().unwrap(); // save the root and prune some nodes so we have prune data { - let mut pmmr = PMMR::at(&mut backend, mmr_size); + let mut pmmr:PMMR = PMMR::at(&mut backend, mmr_size); + pmmr.dump(false); root = pmmr.root(); pmmr.prune(1, 1).unwrap(); pmmr.prune(4, 1).unwrap(); } backend.sync().unwrap(); + backend.check_compact(1).unwrap(); backend.sync().unwrap(); assert_eq!(backend.unpruned_size().unwrap(), mmr_size); // prune some more to get rm log data { - let mut pmmr = PMMR::at(&mut backend, mmr_size); + let mut pmmr:PMMR = PMMR::at(&mut backend, mmr_size); pmmr.prune(5, 1).unwrap(); } backend.sync().unwrap(); @@ -146,37 +143,38 @@ fn sumtree_reload() { // create a new backend and check everything is kosher { - let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap(); + let mut backend:store::pmmr::PMMRBackend = + store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap(); assert_eq!(backend.unpruned_size().unwrap(), mmr_size); { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); assert_eq!(root, pmmr.root()); } - assert_eq!(backend.get(5), None); + assert_eq!(backend.get(5, false), None); } teardown(data_dir); } #[test] -fn sumtree_rewind() { +fn pmmr_rewind() { let (data_dir, elems) = setup("rewind"); - let mut backend = store::sumtree::PMMRBackend::new(data_dir.clone()).unwrap(); + let mut backend = store::pmmr::PMMRBackend::new(data_dir.clone()).unwrap(); // adding elements and keeping the corresponding root let mut mmr_size = load(0, &elems[0..4], &mut backend); backend.sync().unwrap(); - let root1: HashSum; + let root1: Hash; { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); root1 = pmmr.root(); } mmr_size = load(mmr_size, &elems[4..6], &mut backend); backend.sync().unwrap(); - let root2: HashSum; + let root2: Hash; { - let pmmr = PMMR::at(&mut backend, mmr_size); + let pmmr:PMMR = PMMR::at(&mut backend, mmr_size); root2 = pmmr.root(); } @@ -185,7 +183,7 @@ fn sumtree_rewind() { // prune and compact the 2 first elements to spice things up { - let mut pmmr = PMMR::at(&mut backend, mmr_size); + let mut pmmr:PMMR = PMMR::at(&mut backend, mmr_size); pmmr.prune(1, 1).unwrap(); pmmr.prune(2, 1).unwrap(); } @@ -194,24 +192,24 @@ fn sumtree_rewind() { // rewind and check the roots still match { - let mut pmmr = PMMR::at(&mut backend, mmr_size); + let mut pmmr:PMMR = PMMR::at(&mut backend, mmr_size); pmmr.rewind(9, 3).unwrap(); assert_eq!(pmmr.root(), root2); } backend.sync().unwrap(); { - let pmmr = PMMR::at(&mut backend, 10); + let pmmr:PMMR = PMMR::at(&mut backend, 10); assert_eq!(pmmr.root(), root2); } { - let mut pmmr = PMMR::at(&mut backend, 10); + let mut pmmr:PMMR = PMMR::at(&mut backend, 10); pmmr.rewind(5, 3).unwrap(); assert_eq!(pmmr.root(), root1); } backend.sync().unwrap(); { - let pmmr = PMMR::at(&mut backend, 7); + let pmmr:PMMR = PMMR::at(&mut backend, 7); assert_eq!(pmmr.root(), root1); } @@ -242,7 +240,7 @@ fn teardown(data_dir: String) { fs::remove_dir_all(data_dir).unwrap(); } -fn load(pos: u64, elems: &[TestElem], backend: &mut store::sumtree::PMMRBackend) -> u64 { +fn load(pos: u64, elems: &[TestElem], backend: &mut store::pmmr::PMMRBackend) -> u64 { let mut pmmr = PMMR::at(backend, pos); for elem in elems { pmmr.push(elem.clone()).unwrap(); @@ -252,16 +250,10 @@ fn load(pos: u64, elems: &[TestElem], backend: &mut store::sumtree::PMMRBackend< #[derive(Copy, Clone, Debug, PartialEq, Eq)] struct TestElem([u32; 4]); -impl Summable for TestElem { - type Sum = u64; - fn sum(&self) -> u64 { - // sums are not allowed to overflow, so we use this simple - // non-injective "sum" function that will still be homomorphic - self.0[0] as u64 * 0x1000 + self.0[1] as u64 * 0x100 + self.0[2] as u64 * 0x10 - + self.0[3] as u64 - } - fn sum_len() -> usize { - 8 + +impl PMMRable for TestElem { + fn len() -> usize { + 16 } } @@ -273,3 +265,15 @@ impl Writeable for TestElem { writer.write_u32(self.0[3]) } } +impl Readable for TestElem { + fn read(reader: &mut Reader) -> Result { + Ok(TestElem ( + [ + reader.read_u32()?, + reader.read_u32()?, + reader.read_u32()?, + reader.read_u32()?, + ] + )) + } +}