Core/grin
0
0
Fork 0
mirror of https://github.com/mimblewimble/grin.git synced 2025-02-01 08:51:08 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Take the 'Sum' out of 'Sumtree' (#702)

Browse source 
* beginning to remove sum

* continuing to remove sumtree sums

* finished removing sums from pmmr core

* renamed sumtree files, and completed changes+test updates in core and store

* updating grin/chain to include removelogs

* integration of flatfile structure, changes to chain/sumtree to start using them

* tests on chain, core and store passing

* cleaning up api and tests

* formatting

* flatfiles stored as part of PMMR backend instead

* all compiling and tests running

* documentation

* added remove + pruning to flatfiles

* remove unneeded enum

* adding sumtree root struct
This commit is contained in:
Yeastplume 2018-02-22 13:45:13 +00:00 committed by GitHub
parent c2ca6ad03f
commit 05d1c6c817
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 893 additions and 879 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
api/src/handlers.rs
View file

@ -206,13 +206,13 @@ impl Handler for UtxoHandler {
}
// Sum tree handler. Retrieve the roots:
// GET /v1/sumtrees/roots
// GET /v1/pmmrtrees/roots
//
// Last inserted nodes::
// GET /v1/sumtrees/lastutxos (gets last 10)
// GET /v1/sumtrees/lastutxos?n=5
// GET /v1/sumtrees/lastrangeproofs
// GET /v1/sumtrees/lastkernels
// GET /v1/pmmrtrees/lastutxos (gets last 10)
// GET /v1/pmmrtrees/lastutxos?n=5
// GET /v1/pmmrtrees/lastrangeproofs
// GET /v1/pmmrtrees/lastkernels
struct SumTreeHandler {
chain: Weak<chain::Chain>,
}
@ -224,18 +224,18 @@ impl SumTreeHandler {
}
// gets last n utxos inserted in to the tree
fn get_last_n_utxo(&self, distance: u64) -> Vec<SumTreeNode> {
SumTreeNode::get_last_n_utxo(w(&self.chain), distance)
fn get_last_n_utxo(&self, distance: u64) -> Vec<PmmrTreeNode> {
PmmrTreeNode::get_last_n_utxo(w(&self.chain), distance)
}
// gets last n utxos inserted in to the tree
fn get_last_n_rangeproof(&self, distance: u64) -> Vec<SumTreeNode> {
SumTreeNode::get_last_n_rangeproof(w(&self.chain), distance)
fn get_last_n_rangeproof(&self, distance: u64) -> Vec<PmmrTreeNode> {
PmmrTreeNode::get_last_n_rangeproof(w(&self.chain), distance)
}
// gets last n utxos inserted in to the tree
fn get_last_n_kernel(&self, distance: u64) -> Vec<SumTreeNode> {
SumTreeNode::get_last_n_kernel(w(&self.chain), distance)
fn get_last_n_kernel(&self, distance: u64) -> Vec<PmmrTreeNode> {
PmmrTreeNode::get_last_n_kernel(w(&self.chain), distance)
}
}
@ -620,10 +620,10 @@ pub fn start_rest_apis<T>(
"get chain".to_string(),
"get chain/utxos".to_string(),
"get status".to_string(),
"get sumtrees/roots".to_string(),
"get sumtrees/lastutxos?n=10".to_string(),
"get sumtrees/lastrangeproofs".to_string(),
"get sumtrees/lastkernels".to_string(),
"get pmmrtrees/roots".to_string(),
"get pmmrtrees/lastutxos?n=10".to_string(),
"get pmmrtrees/lastrangeproofs".to_string(),
"get pmmrtrees/lastkernels".to_string(),
"get pool".to_string(),
"post pool/push".to_string(),
"post peers/a.b.c.d:p/ban".to_string(),
@ -641,7 +641,7 @@ pub fn start_rest_apis<T>(
chain_tip: get "/chain" => chain_tip_handler,
chain_utxos: get "/chain/utxos/*" => utxo_handler,
status: get "/status" => status_handler,
sumtree_roots: get "/sumtrees/*" => sumtree_handler,
sumtree_roots: get "/pmmrtrees/*" => sumtree_handler,
pool_info: get "/pool" => pool_info_handler,
pool_push: post "/pool/push" => pool_push_handler,
peers_all: get "/peers/all" => peers_all_handler,

37
api/src/types.rs
View file

@ -16,7 +16,6 @@ use std::sync::Arc;
use core::{core, ser};
use core::core::hash::Hashed;
use core::core::SumCommit;
use core::core::SwitchCommitHash;
use chain;
use p2p;
@ -89,8 +88,6 @@ impl Status {
pub struct SumTrees {
/// UTXO Root Hash
pub utxo_root_hash: String,
// UTXO Root Sum
pub utxo_root_sum: String,
// Rangeproof root hash
pub range_proof_root_hash: String,
// Kernel set root hash
@ -101,10 +98,9 @@ impl SumTrees {
pub fn from_head(head: Arc<chain::Chain>) -> SumTrees {
let roots = head.get_sumtree_roots();
SumTrees {
utxo_root_hash: roots.0.hash.to_hex(),
utxo_root_sum: roots.0.sum.to_hex(),
range_proof_root_hash: roots.1.hash.to_hex(),
kernel_root_hash: roots.2.hash.to_hex(),
utxo_root_hash: roots.0.to_hex(),
range_proof_root_hash: roots.1.to_hex(),
kernel_root_hash: roots.2.to_hex(),
}
}
}
@ -112,45 +108,40 @@ impl SumTrees {
/// Wrapper around a list of sumtree nodes, so it can be
/// presented properly via json
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct SumTreeNode {
pub struct PmmrTreeNode {
// The hash
pub hash: String,
// SumCommit (features|commitment), optional (only for utxos)
pub sum: Option<SumCommit>,
}
impl SumTreeNode {
pub fn get_last_n_utxo(chain: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
impl PmmrTreeNode {
pub fn get_last_n_utxo(chain: Arc<chain::Chain>, distance: u64) -> Vec<PmmrTreeNode> {
let mut return_vec = Vec::new();
let last_n = chain.get_last_n_utxo(distance);
for x in last_n {
return_vec.push(SumTreeNode {
hash: util::to_hex(x.hash.to_vec()),
sum: Some(x.sum),
return_vec.push(PmmrTreeNode {
hash: util::to_hex(x.0.to_vec()),
});
}
return_vec
}
pub fn get_last_n_rangeproof(head: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
pub fn get_last_n_rangeproof(head: Arc<chain::Chain>, distance: u64) -> Vec<PmmrTreeNode> {
let mut return_vec = Vec::new();
let last_n = head.get_last_n_rangeproof(distance);
for elem in last_n {
return_vec.push(SumTreeNode {
hash: util::to_hex(elem.hash.to_vec()),
sum: None,
return_vec.push(PmmrTreeNode {
hash: util::to_hex(elem.0.to_vec()),
});
}
return_vec
}
pub fn get_last_n_kernel(head: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
pub fn get_last_n_kernel(head: Arc<chain::Chain>, distance: u64) -> Vec<PmmrTreeNode> {
let mut return_vec = Vec::new();
let last_n = head.get_last_n_kernel(distance);
for elem in last_n {
return_vec.push(SumTreeNode {
hash: util::to_hex(elem.hash.to_vec()),
sum: None,
return_vec.push(PmmrTreeNode {
hash: util::to_hex(elem.0.to_vec()),
});
}
return_vec

31
chain/src/chain.rs
View file

@ -20,21 +20,18 @@ use std::fs::File;
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
use util::secp::pedersen::RangeProof;
use core::core::{Input, OutputIdentifier, SumCommit};
use core::core::hash::Hashed;
use core::core::pmmr::{HashSum, NoSum};
use core::core::{Input, OutputIdentifier, OutputStoreable, TxKernel};
use core::core::hash::{Hash, Hashed};
use core::global;
use core::core::{Block, BlockHeader, TxKernel};
use core::core::{Block, BlockHeader};
use core::core::target::Difficulty;
use core::core::hash::Hash;
use grin_store::Error::NotFoundErr;
use pipe;
use store;
use sumtree;
use types::*;
use util::secp::pedersen::RangeProof;
use util::LOGGER;
@ -428,9 +425,9 @@ impl Chain {
Ok(extension.roots())
})?;
b.header.utxo_root = roots.0.hash;
b.header.range_proof_root = roots.1.hash;
b.header.kernel_root = roots.2.hash;
b.header.utxo_root = roots.utxo_root;
b.header.range_proof_root = roots.rproof_root;
b.header.kernel_root = roots.kernel_root;
Ok(())
}
@ -438,9 +435,9 @@ impl Chain {
pub fn get_sumtree_roots(
&self,
) -> (
HashSum<SumCommit>,
HashSum<NoSum<RangeProof>>,
HashSum<NoSum<TxKernel>>,
Hash,
Hash,
Hash,
) {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.roots()
@ -507,7 +504,7 @@ impl Chain {
{
let mut head = self.head.lock().unwrap();
*head = Tip::from_block(&header);
self.store.save_body_head(&head)?;
let _ = self.store.save_body_head(&head);
self.store.save_header_height(&header)?;
}
@ -517,19 +514,19 @@ impl Chain {
}
/// returns the last n nodes inserted into the utxo sum tree
pub fn get_last_n_utxo(&self, distance: u64) -> Vec<HashSum<SumCommit>> {
pub fn get_last_n_utxo(&self, distance: u64) -> Vec<(Hash, Option<OutputStoreable>)> {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.last_n_utxo(distance)
}
/// as above, for rangeproofs
pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<HashSum<NoSum<RangeProof>>> {
pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, Option<RangeProof>)> {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.last_n_rangeproof(distance)
}
/// as above, for kernels
pub fn get_last_n_kernel(&self, distance: u64) -> Vec<HashSum<NoSum<TxKernel>>> {
pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, Option<TxKernel>)> {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.last_n_kernel(distance)
}

12
chain/src/pipe.rs
View file

@ -305,28 +305,28 @@ fn validate_block(
// apply the new block to the MMR trees and check the new root hashes
ext.apply_block(&b)?;
let (utxo_root, rproof_root, kernel_root) = ext.roots();
if utxo_root.hash != b.header.utxo_root || rproof_root.hash != b.header.range_proof_root
|| kernel_root.hash != b.header.kernel_root
let roots = ext.roots();
if roots.utxo_root != b.header.utxo_root || roots.rproof_root != b.header.range_proof_root
|| roots.kernel_root != b.header.kernel_root
{
ext.dump(false);
debug!(
LOGGER,
"validate_block: utxo roots - {:?}, {:?}",
utxo_root.hash,
roots.utxo_root,
b.header.utxo_root,
);
debug!(
LOGGER,
"validate_block: rproof roots - {:?}, {:?}",
rproof_root.hash,
roots.rproof_root,
b.header.range_proof_root,
);
debug!(
LOGGER,
"validate_block: kernel roots - {:?}, {:?}",
kernel_root.hash,
roots.kernel_root,
b.header.kernel_root,
);

254
chain/src/sumtree.rs
View file

@ -1,4 +1,3 @@
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -26,26 +25,26 @@ use util::static_secp_instance;
use util::secp::pedersen::{RangeProof, Commitment};
use core::consensus::reward;
use core::core::{Block, BlockHeader, SumCommit, Input, Output, OutputIdentifier, OutputFeatures, TxKernel};
use core::core::pmmr::{self, HashSum, NoSum, Summable, PMMR};
use core::core::hash::Hashed;
use core::ser;
use core::core::{Block, BlockHeader, Input, Output, OutputIdentifier,
OutputFeatures, OutputStoreable, TxKernel};
use core::core::pmmr::{self, PMMR};
use core::core::hash::{Hash, Hashed};
use core::ser::{self, PMMRable};
use grin_store;
use grin_store::sumtree::{PMMRBackend, AppendOnlyFile};
use types::ChainStore;
use types::Error;
use grin_store::pmmr::PMMRBackend;
use types::{ChainStore, SumTreeRoots, Error};
use util::{LOGGER, zip};
const SUMTREES_SUBDIR: &'static str = "sumtrees";
const UTXO_SUBDIR: &'static str = "utxo";
const RANGE_PROOF_SUBDIR: &'static str = "rangeproof";
const KERNEL_SUBDIR: &'static str = "kernel";
const KERNEL_FILE: &'static str = "kernel_full_data.bin";
const SUMTREES_ZIP: &'static str = "sumtrees_snapshot.zip";
struct PMMRHandle<T>
where
T: Summable + Clone,
T: PMMRable,
{
backend: PMMRBackend<T>,
last_pos: u64,
@ -53,7 +52,7 @@ where
impl<T> PMMRHandle<T>
where
T: Summable + Clone,
T: PMMRable,
{
fn new(root_dir: String, file_name: &str) -> Result<PMMRHandle<T>, Error> {
let path = Path::new(&root_dir).join(SUMTREES_SUBDIR).join(file_name);
@ -70,20 +69,17 @@ where
/// An easy to manipulate structure holding the 3 sum trees necessary to
/// validate blocks and capturing the UTXO set, the range proofs and the
/// kernels. Also handles the index of Commitments to positions in the
/// output and range proof sum trees.
/// output and range proof pmmr trees.
///
/// Note that the index is never authoritative, only the trees are
/// guaranteed to indicate whether an output is spent or not. The index
/// may have commitments that have already been spent, even with
/// pruning enabled.
///
/// In addition of the sumtrees, this maintains the full list of kernel
/// data so it can be easily packaged for sync or validation.
pub struct SumTrees {
output_pmmr_h: PMMRHandle<SumCommit>,
rproof_pmmr_h: PMMRHandle<NoSum<RangeProof>>,
kernel_pmmr_h: PMMRHandle<NoSum<TxKernel>>,
kernel_file: AppendOnlyFile,
utxo_pmmr_h: PMMRHandle<OutputStoreable>,
rproof_pmmr_h: PMMRHandle<RangeProof>,
kernel_pmmr_h: PMMRHandle<TxKernel>,
// chain store used as index of commitments to MMR positions
commit_index: Arc<ChainStore>,
@ -92,16 +88,20 @@ pub struct SumTrees {
impl SumTrees {
/// Open an existing or new set of backends for the SumTrees
pub fn open(root_dir: String, commit_index: Arc<ChainStore>) -> Result<SumTrees, Error> {
let mut kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect();
let utxo_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, UTXO_SUBDIR].iter().collect();
fs::create_dir_all(utxo_file_path.clone())?;
let rproof_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, RANGE_PROOF_SUBDIR].iter().collect();
fs::create_dir_all(rproof_file_path.clone())?;
let kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect();
fs::create_dir_all(kernel_file_path.clone())?;
kernel_file_path.push(KERNEL_FILE);
let kernel_file = AppendOnlyFile::open(kernel_file_path.to_str().unwrap().to_owned())?;
Ok(SumTrees {
output_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?,
utxo_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?,
rproof_pmmr_h: PMMRHandle::new(root_dir.clone(), RANGE_PROOF_SUBDIR)?,
kernel_pmmr_h: PMMRHandle::new(root_dir.clone(), KERNEL_SUBDIR)?,
kernel_file: kernel_file,
commit_index: commit_index,
})
}
@ -109,19 +109,19 @@ impl SumTrees {
/// Check is an output is unspent.
/// We look in the index to find the output MMR pos.
/// Then we check the entry in the output MMR and confirm the hash matches.
pub fn is_unspent(&mut self, output: &OutputIdentifier) -> Result<(), Error> {
match self.commit_index.get_output_pos(&output.commit) {
pub fn is_unspent(&mut self, output_id: &OutputIdentifier) -> Result<(), Error> {
match self.commit_index.get_output_pos(&output_id.commit) {
Ok(pos) => {
let output_pmmr = PMMR::at(
&mut self.output_pmmr_h.backend,
self.output_pmmr_h.last_pos,
let output_pmmr:PMMR<OutputStoreable, _> = PMMR::at(
&mut self.utxo_pmmr_h.backend,
self.utxo_pmmr_h.last_pos,
);
if let Some(HashSum { hash, sum: _ }) = output_pmmr.get(pos) {
let sum_commit = output.as_sum_commit();
let hash_sum = HashSum::from_summable(pos, &sum_commit);
if hash == hash_sum.hash {
if let Some((hash, _)) = output_pmmr.get(pos, false) {
println!("Getting output ID hash");
if hash == output_id.hash() {
Ok(())
} else {
println!("MISMATCH BECAUSE THE BLOODY THING MISMATCHES");
Err(Error::SumTreeErr(format!("sumtree hash mismatch")))
}
} else {
@ -164,20 +164,21 @@ impl SumTrees {
/// returns the last N nodes inserted into the tree (i.e. the 'bottom'
/// nodes at level 0
pub fn last_n_utxo(&mut self, distance: u64) -> Vec<HashSum<SumCommit>> {
let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
output_pmmr.get_last_n_insertions(distance)
/// TODO: These need to return the actual data from the flat-files instead of hashes now
pub fn last_n_utxo(&mut self, distance: u64) -> Vec<(Hash, Option<OutputStoreable>)> {
let utxo_pmmr:PMMR<OutputStoreable, _> = PMMR::at(&mut self.utxo_pmmr_h.backend, self.utxo_pmmr_h.last_pos);
utxo_pmmr.get_last_n_insertions(distance)
}
/// as above, for range proofs
pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<HashSum<NoSum<RangeProof>>> {
let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<(Hash, Option<RangeProof>)> {
let rproof_pmmr:PMMR<RangeProof, _> = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
rproof_pmmr.get_last_n_insertions(distance)
}
/// as above, for kernels
pub fn last_n_kernel(&mut self, distance: u64) -> Vec<HashSum<NoSum<TxKernel>>> {
let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
pub fn last_n_kernel(&mut self, distance: u64) -> Vec<(Hash, Option<TxKernel>)> {
let kernel_pmmr:PMMR<TxKernel, _> = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
kernel_pmmr.get_last_n_insertions(distance)
}
@ -186,17 +187,19 @@ impl SumTrees {
indexes_at(block, self.commit_index.deref())
}
/// Get sum tree roots
/// TODO: Return data instead of hashes
pub fn roots(
&mut self,
) -> (
HashSum<SumCommit>,
HashSum<NoSum<RangeProof>>,
HashSum<NoSum<TxKernel>>,
Hash,
Hash,
Hash,
) {
let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
let output_pmmr:PMMR<OutputStoreable, _> = PMMR::at(&mut self.utxo_pmmr_h.backend, self.utxo_pmmr_h.last_pos);
let rproof_pmmr:PMMR<RangeProof, _> = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
let kernel_pmmr:PMMR<TxKernel, _> = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
(output_pmmr.root(), rproof_pmmr.root(), kernel_pmmr.root())
}
}
@ -231,26 +234,23 @@ where
match res {
Err(e) => {
debug!(LOGGER, "Error returned, discarding sumtree extension.");
trees.output_pmmr_h.backend.discard();
trees.utxo_pmmr_h.backend.discard();
trees.rproof_pmmr_h.backend.discard();
trees.kernel_pmmr_h.backend.discard();
trees.kernel_file.discard();
Err(e)
}
Ok(r) => {
if rollback {
debug!(LOGGER, "Rollbacking sumtree extension.");
trees.output_pmmr_h.backend.discard();
trees.utxo_pmmr_h.backend.discard();
trees.rproof_pmmr_h.backend.discard();
trees.kernel_pmmr_h.backend.discard();
trees.kernel_file.discard();
} else {
debug!(LOGGER, "Committing sumtree extension.");
trees.output_pmmr_h.backend.sync()?;
trees.utxo_pmmr_h.backend.sync()?;
trees.rproof_pmmr_h.backend.sync()?;
trees.kernel_pmmr_h.backend.sync()?;
trees.kernel_file.flush()?;
trees.output_pmmr_h.last_pos = sizes.0;
trees.utxo_pmmr_h.last_pos = sizes.0;
trees.rproof_pmmr_h.last_pos = sizes.1;
trees.kernel_pmmr_h.last_pos = sizes.2;
}
@ -265,11 +265,10 @@ where
/// reversible manner within a unit of work provided by the `extending`
/// function.
pub struct Extension<'a> {
output_pmmr: PMMR<'a, SumCommit, PMMRBackend<SumCommit>>,
rproof_pmmr: PMMR<'a, NoSum<RangeProof>, PMMRBackend<NoSum<RangeProof>>>,
kernel_pmmr: PMMR<'a, NoSum<TxKernel>, PMMRBackend<NoSum<TxKernel>>>,
utxo_pmmr: PMMR<'a, OutputStoreable, PMMRBackend<OutputStoreable>>,
rproof_pmmr: PMMR<'a, RangeProof, PMMRBackend<RangeProof>>,
kernel_pmmr: PMMR<'a, TxKernel, PMMRBackend<TxKernel>>,
kernel_file: &'a mut AppendOnlyFile,
commit_index: Arc<ChainStore>,
new_output_commits: HashMap<Commitment, u64>,
new_kernel_excesses: HashMap<Commitment, u64>,
@ -284,9 +283,9 @@ impl<'a> Extension<'a> {
) -> Extension<'a> {
Extension {
output_pmmr: PMMR::at(
&mut trees.output_pmmr_h.backend,
trees.output_pmmr_h.last_pos,
utxo_pmmr: PMMR::at(
&mut trees.utxo_pmmr_h.backend,
trees.utxo_pmmr_h.last_pos,
),
rproof_pmmr: PMMR::at(
&mut trees.rproof_pmmr_h.backend,
@ -296,10 +295,10 @@ impl<'a> Extension<'a> {
&mut trees.kernel_pmmr_h.backend,
trees.kernel_pmmr_h.last_pos,
),
kernel_file: &mut trees.kernel_file,
commit_index: commit_index,
new_output_commits: HashMap::new(),
new_kernel_excesses: HashMap::new(),
rollback: false,
}
}
@ -354,15 +353,14 @@ impl<'a> Extension<'a> {
fn apply_input(&mut self, input: &Input, height: u64) -> Result<(), Error> {
let commit = input.commitment();
let pos_res = self.get_output_pos(&commit);
let output_id_hash = OutputIdentifier::from_input(input).hash();
if let Ok(pos) = pos_res {
if let Some(HashSum { hash, sum: _ }) = self.output_pmmr.get(pos) {
let sum_commit = SumCommit::from_input(&input);
// check hash from pmmr matches hash from input
if let Some((read_hash, read_elem)) = self.utxo_pmmr.get(pos, true) {
// check hash from pmmr matches hash from input (or corresponding output)
// if not then the input is not being honest about
// what it is attempting to spend...
let hash_sum = HashSum::from_summable(pos, &sum_commit);
if hash != hash_sum.hash {
if output_id_hash != read_hash ||
output_id_hash != read_elem.expect("no output at position").hash() {
return Err(Error::SumTreeErr(format!("output pmmr hash mismatch")));
}
@ -379,9 +377,10 @@ impl<'a> Extension<'a> {
}
}
// Now prune the output_pmmr and rproof_pmmr.
// Now prune the utxo_pmmr, rproof_pmmr and their storage.
// Input is not valid if we cannot prune successfully (to spend an unspent output).
match self.output_pmmr.prune(pos, height as u32) {
// TODO: rm log, skip list for utxo and range proofs
match self.utxo_pmmr.prune(pos, height as u32) {
Ok(true) => {
self.rproof_pmmr
.prune(pos, height as u32)
@ -398,7 +397,6 @@ impl<'a> Extension<'a> {
fn apply_output(&mut self, out: &Output) -> Result<(), Error> {
let commit = out.commitment();
let sum_commit = SumCommit::from_output(out);
if let Ok(pos) = self.get_output_pos(&commit) {
// we need to check whether the commitment is in the current MMR view
@ -406,27 +404,24 @@ impl<'a> Extension<'a> {
// (non-historical node will have a much smaller one)
// note that this doesn't show the commitment *never* existed, just
// that this is not an existing unspent commitment right now
if let Some(c) = self.output_pmmr.get(pos) {
let hash_sum = HashSum::from_summable(pos, &sum_commit);
if let Some((hash, _)) = self.utxo_pmmr.get(pos, false) {
// processing a new fork so we may get a position on the old
// fork that exists but matches a different node
// filtering that case out
if c.hash == hash_sum.hash {
if hash == OutputStoreable::from_output(out).hash() {
return Err(Error::DuplicateCommitment(commit));
}
}
}
// push new outputs commitments in their MMR and save them in the index
let pos = self.output_pmmr
.push(sum_commit)
// push new outputs in their MMR and save them in the index
let pos = self.utxo_pmmr
.push(OutputStoreable::from_output(out))
.map_err(&Error::SumTreeErr)?;
self.new_output_commits.insert(out.commitment(), pos);
// push range proofs in their MMR
// push range proofs in their MMR and file
self.rproof_pmmr
.push(NoSum(out.proof))
.push(out.proof)
.map_err(&Error::SumTreeErr)?;
Ok(())
}
@ -434,9 +429,8 @@ impl<'a> Extension<'a> {
fn apply_kernel(&mut self, kernel: &TxKernel) -> Result<(), Error> {
if let Ok(pos) = self.get_kernel_pos(&kernel.excess) {
// same as outputs
if let Some(k) = self.kernel_pmmr.get(pos) {
let hashsum = HashSum::from_summable(pos, &NoSum(kernel));
if k.hash == hashsum.hash {
if let Some((h,_)) = self.kernel_pmmr.get(pos, false) {
if h == kernel.hash() {
return Err(Error::DuplicateKernel(kernel.excess.clone()));
}
}
@ -444,10 +438,9 @@ impl<'a> Extension<'a> {
// push kernels in their MMR and file
let pos = self.kernel_pmmr
.push(NoSum(kernel.clone()))
.push(kernel.clone())
.map_err(&Error::SumTreeErr)?;
self.new_kernel_excesses.insert(kernel.excess, pos);
self.kernel_file.append(&mut ser::ser_vec(&kernel).unwrap());
Ok(())
}
@ -465,13 +458,6 @@ impl<'a> Extension<'a> {
// rewind each MMR
let (out_pos_rew, kern_pos_rew) = indexes_at(block, self.commit_index.deref())?;
self.rewind_pos(block.header.height, out_pos_rew, kern_pos_rew)?;
// rewind the kernel file store, the position is the number of kernels
// multiplied by their size
// the number of kernels is the number of leaves in the MMR
let pos = pmmr::n_leaves(kern_pos_rew);
self.kernel_file.rewind(pos * (TxKernel::size() as u64));
Ok(())
}
@ -485,7 +471,7 @@ impl<'a> Extension<'a> {
kern_pos_rew,
);
self.output_pmmr
self.utxo_pmmr
.rewind(out_pos_rew, height as u32)
.map_err(&Error::SumTreeErr)?;
self.rproof_pmmr
@ -495,7 +481,6 @@ impl<'a> Extension<'a> {
.rewind(kern_pos_rew, height as u32)
.map_err(&Error::SumTreeErr)?;
self.dump(true);
Ok(())
}
@ -515,26 +500,23 @@ impl<'a> Extension<'a> {
}
}
/// Current root hashes and sums (if applicable) for the UTXO, range proof
/// and kernel sum trees.
pub fn roots(
&self,
) -> (
HashSum<SumCommit>,
HashSum<NoSum<RangeProof>>,
HashSum<NoSum<TxKernel>>,
) {
(
self.output_pmmr.root(),
self.rproof_pmmr.root(),
self.kernel_pmmr.root(),
)
) -> SumTreeRoots {
SumTreeRoots {
utxo_root: self.utxo_pmmr.root(),
rproof_root: self.rproof_pmmr.root(),
kernel_root: self.kernel_pmmr.root(),
}
}
/// Validate the current sumtree state against a block header
pub fn validate(&self, header: &BlockHeader) -> Result<(), Error> {
// validate all hashes and sums within the trees
if let Err(e) = self.output_pmmr.validate() {
if let Err(e) = self.utxo_pmmr.validate() {
return Err(Error::InvalidSumtree(e));
}
if let Err(e) = self.rproof_pmmr.validate() {
@ -545,9 +527,9 @@ impl<'a> Extension<'a> {
}
// validate the tree roots against the block header
let (utxo_root, rproof_root, kernel_root) = self.roots();
if utxo_root.hash != header.utxo_root || rproof_root.hash != header.range_proof_root
|| kernel_root.hash != header.kernel_root
let roots = self.roots();
if roots.utxo_root != header.utxo_root || roots.rproof_root != header.range_proof_root
|| roots.kernel_root != header.kernel_root
{
return Err(Error::InvalidRoot);
}
@ -574,11 +556,11 @@ impl<'a> Extension<'a> {
/// by iterating over the whole MMR data. This is a costly operation
/// performed only when we receive a full new chain state.
pub fn rebuild_index(&self) -> Result<(), Error> {
for n in 1..self.output_pmmr.unpruned_size()+1 {
for n in 1..self.utxo_pmmr.unpruned_size()+1 {
// non-pruned leaves only
if pmmr::bintree_postorder_height(n) == 0 {
if let Some(hs) = self.output_pmmr.get(n) {
self.commit_index.save_output_pos(&hs.sum.commit, n)?;
if let Some((_, out)) = self.utxo_pmmr.get(n, true) {
self.commit_index.save_output_pos(&out.expect("not a leaf node").commit, n)?;
}
}
}
@ -594,7 +576,7 @@ impl<'a> Extension<'a> {
/// version only prints the UTXO tree.
pub fn dump(&self, short: bool) {
debug!(LOGGER, "-- outputs --");
self.output_pmmr.dump(short);
self.utxo_pmmr.dump(short);
if !short {
debug!(LOGGER, "-- range proofs --");
self.rproof_pmmr.dump(short);
@ -606,7 +588,7 @@ impl<'a> Extension<'a> {
// Sizes of the sum trees, used by `extending` on rollback.
fn sizes(&self) -> (u64, u64, u64) {
(
self.output_pmmr.unpruned_size(),
self.utxo_pmmr.unpruned_size(),
self.rproof_pmmr.unpruned_size(),
self.kernel_pmmr.unpruned_size(),
)
@ -619,7 +601,7 @@ impl<'a> Extension<'a> {
let mmr_sz = self.kernel_pmmr.unpruned_size();
let count = pmmr::n_leaves(mmr_sz);
let mut kernel_file = File::open(self.kernel_file.path())?;
let mut kernel_file = File::open(self.kernel_pmmr.data_file_path())?;
let first: TxKernel = ser::deserialize(&mut kernel_file)?;
first.verify()?;
let mut sum_kernel = first.excess;
@ -651,14 +633,14 @@ impl<'a> Extension<'a> {
let mut sum_utxo = None;
let mut utxo_count = 0;
let secp = static_secp_instance();
for n in 1..self.output_pmmr.unpruned_size()+1 {
for n in 1..self.utxo_pmmr.unpruned_size()+1 {
if pmmr::bintree_postorder_height(n) == 0 {
if let Some(hs) = self.output_pmmr.get(n) {
if let Some((_,output)) = self.utxo_pmmr.get(n, true) {
if n == 1 {
sum_utxo = Some(hs.sum.commit);
sum_utxo = Some(output.expect("not a leaf node").commit);
} else {
let secp = secp.lock().unwrap();
sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), hs.sum.commit], vec![])?);
sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), output.expect("not a leaf node").commit], vec![])?);
}
utxo_count += 1;
}
@ -669,6 +651,42 @@ impl<'a> Extension<'a> {
}
}
/*fn store_element<T>(file_store: &mut FlatFileStore<T>, data: T)
-> Result<(), String>
where
T: ser::Readable + ser::Writeable + Clone
{
file_store.append(vec![data])
}
fn read_element_at_pmmr_index<T>(file_store: &FlatFileStore<T>, pos: u64) -> Option<T>
where
T: ser::Readable + ser::Writeable + Clone
{
let leaf_index = pmmr::leaf_index(pos);
// flat files are zero indexed
file_store.get(leaf_index - 1)
}
fn _remove_element_at_pmmr_index<T>(file_store: &mut FlatFileStore<T>, pos: u64)
-> Result<(), String>
where
T: ser::Readable + ser::Writeable + Clone
{
let leaf_index = pmmr::leaf_index(pos);
// flat files are zero indexed
file_store.remove(vec![leaf_index - 1])
}
fn rewind_to_pmmr_index<T>(file_store: &mut FlatFileStore<T>, pos: u64) -> Result<(), String>
where
T: ser::Readable + ser::Writeable + Clone
{
let leaf_index = pmmr::leaf_index(pos);
// flat files are zero indexed
file_store.rewind(leaf_index - 1)
}*/
/// Output and kernel MMR indexes at the end of the provided block
fn indexes_at(block: &Block, commit_index: &ChainStore) -> Result<(u64, u64), Error> {
let out_idx = match block.outputs.last() {

13
chain/src/types.rs
View file

@ -40,6 +40,17 @@ bitflags! {
}
}
/// A helper to hold the roots of the sumtrees in order to keep them
/// readable
pub struct SumTreeRoots {
/// UTXO root
pub utxo_root: Hash,
/// Range Proof root
pub rproof_root: Hash,
/// Kernel root
pub kernel_root: Hash,
}
/// Errors
#[derive(Debug)]
pub enum Error {
@ -81,6 +92,8 @@ pub enum Error {
InvalidSumtree(String),
/// Internal issue when trying to save or load data from store
StoreErr(grin_store::Error, String),
/// Internal issue when trying to save or load data from append only files
FileReadErr(String),
/// Error serializing or deserializing a type
SerErr(ser::Error),
/// Error with the sumtrees

11
chain/tests/mine_simple_chain.rs
View file

@ -253,6 +253,8 @@ fn spend_in_fork() {
fork_head = b.header.clone();
chain.process_block(b, chain::Options::SKIP_POW).unwrap();
println!("First block");
// now mine three further blocks
for n in 3..6 {
let b = prepare_block(&kc, &fork_head, &chain, n);
@ -263,6 +265,8 @@ fn spend_in_fork() {
let lock_height = 1 + global::coinbase_maturity();
assert_eq!(lock_height, 4);
println!("3 Further Blocks: should have 4 blocks or 264 bytes in file ");
let tx1 = build::transaction(
vec![
build::coinbase_input(consensus::REWARD, block_hash, kc.derive_key_id(2).unwrap()),
@ -272,10 +276,14 @@ fn spend_in_fork() {
&kc,
).unwrap();
println!("Built coinbase input and output");
let next = prepare_block_tx(&kc, &fork_head, &chain, 7, vec![&tx1]);
let prev_main = next.header.clone();
chain.process_block(next.clone(), chain::Options::SKIP_POW).unwrap();
println!("tx 1 processed, should have 6 outputs or 396 bytes in file, first skipped");
let tx2 = build::transaction(
vec![
build::input(consensus::REWARD - 20000, next.hash(), kc.derive_key_id(30).unwrap()),
@ -289,6 +297,9 @@ fn spend_in_fork() {
let prev_main = next.header.clone();
chain.process_block(next, chain::Options::SKIP_POW).unwrap();
println!("tx 2 processed");
/*panic!("Stop");*/
// mine 2 forked blocks from the first
let fork = prepare_fork_block_tx(&kc, &fork_head, &chain, 6, vec![&tx1]);
let prev_fork = fork.header.clone();

10
core/src/core/hash.rs
View file

@ -20,13 +20,13 @@
use std::cmp::min;
use std::{fmt, ops};
use std::convert::AsRef;
use std::ops::Add;
use blake2::blake2b::Blake2b;
use consensus;
use ser::{self, AsFixedBytes, Error, Readable, Reader, Writeable, Writer};
use util;
use util::LOGGER;
/// A hash consisting of all zeroes, used as a sentinel. No known preimage.
pub const ZERO_HASH: Hash = Hash([0; 32]);
@ -147,6 +147,13 @@ impl Writeable for Hash {
}
}
impl Add for Hash {
type Output = Hash;
fn add(self, other: Hash) -> Hash {
self.hash_with(other)
}
}
/// Serializer that outputs a hash of the serialized object
pub struct HashWriter {
state: Blake2b,
@ -205,7 +212,6 @@ impl<W: ser::Writeable> Hashed for W {
fn hash_with<T: Writeable>(&self, other: T) -> Hash {
let mut hasher = HashWriter::default();
ser::Writeable::write(self, &mut hasher).unwrap();
trace!(LOGGER, "Hashing with additional data");
ser::Writeable::write(&other, &mut hasher).unwrap();
let mut ret = [0; 32];
hasher.finalize(&mut ret);

424
core/src/core/pmmr.rs
View file

@ -29,169 +29,30 @@
//! position of siblings, parents, etc. As all those functions only rely on
//! binary operations, they're extremely fast. For more information, see the
//! doc on bintree_jump_left_sibling.
//! 2. The implementation of a prunable MMR sum tree using the above. Each leaf
//! is required to be Summable and Hashed. Tree roots can be trivially and
//! 2. The implementation of a prunable MMR tree using the above. Each leaf
//! is required to be Writeable (which implements Hashed). Tree roots can be trivially and
//! efficiently calculated without materializing the full tree. The underlying
//! (Hash, Sum) pais are stored in a Backend implementation that can either be
//! Hashes are stored in a Backend implementation that can either be
//! a simple Vec or a database.
use std::clone::Clone;
use std::ops::Deref;
use std::marker::PhantomData;
use std::ops::{self, Deref};
use core::hash::{Hash, Hashed};
use ser::{self, Readable, Reader, Writeable, Writer};
use ser::PMMRable;
use util::LOGGER;
/// Trait for an element of the tree that has a well-defined sum and hash that
/// the tree can sum over
pub trait Summable {
/// The type of the sum
type Sum: Clone + ops::Add<Output = Self::Sum> + Readable + Writeable + PartialEq;
/// Obtain the sum of the element
fn sum(&self) -> Self::Sum;
/// Length of the Sum type when serialized. Can be used as a hint by
/// underlying storages.
fn sum_len() -> usize;
}
/// An empty sum that takes no space, to store elements that do not need summing
/// but can still leverage the hierarchical hashing.
#[derive(Copy, Clone, Debug)]
pub struct NullSum;
impl ops::Add for NullSum {
type Output = NullSum;
fn add(self, _: NullSum) -> NullSum {
NullSum
}
}
impl Readable for NullSum {
fn read(_: &mut Reader) -> Result<NullSum, ser::Error> {
Ok(NullSum)
}
}
impl Writeable for NullSum {
fn write<W: Writer>(&self, _: &mut W) -> Result<(), ser::Error> {
Ok(())
}
}
impl PartialEq for NullSum {
fn eq(&self, _other: &NullSum) -> bool {
true
}
}
/// Wrapper for a type that allows it to be inserted in a tree without summing
#[derive(Clone, Debug)]
pub struct NoSum<T>(pub T);
impl<T> Summable for NoSum<T> {
type Sum = NullSum;
fn sum(&self) -> NullSum {
NullSum
}
fn sum_len() -> usize {
return 0;
}
}
impl<T> Writeable for NoSum<T>
where
T: Writeable,
{
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
self.0.write(writer)
}
}
/// A utility type to handle (Hash, Sum) pairs more conveniently. The addition
/// of two HashSums is the (Hash(h1|h2), h1 + h2) HashSum.
#[derive(Debug, Clone, Eq)]
pub struct HashSum<T>
where
T: Summable,
{
/// The hash
pub hash: Hash,
/// The sum
pub sum: T::Sum,
}
impl<T> HashSum<T>
where
T: Summable + Hashed,
{
/// Create a hash sum from a summable
pub fn from_summable(idx: u64, elmt: &T) -> HashSum<T> {
let hash = elmt.hash();
let sum = elmt.sum();
let node_hash = (idx, &sum, hash).hash();
HashSum {
hash: node_hash,
sum: sum,
}
}
}
impl<T> PartialEq for HashSum<T>
where
T: Summable,
{
fn eq(&self, other: &HashSum<T>) -> bool {
self.hash == other.hash && self.sum == other.sum
}
}
impl<T> Readable for HashSum<T>
where
T: Summable,
{
fn read(r: &mut Reader) -> Result<HashSum<T>, ser::Error> {
Ok(HashSum {
hash: Hash::read(r)?,
sum: T::Sum::read(r)?,
})
}
}
impl<T> Writeable for HashSum<T>
where
T: Summable,
{
fn write<W: Writer>(&self, w: &mut W) -> Result<(), ser::Error> {
self.hash.write(w)?;
self.sum.write(w)
}
}
impl<T> ops::Add for HashSum<T>
where
T: Summable,
{
type Output = HashSum<T>;
fn add(self, other: HashSum<T>) -> HashSum<T> {
HashSum {
hash: (self.hash, other.hash).hash(),
sum: self.sum + other.sum,
}
}
}
/// Storage backend for the MMR, just needs to be indexed by order of insertion.
/// The PMMR itself does not need the Backend to be accurate on the existence
/// of an element (i.e. remove could be a no-op) but layers above can
/// depend on an accurate Backend to check existence.
pub trait Backend<T>
where
T: Summable,
{
/// Append the provided HashSums to the backend storage. The position of the
/// first element of the Vec in the MMR is provided to help the
/// implementation.
fn append(&mut self, position: u64, data: Vec<HashSum<T>>) -> Result<(), String>;
pub trait Backend<T> where
T:PMMRable {
/// Append the provided Hashes to the backend storage, and optionally an associated
/// data element to flatfile storage (for leaf nodes only). The position of the
/// first element of the Vec in the MMR is provided to help the implementation.
fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String>;
/// Rewind the backend state to a previous position, as if all append
/// operations after that had been canceled. Expects a position in the PMMR
@ -199,14 +60,20 @@ where
/// occurred (see remove).
fn rewind(&mut self, position: u64, index: u32) -> Result<(), String>;
/// Get a HashSum by insertion position
fn get(&self, position: u64) -> Option<HashSum<T>>;
/// Get a Hash/Element by insertion position. If include_data is true, will
/// also return the associated data element
fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option<T>)>;
/// Remove HashSums by insertion position. An index is also provided so the
/// Remove Hashes/Data by insertion position. An index is also provided so the
/// underlying backend can implement some rollback of positions up to a
/// given index (practically the index is a the height of a block that
/// triggered removal).
fn remove(&mut self, positions: Vec<u64>, index: u32) -> Result<(), String>;
/// Returns the data file path.. this is a bit of a hack now that doesn't
/// sit well with the design, but TxKernels have to be summed and the
/// fastest way to to be able to allow direct access to the file
fn get_data_file_path(&self) -> String;
}
/// Prunable Merkle Mountain Range implementation. All positions within the tree
@ -218,18 +85,18 @@ where
/// we are in the sequence of nodes making up the MMR.
pub struct PMMR<'a, T, B>
where
T: Summable,
T: PMMRable,
B: 'a + Backend<T>,
{
last_pos: u64,
backend: &'a mut B,
// only needed for parameterizing Backend
summable: PhantomData<T>,
writeable: PhantomData<T>,
}
impl<'a, T, B> PMMR<'a, T, B>
where
T: Summable + Hashed + Clone,
T: PMMRable,
B: 'a + Backend<T>,
{
/// Build a new prunable Merkle Mountain Range using the provided backend.
@ -237,7 +104,7 @@ where
PMMR {
last_pos: 0,
backend: backend,
summable: PhantomData,
writeable: PhantomData,
}
}
@ -248,48 +115,51 @@ where
PMMR {
last_pos: last_pos,
backend: backend,
summable: PhantomData,
writeable: PhantomData,
}
}
/// Computes the root of the MMR. Find all the peaks in the current
/// tree and "bags" them to get a single peak.
pub fn root(&self) -> HashSum<T> {
pub fn root(&self) -> Hash {
let peaks_pos = peaks(self.last_pos);
let peaks: Vec<Option<HashSum<T>>> = map_vec!(peaks_pos, |&pi| self.backend.get(pi));
let peaks: Vec<Option<(Hash, Option<T>)>> = peaks_pos.into_iter()
.map(|pi| self.backend.get(pi, false))
.collect();
let mut ret = None;
for peak in peaks {
ret = match (ret, peak) {
(None, x) => x,
(Some(hsum), None) => Some(hsum),
(Some(lhsum), Some(rhsum)) => Some(lhsum + rhsum),
(Some(hash), None) => Some(hash),
(Some(lhash), Some(rhash)) => Some((lhash.0.hash_with(rhash.0), None)),
}
}
ret.expect("no root, invalid tree")
ret.expect("no root, invalid tree").0
}
/// Push a new Summable element in the MMR. Computes new related peaks at
/// Push a new element into the MMR. Computes new related peaks at
/// the same time if applicable.
pub fn push(&mut self, elmt: T) -> Result<u64, String> {
let elmt_pos = self.last_pos + 1;
let mut current_hashsum = HashSum::from_summable(elmt_pos, &elmt);
let mut to_append = vec![current_hashsum.clone()];
let mut current_hash = elmt.hash();
let mut to_append = vec![(current_hash, Some(elmt))];
let mut height = 0;
let mut pos = elmt_pos;
// we look ahead one position in the MMR, if the expected node has a higher
// height it means we have to build a higher peak by summing with a previous
// height it means we have to build a higher peak by hashing with a previous
// sibling. we do it iteratively in case the new peak itself allows the
// creation of another parent.
while bintree_postorder_height(pos + 1) > height {
let left_sibling = bintree_jump_left_sibling(pos);
let left_hashsum = self.backend.get(left_sibling).expect(
let left_elem = self.backend.get(left_sibling, false).expect(
"missing left sibling in tree, should not have been pruned",
);
current_hashsum = left_hashsum + current_hashsum;
current_hash = left_elem.0 + current_hash;
to_append.push(current_hashsum.clone());
to_append.push((current_hash.clone(), None));
height += 1;
pos += 1;
}
@ -322,7 +192,7 @@ where
/// to keep an index of elements to positions in the tree. Prunes parent
/// nodes as well when they become childless.
pub fn prune(&mut self, position: u64, index: u32) -> Result<bool, String> {
if let None = self.backend.get(position) {
if let None = self.backend.get(position, false) {
return Ok(false);
}
let prunable_height = bintree_postorder_height(position);
@ -345,7 +215,7 @@ where
// if we have a pruned sibling, we can continue up the tree
// otherwise we're done
if let None = self.backend.get(sibling) {
if let None = self.backend.get(sibling, false) {
current = parent;
} else {
break;
@ -356,26 +226,27 @@ where
Ok(true)
}
/// Helper function to get the HashSum of a node at a given position from
/// Helper function to get a node at a given position from
/// the backend.
pub fn get(&self, position: u64) -> Option<HashSum<T>> {
pub fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option<T>)> {
if position > self.last_pos {
None
} else {
self.backend.get(position)
self.backend.get(position, include_data)
}
}
/// Helper function to get the last N nodes inserted, i.e. the last
/// n nodes along the bottom of the tree
pub fn get_last_n_insertions(&self, n: u64) -> Vec<HashSum<T>> {
pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, Option<T>)> {
let mut return_vec = Vec::new();
let mut last_leaf = self.last_pos;
let size = self.unpruned_size();
// Special case that causes issues in bintree functions,
// just return
if size == 1 {
return_vec.push(self.backend.get(last_leaf).unwrap());
return_vec.push(self.backend.get(last_leaf, true).unwrap());
return return_vec;
}
// if size is even, we're already at the bottom, otherwise
@ -390,7 +261,7 @@ where
if bintree_postorder_height(last_leaf) > 0 {
last_leaf = bintree_rightmost(last_leaf);
}
return_vec.push(self.backend.get(last_leaf).unwrap());
return_vec.push(self.backend.get(last_leaf, true).unwrap());
last_leaf = bintree_jump_left_sibling(last_leaf);
}
@ -398,21 +269,20 @@ where
}
/// Walks all unpruned nodes in the MMR and revalidate all parent hashes
/// and sums.
pub fn validate(&self) -> Result<(), String> {
// iterate on all parent nodes
for n in 1..(self.last_pos + 1) {
if bintree_postorder_height(n) > 0 {
if let Some(hs) = self.get(n) {
if let Some(hs) = self.get(n, false) {
// take the left and right children, if they exist
let left_pos = bintree_move_down_left(n).unwrap();
let right_pos = bintree_jump_right_sibling(left_pos);
if let Some(left_child_hs) = self.get(left_pos) {
if let Some(right_child_hs) = self.get(right_pos) {
// sum and compare
if left_child_hs + right_child_hs != hs {
return Err(format!("Invalid MMR, hashsum of parent at {} does \
if let Some(left_child_hs) = self.get(left_pos, false) {
if let Some(right_child_hs) = self.get(right_pos, false) {
// add hashes and compare
if left_child_hs.0+right_child_hs.0 != hs.0 {
return Err(format!("Invalid MMR, hash of parent at {} does \
not match children.", n));
}
}
@ -429,6 +299,11 @@ where
self.last_pos
}
/// Return the path of the data file (needed to sum kernels efficiently)
pub fn data_file_path(&self) -> String {
self.backend.get_data_file_path()
}
/// Debugging utility to print information about the MMRs. Short version
/// only prints the last 8 nodes.
pub fn dump(&self, short: bool) {
@ -445,40 +320,36 @@ where
break;
}
idx.push_str(&format!("{:>8} ", m + 1));
let ohs = self.get(m + 1);
let ohs = self.get(m + 1, false);
match ohs {
Some(hs) => hashes.push_str(&format!("{} ", hs.hash)),
Some(hs) => hashes.push_str(&format!("{} ", hs.0)),
None => hashes.push_str(&format!("{:>8} ", "??")),
}
}
debug!(LOGGER, "{}", idx);
debug!(LOGGER, "{}", hashes);
trace!(LOGGER, "{}", idx);
trace!(LOGGER, "{}", hashes);
}
}
}
/// Simple MMR backend implementation based on a Vector. Pruning does not
/// compact the Vector itself but still frees the reference to the
/// underlying HashSum.
/// underlying Hash.
#[derive(Clone)]
pub struct VecBackend<T>
where
T: Summable + Clone,
{
where T:PMMRable {
/// Backend elements
pub elems: Vec<Option<HashSum<T>>>,
pub elems: Vec<Option<(Hash, Option<T>)>>,
}
impl<T> Backend<T> for VecBackend<T>
where
T: Summable + Clone,
{
impl <T> Backend <T> for VecBackend<T>
where T: PMMRable {
#[allow(unused_variables)]
fn append(&mut self, position: u64, data: Vec<HashSum<T>>) -> Result<(), String> {
fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String> {
self.elems.append(&mut map_vec!(data, |d| Some(d.clone())));
Ok(())
}
fn get(&self, position: u64) -> Option<HashSum<T>> {
fn get(&self, position: u64, _include_data:bool) -> Option<(Hash, Option<T>)> {
self.elems[(position - 1) as usize].clone()
}
#[allow(unused_variables)]
@ -493,18 +364,19 @@ where
self.elems = self.elems[0..(position as usize) + 1].to_vec();
Ok(())
}
fn get_data_file_path(&self) -> String {
"".to_string()
}
}
impl<T> VecBackend<T>
where
T: Summable + Clone,
{
impl <T> VecBackend <T>
where T:PMMRable {
/// Instantiates a new VecBackend<T>
pub fn new() -> VecBackend<T> {
VecBackend { elems: vec![] }
}
/// Current number of HashSum elements in the underlying Vec.
/// Current number of elements in the underlying Vec.
pub fn used_size(&self) -> usize {
let mut usz = self.elems.len();
for elem in self.elems.deref() {
@ -568,6 +440,28 @@ impl PruneList {
}
}
/// As above, but only returning the number of leaf nodes to skip for a
/// given leaf. Helpful if, for instance, data for each leaf is being stored
/// separately in a continuous flat-file
pub fn get_leaf_shift(&self, pos: u64) -> Option<u64> {
// get the position where the node at pos would fit in the pruned list, if
// it's already pruned, nothing to skip
match self.pruned_pos(pos) {
None => None,
Some(idx) => {
// skip by the number of leaf nodes pruned in the preceeding subtrees
// which just 2^height
Some(
self.pruned_nodes[0..(idx as usize)]
.iter()
.map(|n| 1 << bintree_postorder_height(*n))
.sum(),
)
}
}
}
/// Push the node at the provided position in the prune list. Compacts the
/// list if pruning the additional node means a parent can get pruned as
/// well.
@ -591,7 +485,7 @@ impl PruneList {
}
/// Gets the position a new pruned node should take in the prune list.
/// If the node has already bee pruned, either directly or through one of
/// If the node has already been pruned, either directly or through one of
/// its parents contained in the prune list, returns None.
pub fn pruned_pos(&self, pos: u64) -> Option<usize> {
match self.pruned_nodes.binary_search(&pos) {
@ -833,7 +727,20 @@ fn most_significant_pos(num: u64) -> u64 {
#[cfg(test)]
mod test {
use super::*;
use core::hash::Hashed;
use ser::{Writeable, Readable, Error};
use core::{Writer, Reader};
use core::hash::{Hash};
#[test]
fn test_leaf_index(){
assert_eq!(n_leaves(1),1);
assert_eq!(n_leaves(2),2);
assert_eq!(n_leaves(4),3);
assert_eq!(n_leaves(5),4);
assert_eq!(n_leaves(8),5);
assert_eq!(n_leaves(9),6);
}
#[test]
fn some_all_ones() {
@ -890,23 +797,17 @@ mod test {
assert_eq!(peaks(42), vec![31, 38, 41, 42]);
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
struct TestElem([u32; 4]);
impl Summable for TestElem {
type Sum = u64;
fn sum(&self) -> u64 {
// sums are not allowed to overflow, so we use this simple
// non-injective "sum" function that will still be homomorphic
self.0[0] as u64 * 0x1000 + self.0[1] as u64 * 0x100 + self.0[2] as u64 * 0x10 +
self.0[3] as u64
}
fn sum_len() -> usize {
8
impl PMMRable for TestElem {
fn len() -> usize {
16
}
}
impl Writeable for TestElem {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
try!(writer.write_u32(self.0[0]));
try!(writer.write_u32(self.0[1]));
try!(writer.write_u32(self.0[2]));
@ -914,6 +815,19 @@ mod test {
}
}
impl Readable for TestElem {
fn read(reader: &mut Reader) -> Result<TestElem, Error> {
Ok(TestElem (
[
reader.read_u32()?,
reader.read_u32()?,
reader.read_u32()?,
reader.read_u32()?,
]
))
}
}
#[test]
#[allow(unused_variables)]
fn pmmr_push_root() {
@ -934,72 +848,67 @@ mod test {
// one element
pmmr.push(elems[0]).unwrap();
let hash = Hashed::hash(&elems[0]);
let sum = elems[0].sum();
let node_hash = (1 as u64, &sum, hash).hash();
let node_hash = elems[0].hash();
assert_eq!(
pmmr.root(),
HashSum {
hash: node_hash,
sum: sum,
}
node_hash,
);
assert_eq!(pmmr.unpruned_size(), 1);
pmmr.dump(false);
// two elements
pmmr.push(elems[1]).unwrap();
let sum2 = HashSum::from_summable(1, &elems[0]) +
HashSum::from_summable(2, &elems[1]);
let sum2 = elems[0].hash() + elems[1].hash();
pmmr.dump(false);
assert_eq!(pmmr.root(), sum2);
assert_eq!(pmmr.unpruned_size(), 3);
// three elements
pmmr.push(elems[2]).unwrap();
let sum3 = sum2.clone() + HashSum::from_summable(4, &elems[2]);
let sum3 = sum2 + elems[2].hash();
pmmr.dump(false);
assert_eq!(pmmr.root(), sum3);
assert_eq!(pmmr.unpruned_size(), 4);
// four elements
pmmr.push(elems[3]).unwrap();
let sum4 = sum2 +
(HashSum::from_summable(4, &elems[2]) +
HashSum::from_summable(5, &elems[3]));
let sum_one = elems[2].hash() + elems[3].hash();
let sum4 = sum2 + sum_one;
pmmr.dump(false);
assert_eq!(pmmr.root(), sum4);
assert_eq!(pmmr.unpruned_size(), 7);
// five elements
pmmr.push(elems[4]).unwrap();
let sum5 = sum4.clone() + HashSum::from_summable(8, &elems[4]);
assert_eq!(pmmr.root(), sum5);
let sum3 = sum4 + elems[4].hash();
pmmr.dump(false);
assert_eq!(pmmr.root(), sum3);
assert_eq!(pmmr.unpruned_size(), 8);
// six elements
pmmr.push(elems[5]).unwrap();
let sum6 = sum4.clone() +
(HashSum::from_summable(8, &elems[4]) +
HashSum::from_summable(9, &elems[5]));
let sum6 = sum4 +
(elems[4].hash() + elems[5].hash());
assert_eq!(pmmr.root(), sum6.clone());
assert_eq!(pmmr.unpruned_size(), 10);
// seven elements
pmmr.push(elems[6]).unwrap();
let sum7 = sum6 + HashSum::from_summable(11, &elems[6]);
let sum7 = sum6 + elems[6].hash();
assert_eq!(pmmr.root(), sum7);
assert_eq!(pmmr.unpruned_size(), 11);
// eight elements
pmmr.push(elems[7]).unwrap();
let sum8 = sum4 +
((HashSum::from_summable(8, &elems[4]) +
HashSum::from_summable(9, &elems[5])) +
(HashSum::from_summable(11, &elems[6]) +
HashSum::from_summable(12, &elems[7])));
((elems[4].hash() + elems[5].hash()) +
(elems[6].hash() + elems[7].hash()));
assert_eq!(pmmr.root(), sum8);
assert_eq!(pmmr.unpruned_size(), 15);
// nine elements
pmmr.push(elems[8]).unwrap();
let sum9 = sum8 + HashSum::from_summable(16, &elems[8]);
let sum9 = sum8 + elems[8].hash();
assert_eq!(pmmr.root(), sum9);
assert_eq!(pmmr.unpruned_size(), 16);
}
@ -1015,8 +924,9 @@ mod test {
TestElem([0, 0, 0, 6]),
TestElem([0, 0, 0, 7]),
TestElem([0, 0, 0, 8]),
TestElem([0, 0, 0, 9]),
TestElem([1, 0, 0, 0]),
];
let mut ba = VecBackend::new();
let mut pmmr = PMMR::new(&mut ba);
@ -1026,23 +936,23 @@ mod test {
pmmr.push(elems[0]).unwrap();
let res = pmmr.get_last_n_insertions(19);
assert!(res.len() == 1 && res[0].sum == 1);
assert!(res.len() == 1);
pmmr.push(elems[1]).unwrap();
let res = pmmr.get_last_n_insertions(12);
assert!(res[0].sum == 2 && res[1].sum == 1);
assert!(res.len() == 2);
pmmr.push(elems[2]).unwrap();
let res = pmmr.get_last_n_insertions(2);
assert!(res[0].sum == 3 && res[1].sum == 2);
assert!(res.len() == 2);
pmmr.push(elems[3]).unwrap();
let res = pmmr.get_last_n_insertions(19);
assert!(
res[0].sum == 4 && res[1].sum == 3 && res[2].sum == 2 && res[3].sum == 1 && res.len() == 4
res.len() == 4
);
pmmr.push(elems[5]).unwrap();
@ -1052,7 +962,7 @@ mod test {
let res = pmmr.get_last_n_insertions(7);
assert!(
res[0].sum == 9 && res[1].sum == 8 && res[2].sum == 7 && res[3].sum == 6 && res.len() == 7
res.len() == 7
);
}
@ -1071,7 +981,7 @@ mod test {
TestElem([1, 0, 0, 0]),
];
let orig_root: HashSum<TestElem>;
let orig_root: Hash;
let sz: u64;
let mut ba = VecBackend::new();
{
@ -1085,7 +995,7 @@ mod test {
// pruning a leaf with no parent should do nothing
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(16, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@ -1093,14 +1003,14 @@ mod test {
// pruning leaves with no shared parent just removes 1 element
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(2, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
assert_eq!(ba.used_size(), 15);
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(4, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@ -1108,7 +1018,7 @@ mod test {
// pruning a non-leaf node has no effect
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(3, 0).unwrap_err();
assert_eq!(orig_root, pmmr.root());
}
@ -1116,7 +1026,7 @@ mod test {
// pruning sibling removes subtree
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(5, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@ -1124,7 +1034,7 @@ mod test {
// pruning all leaves under level >1 removes all subtree
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(1, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@ -1132,7 +1042,7 @@ mod test {
// pruning everything should only leave us the peaks
{
let mut pmmr = PMMR::at(&mut ba, sz);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
for n in 1..16 {
let _ = pmmr.prune(n, 0);
}

163
core/src/core/transaction.rs
View file

@ -19,16 +19,14 @@ use util::{static_secp_instance, kernel_sig_msg};
use util::secp::pedersen::{Commitment, RangeProof};
use std::cmp::{min, max};
use std::cmp::Ordering;
use std::ops;
use consensus;
use consensus::VerifySortOrder;
use core::Committed;
use core::hash::{Hash, Hashed, ZERO_HASH};
use core::pmmr::Summable;
use keychain;
use keychain::{Identifier, Keychain, BlindingFactor};
use ser::{self, read_and_verify_sorted, Readable, Reader, Writeable, WriteableSorted, Writer};
use keychain;
use ser::{self, read_and_verify_sorted, PMMRable, Readable, Reader, Writeable, WriteableSorted, Writer};
use util;
/// The size of the blake2 hash of a switch commitment (256 bits)
@ -217,9 +215,10 @@ impl TxKernel {
..self
}
}
}
/// Size in bytes of a kernel, necessary for binary storage
pub fn size() -> usize {
impl PMMRable for TxKernel {
fn len() -> usize {
17 + // features plus fee and lock_height
secp::constants::PEDERSEN_COMMITMENT_SIZE +
secp::constants::AGG_SIGNATURE_SIZE
@ -671,17 +670,13 @@ impl SwitchCommitHash {
/// provides future-proofing against quantum-based attacks, as well as providing
/// wallet implementations with a way to identify their outputs for wallet
/// reconstruction.
///
/// The hash of an output only covers its features, commitment,
/// and switch commitment. The range proof is expected to have its own hash
/// and is stored and committed to separately.
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub struct Output {
/// Options for an output's structure or use
pub features: OutputFeatures,
/// The homomorphic commitment representing the output amount
pub commit: Commitment,
/// The switch commitment hash, a 160 bit length blake2 hash of blind*J
/// The switch commitment hash, a 256 bit length blake2 hash of blind*J
pub switch_commit_hash: SwitchCommitHash,
/// A proof that the commitment is in the right range
pub proof: RangeProof,
@ -704,9 +699,13 @@ impl Writeable for Output {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
writer.write_u8(self.features.bits())?;
writer.write_fixed_bytes(&self.commit)?;
writer.write_fixed_bytes(&self.switch_commit_hash)?;
// The hash of an output doesn't include the range proof
// Hash of an output doesn't cover the switch commit, it should
// be wound into the range proof separately
if writer.serialization_mode() != ser::SerializationMode::Hash {
writer.write_fixed_bytes(&self.switch_commit_hash)?;
}
// The hash of an output doesn't include the range proof, which
// is commit to separately
if writer.serialization_mode() == ser::SerializationMode::Full {
writer.write_bytes(&self.proof)?
}
@ -818,21 +817,6 @@ impl OutputIdentifier {
util::to_hex(self.commit.0.to_vec()),
)
}
/// Convert an output_indentifier to a sum_commit representation
/// so we can use it to query the the output MMR
pub fn as_sum_commit(&self) -> SumCommit {
SumCommit {
features: self.features,
commit: self.commit,
switch_commit_hash: SwitchCommitHash::zero(),
}
}
/// Convert a sum_commit back to an output_identifier.
pub fn from_sum_commit(sum_commit: &SumCommit) -> OutputIdentifier {
OutputIdentifier::new(sum_commit.features, &sum_commit.commit)
}
}
impl Writeable for OutputIdentifier {
@ -855,140 +839,73 @@ impl Readable for OutputIdentifier {
}
}
/// Wrapper to Output commitments to provide the Summable trait.
/// Yet another output version to read/write from disk. Ends up being far too awkward
/// to use the write serialisation property to do this
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct SumCommit {
pub struct OutputStoreable {
/// Output features (coinbase vs. regular transaction output)
/// We need to include this when hashing to ensure coinbase maturity can be enforced.
pub features: OutputFeatures,
/// Output commitment
pub commit: Commitment,
/// The corresponding switch commit hash
/// Switch commit hash
pub switch_commit_hash: SwitchCommitHash,
}
impl SumCommit {
/// Build a new sum_commit.
pub fn new(
features: OutputFeatures,
commit: &Commitment,
switch_commit_hash: &SwitchCommitHash,
) -> SumCommit {
SumCommit {
features: features.clone(),
commit: commit.clone(),
switch_commit_hash: switch_commit_hash.clone(),
}
}
/// Build a new sum_commit from an existing output.
pub fn from_output(output: &Output) -> SumCommit {
SumCommit {
impl OutputStoreable {
/// Build a StoreableOutput from an existing output.
pub fn from_output(output: &Output) -> OutputStoreable {
OutputStoreable {
features: output.features,
commit: output.commit,
switch_commit_hash: output.switch_commit_hash,
}
}
/// Build a new sum_commit from an existing input.
pub fn from_input(input: &Input) -> SumCommit {
SumCommit {
features: input.features,
commit: input.commit,
switch_commit_hash: SwitchCommitHash::zero(),
/// Return a regular output
pub fn to_output(self) -> Output {
Output{
features: self.features,
commit: self.commit,
switch_commit_hash: self.switch_commit_hash,
proof: RangeProof{
proof:[0; secp::constants::MAX_PROOF_SIZE],
plen: 0,
},
}
}
/// Hex string representation of a sum_commit.
pub fn to_hex(&self) -> String {
format!(
"{:b}{}{}",
self.features.bits(),
util::to_hex(self.commit.0.to_vec()),
self.switch_commit_hash.to_hex(),
)
}
}
/// Outputs get summed through their commitments.
impl Summable for SumCommit {
type Sum = SumCommit;
fn sum(&self) -> SumCommit {
SumCommit {
commit: self.commit.clone(),
features: self.features.clone(),
switch_commit_hash: self.switch_commit_hash.clone(),
}
}
fn sum_len() -> usize {
secp::constants::PEDERSEN_COMMITMENT_SIZE + SWITCH_COMMIT_HASH_SIZE + 1
impl PMMRable for OutputStoreable {
fn len() -> usize {
1 + secp::constants::PEDERSEN_COMMITMENT_SIZE + SWITCH_COMMIT_HASH_SIZE
}
}
impl Writeable for SumCommit {
impl Writeable for OutputStoreable {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
writer.write_u8(self.features.bits())?;
self.commit.write(writer)?;
if writer.serialization_mode() == ser::SerializationMode::Full {
if writer.serialization_mode() != ser::SerializationMode::Hash {
self.switch_commit_hash.write(writer)?;
}
Ok(())
}
}
impl Readable for SumCommit {
fn read(reader: &mut Reader) -> Result<SumCommit, ser::Error> {
impl Readable for OutputStoreable {
fn read(reader: &mut Reader) -> Result<OutputStoreable, ser::Error> {
let features = OutputFeatures::from_bits(reader.read_u8()?).ok_or(
ser::Error::CorruptedData,
)?;
Ok(SumCommit {
features: features,
Ok(OutputStoreable {
commit: Commitment::read(reader)?,
switch_commit_hash: SwitchCommitHash::read(reader)?,
features: features,
})
}
}
impl ops::Add for SumCommit {
type Output = SumCommit;
fn add(self, other: SumCommit) -> SumCommit {
// Build a new commitment by summing the two commitments.
let secp = static_secp_instance();
let sum = match secp.lock().unwrap().commit_sum(
vec![
self.commit.clone(),
other.commit.clone(),
],
vec![],
) {
Ok(s) => s,
Err(_) => Commitment::from_vec(vec![1; 33]),
};
// Now build a new switch_commit_hash by concatenating the two switch_commit_hash value
// and hashing the result.
let mut bytes = self.switch_commit_hash.0.to_vec();
bytes.extend(other.switch_commit_hash.0.iter().cloned());
let key = SwitchCommitHashKey::zero();
let hash = blake2b(SWITCH_COMMIT_HASH_SIZE, &key.0, &bytes);
let hash = hash.as_bytes();
let mut h = [0; SWITCH_COMMIT_HASH_SIZE];
for i in 0..SWITCH_COMMIT_HASH_SIZE {
h[i] = hash[i];
}
let switch_commit_hash_sum = SwitchCommitHash(h);
SumCommit {
features: self.features | other.features,
commit: sum,
switch_commit_hash: switch_commit_hash_sum,
}
}
}
#[cfg(test)]
mod test {
use super::*;

20
core/src/ser.rs
View file

@ -37,6 +37,8 @@ use util::secp::constants::{
SECRET_KEY_SIZE,
};
const BULLET_PROOF_SIZE:usize = 674;
/// Possible errors deriving from serializing or deserializing.
#[derive(Debug)]
pub enum Error {
@ -119,6 +121,9 @@ pub enum SerializationMode {
Hash,
/// Serialize everything that a signer of the object should know
SigHash,
/// Serialize for local storage, for instance in the case where
/// an output doesn't wish to store its range proof
Storage,
}
/// Implementations defined how different numbers and binary structures are
@ -255,6 +260,7 @@ pub fn ser_vec<W: Writeable>(thing: &W) -> Result<Vec<u8>, Error> {
Ok(vec)
}
/// Utility to read from a binary source
struct BinReader<'a> {
source: &'a mut Read,
}
@ -364,7 +370,7 @@ impl Writeable for RangeProof {
impl Readable for RangeProof {
fn read(reader: &mut Reader) -> Result<RangeProof, Error> {
let p = try!(reader.read_limited_vec(MAX_PROOF_SIZE));
let p = try!(reader.read_limited_vec(BULLET_PROOF_SIZE));
let mut a = [0; MAX_PROOF_SIZE];
for i in 0..p.len() {
a[i] = p[i];
@ -376,6 +382,12 @@ impl Readable for RangeProof {
}
}
impl PMMRable for RangeProof {
fn len() -> usize {
BULLET_PROOF_SIZE
}
}
impl Readable for Signature {
fn read(reader: &mut Reader) -> Result<Signature, Error> {
let a = try!(reader.read_fixed_bytes(AGG_SIGNATURE_SIZE));
@ -542,6 +554,12 @@ impl Writeable for [u8; 4] {
}
}
/// Trait for types that can serialize and report their size
pub trait PMMRable: Readable + Writeable + Hashed + Clone {
/// Length in bytes
fn len() -> usize;
}
/// Useful marker trait on types that can be sized byte slices
pub trait AsFixedBytes: Sized + AsRef<[u8]> {
/// The length in bytes

26
grin/tests/api.rs
View file

@ -283,38 +283,38 @@ fn get_utxos_by_height(base_addr: &String, api_server_port: u16, start_height: u
// Sumtree handler functions
fn get_sumtree_roots(base_addr: &String, api_server_port: u16) -> Result<api::SumTrees, Error> {
let url = format!("http://{}:{}/v1/sumtrees/roots", base_addr, api_server_port);
let url = format!("http://{}:{}/v1/pmmrtrees/roots", base_addr, api_server_port);
api::client::get::<api::SumTrees>(url.as_str()).map_err(|e| Error::API(e))
}
fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::PmmrTreeNode>, Error> {
let url: String;
if n == 0 {
url = format!("http://{}:{}/v1/sumtrees/lastutxos", base_addr, api_server_port);
url = format!("http://{}:{}/v1/pmmrtrees/lastutxos", base_addr, api_server_port);
} else {
url = format!("http://{}:{}/v1/sumtrees/lastutxos?n={}", base_addr, api_server_port, n);
url = format!("http://{}:{}/v1/pmmrtrees/lastutxos?n={}", base_addr, api_server_port, n);
}
api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
api::client::get::<Vec<api::PmmrTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
}
fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::PmmrTreeNode>, Error> {
let url: String;
if n == 0 {
url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs", base_addr, api_server_port);
url = format!("http://{}:{}/v1/pmmrtrees/lastrangeproofs", base_addr, api_server_port);
} else {
url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs?n={}", base_addr, api_server_port, n);
url = format!("http://{}:{}/v1/pmmrtrees/lastrangeproofs?n={}", base_addr, api_server_port, n);
}
api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
api::client::get::<Vec<api::PmmrTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
}
fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::PmmrTreeNode>, Error> {
let url: String;
if n == 0 {
url = format!("http://{}:{}/v1/sumtrees/lastkernels", base_addr, api_server_port);
url = format!("http://{}:{}/v1/pmmrtrees/lastkernels", base_addr, api_server_port);
} else {
url = format!("http://{}:{}/v1/sumtrees/lastkernels?n={}", base_addr, api_server_port, n);
url = format!("http://{}:{}/v1/pmmrtrees/lastkernels?n={}", base_addr, api_server_port, n);
}
api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
api::client::get::<Vec<api::PmmrTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
}
// Helper function to get a vec of commitment output ids from a vec of block outputs

4
keychain/src/keychain.rs
View file

@ -29,9 +29,9 @@ use blind::{BlindSum, BlindingFactor};
use extkey::{self, Identifier};
#[cfg(feature = "use-bullet-proofs")]
const USE_BULLET_PROOFS:bool = true;
pub const USE_BULLET_PROOFS:bool = true;
#[cfg(not(feature = "use-bullet-proofs"))]
const USE_BULLET_PROOFS:bool = false;
pub const USE_BULLET_PROOFS:bool = false;
#[derive(PartialEq, Eq, Clone, Debug)]
pub enum Error {

3
store/src/lib.rs
View file

@ -30,7 +30,8 @@ extern crate rocksdb;
#[macro_use]
extern crate slog;
pub mod sumtree;
pub mod pmmr;
pub mod types;
const SEP: u8 = ':' as u8;

318
store/src/pmmr.rs Normal file
View file

@ -0,0 +1,318 @@
// Copyright 2017 The Grin Developers
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Implementation of the persistent Backend for the prunable MMR tree.
use std::fs::{self};
use std::io::{self};
use std::marker::PhantomData;
use core::core::pmmr::{self, Backend};
use core::ser::{self, PMMRable};
use core::core::hash::Hash;
use util::LOGGER;
use types::{AppendOnlyFile, RemoveLog, read_ordered_vec, write_vec};
const PMMR_HASH_FILE: &'static str = "pmmr_hash.bin";
const PMMR_DATA_FILE: &'static str = "pmmr_data.bin";
const PMMR_RM_LOG_FILE: &'static str = "pmmr_rm_log.bin";
const PMMR_PRUNED_FILE: &'static str = "pmmr_pruned.bin";
/// Maximum number of nodes in the remove log before it gets flushed
pub const RM_LOG_MAX_NODES: usize = 10000;
/// PMMR persistent backend implementation. Relies on multiple facilities to
/// handle writing, reading and pruning.
///
/// * A main storage file appends Hash instances as they come. This
/// AppendOnlyFile is also backed by a mmap for reads.
/// * An in-memory backend buffers the latest batch of writes to ensure the
/// PMMR can always read recent values even if they haven't been flushed to
/// disk yet.
/// * A remove log tracks the positions that need to be pruned from the
/// main storage file.
pub struct PMMRBackend<T>
where
T: PMMRable,
{
data_dir: String,
hash_file: AppendOnlyFile,
data_file: AppendOnlyFile,
rm_log: RemoveLog,
pruned_nodes: pmmr::PruneList,
phantom: PhantomData<T>,
}
impl<T> Backend<T> for PMMRBackend<T>
where
T: PMMRable,
{
/// Append the provided Hashes to the backend storage.
#[allow(unused_variables)]
fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String> {
for d in data {
self.hash_file.append(&mut ser::ser_vec(&d.0).unwrap());
if let Some(elem) = d.1 {
self.data_file.append(&mut ser::ser_vec(&elem).unwrap());
}
}
Ok(())
}
/// Get a Hash by insertion position
fn get(&self, position: u64, include_data:bool) -> Option<(Hash, Option<T>)> {
// Check if this position has been pruned in the remove log or the
// pruned list
if self.rm_log.includes(position) {
return None;
}
let shift = self.pruned_nodes.get_shift(position);
if let None = shift {
return None;
}
// Read PMMR
// The MMR starts at 1, our binary backend starts at 0
let pos = position - 1;
// Must be on disk, doing a read at the correct position
let hash_record_len = 32;
let file_offset = ((pos - shift.unwrap()) as usize) * hash_record_len;
let data = self.hash_file.read(file_offset, hash_record_len);
let hash_val = match ser::deserialize(&mut &data[..]) {
Ok(h) => h,
Err(e) => {
error!(
LOGGER,
"Corrupted storage, could not read an entry from hash store: {:?}",
e
);
return None;
}
};
if !include_data {
return Some(((hash_val), None));
}
// Optionally read flatfile storage to get data element
let flatfile_pos = pmmr::n_leaves(position)
- 1 - self.pruned_nodes.get_leaf_shift(position).unwrap();
let record_len = T::len();
let file_offset = flatfile_pos as usize * T::len();
let data = self.data_file.read(file_offset, record_len);
let data = match ser::deserialize(&mut &data[..]) {
Ok(elem) => Some(elem),
Err(e) => {
error!(
LOGGER,
"Corrupted storage, could not read an entry from backend flatfile store: {:?}",
e
);
None
}
};
Some((hash_val, data))
}
fn rewind(&mut self, position: u64, index: u32) -> Result<(), String> {
self.rm_log
.rewind(index)
.map_err(|e| format!("Could not truncate remove log: {}", e))?;
let shift = self.pruned_nodes.get_shift(position).unwrap_or(0);
let record_len = 32;
let file_pos = (position - shift) * (record_len as u64);
self.hash_file.rewind(file_pos);
//Data file
let flatfile_pos = pmmr::n_leaves(position) - 1;
let file_pos = (flatfile_pos as usize + 1) * T::len();
self.data_file.rewind(file_pos as u64);
Ok(())
}
/// Remove Hashes by insertion position
fn remove(&mut self, positions: Vec<u64>, index: u32) -> Result<(), String> {
self.rm_log.append(positions, index).map_err(|e| {
format!("Could not write to log storage, disk full? {:?}", e)
})
}
/// Return data file path
fn get_data_file_path(&self) -> String {
self.data_file.path()
}
}
impl<T> PMMRBackend<T>
where
T: PMMRable,
{
/// Instantiates a new PMMR backend that will use the provided directly to
/// store its files.
pub fn new(data_dir: String) -> io::Result<PMMRBackend<T>> {
let hash_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_HASH_FILE))?;
let rm_log = RemoveLog::open(format!("{}/{}", data_dir, PMMR_RM_LOG_FILE))?;
let prune_list = read_ordered_vec(format!("{}/{}", data_dir, PMMR_PRUNED_FILE), 8)?;
let data_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_DATA_FILE))?;
Ok(PMMRBackend {
data_dir: data_dir,
hash_file: hash_file,
data_file: data_file,
rm_log: rm_log,
pruned_nodes: pmmr::PruneList {
pruned_nodes: prune_list,
},
phantom: PhantomData,
})
}
/// Total size of the PMMR stored by this backend. Only produces the fully
/// sync'd size.
pub fn unpruned_size(&self) -> io::Result<u64> {
let total_shift = self.pruned_nodes.get_shift(::std::u64::MAX).unwrap();
let record_len = 32;
let sz = self.hash_file.size()?;
Ok(sz / record_len + total_shift)
}
/// Syncs all files to disk. A call to sync is required to ensure all the
/// data has been successfully written to disk.
pub fn sync(&mut self) -> io::Result<()> {
if let Err(e) = self.hash_file.flush() {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
format!("Could not write to log hash storage, disk full? {:?}", e),
));
}
if let Err(e) = self.data_file.flush() {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
format!("Could not write to log data storage, disk full? {:?}", e),
));
}
self.rm_log.flush()?;
Ok(())
}
/// Discard the current, non synced state of the backend.
pub fn discard(&mut self) {
self.hash_file.discard();
self.rm_log.discard();
self.data_file.discard();
}
/// Return the data file path
pub fn data_file_path(&self) -> String {
self.get_data_file_path()
}
/// Checks the length of the remove log to see if it should get compacted.
/// If so, the remove log is flushed into the pruned list, which itself gets
/// saved, and the main hashsum data file is rewritten, cutting the removed
/// data.
///
/// If a max_len strictly greater than 0 is provided, the value will be used
/// to decide whether the remove log has reached its maximum length,
/// otherwise the RM_LOG_MAX_NODES default value is used.
///
/// TODO whatever is calling this should also clean up the commit to
/// position index in db
pub fn check_compact(&mut self, max_len: usize) -> io::Result<()> {
if !(max_len > 0 && self.rm_log.len() > max_len
|| max_len == 0 && self.rm_log.len() > RM_LOG_MAX_NODES)
{
return Ok(());
}
// 0. validate none of the nodes in the rm log are in the prune list (to
// avoid accidental double compaction)
for pos in &self.rm_log.removed[..] {
if let None = self.pruned_nodes.pruned_pos(pos.0) {
// TODO we likely can recover from this by directly jumping to 3
error!(
LOGGER,
"The remove log contains nodes that are already in the pruned \
list, a previous compaction likely failed."
);
return Ok(());
}
}
// 1. save hashsum file to a compact copy, skipping data that's in the
// remove list
let tmp_prune_file_hash = format!("{}/{}.hashprune", self.data_dir, PMMR_HASH_FILE);
let record_len = 32;
let to_rm = self.rm_log
.removed
.iter()
.map(|&(pos, _)| {
let shift = self.pruned_nodes.get_shift(pos);
(pos - 1 - shift.unwrap()) * record_len
})
.collect();
self.hash_file
.save_prune(tmp_prune_file_hash.clone(), to_rm, record_len)?;
// 2. And the same with the data file
let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir, PMMR_DATA_FILE);
let record_len = T::len() as u64;
let to_rm = self.rm_log
.removed.clone()
.into_iter()
.filter(|&(pos, _)| pmmr::bintree_postorder_height(pos) == 0)
.map(|(pos, _)| {
let shift = self.pruned_nodes.get_leaf_shift(pos).unwrap();
let pos = pmmr::n_leaves(pos as u64);
(pos - 1 - shift) * record_len
})
.collect();
self.data_file
.save_prune(tmp_prune_file_data.clone(), to_rm, record_len)?;
// 3. update the prune list and save it in place
for &(rm_pos, _) in &self.rm_log.removed[..] {
self.pruned_nodes.add(rm_pos);
}
write_vec(
format!("{}/{}", self.data_dir, PMMR_PRUNED_FILE),
&self.pruned_nodes.pruned_nodes,
)?;
// 4. move the compact copy of hashes to the hashsum file and re-open it
fs::rename(
tmp_prune_file_hash.clone(),
format!("{}/{}", self.data_dir, PMMR_HASH_FILE),
)?;
self.hash_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_HASH_FILE))?;
// 5. and the same with the data file
fs::rename(
tmp_prune_file_data.clone(),
format!("{}/{}", self.data_dir, PMMR_DATA_FILE),
)?;
self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE))?;
// 6. truncate the rm log
self.rm_log.rewind(0)?;
self.rm_log.flush()?;
Ok(())
}
}

288
store/src/sumtree.rs → store/src/types.rs
View file

@ -1,4 +1,4 @@
// Copyright 2017 The Grin Developers
// Copyright 2018 The Grin Developers
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@ -11,33 +11,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Implementation of the persistent Backend for the prunable MMR sum-tree.
//! Common storage-related types
use memmap;
use std::cmp;
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufRead, BufReader, ErrorKind, Write};
use std::marker::PhantomData;
use std::os::unix::io::AsRawFd;
use std::path::Path;
use std::io::Read;
use std::path::Path;
#[cfg(any(target_os = "linux"))]
use libc::{ftruncate64, off64_t};
#[cfg(not(any(target_os = "linux", target_os = "android")))]
use libc::{ftruncate as ftruncate64, off_t as off64_t};
use core::core::pmmr::{self, Backend, HashSum, Summable};
use core::ser;
use util::LOGGER;
const PMMR_DATA_FILE: &'static str = "pmmr_dat.bin";
const PMMR_RM_LOG_FILE: &'static str = "pmmr_rm_log.bin";
const PMMR_PRUNED_FILE: &'static str = "pmmr_pruned.bin";
/// Maximum number of nodes in the remove log before it gets flushed
pub const RM_LOG_MAX_NODES: usize = 10000;
/// Wrapper for a file that can be read at any position (random read) but for
/// which writes are append only. Reads are backed by a memory map (mmap(2)),
@ -54,6 +43,7 @@ pub struct AppendOnlyFile {
buffer_start: usize,
buffer: Vec<u8>,
buffer_start_bak: usize,
unflushed_data_size: usize,
}
impl AppendOnlyFile {
@ -71,6 +61,7 @@ impl AppendOnlyFile {
buffer_start: 0,
buffer: vec![],
buffer_start_bak: 0,
unflushed_data_size: 0,
};
if let Ok(sz) = aof.size() {
if sz > 0 {
@ -84,6 +75,7 @@ impl AppendOnlyFile {
/// Append data to the file. Until the append-only file is synced, data is
/// only written to memory.
pub fn append(&mut self, buf: &mut Vec<u8>) {
self.unflushed_data_size += buf.len();
self.buffer.append(buf);
}
@ -110,6 +102,7 @@ impl AppendOnlyFile {
self.file.sync_data()?;
self.buffer = vec![];
self.mmap = Some(unsafe { memmap::Mmap::map(&self.file)? });
self.unflushed_data_size = 0;
Ok(())
}
@ -121,11 +114,12 @@ impl AppendOnlyFile {
self.buffer_start_bak = 0;
}
self.buffer = vec![];
self.unflushed_data_size = 0;
}
/// Read length bytes of data at offset from the file. Leverages the memory
/// map.
fn read(&self, offset: usize, length: usize) -> Vec<u8> {
pub fn read(&self, offset: usize, length: usize) -> Vec<u8> {
if offset >= self.buffer_start {
let offset = offset - self.buffer_start;
return self.buffer[offset..(offset+length)].to_vec();
@ -138,7 +132,7 @@ impl AppendOnlyFile {
}
/// Truncates the underlying file to the provided offset
fn truncate(&self, offs: usize) -> io::Result<()> {
pub fn truncate(&self, offs: usize) -> io::Result<()> {
let fd = self.file.as_raw_fd();
let res = unsafe { ftruncate64(fd, offs as off64_t) };
if res == -1 {
@ -150,7 +144,7 @@ impl AppendOnlyFile {
/// Saves a copy of the current file content, skipping data at the provided
/// prune indices. The prune Vec must be ordered.
fn save_prune(&self, target: String, prune_offs: Vec<u64>, prune_len: u64) -> io::Result<()> {
pub fn save_prune(&self, target: String, prune_offs: Vec<u64>, prune_len: u64) -> io::Result<()> {
let mut reader = File::open(self.path.clone())?;
let mut writer = File::create(target)?;
@ -188,10 +182,15 @@ impl AppendOnlyFile {
}
/// Current size of the file in bytes.
fn size(&self) -> io::Result<u64> {
pub fn size(&self) -> io::Result<u64> {
fs::metadata(&self.path).map(|md| md.len())
}
/// Current size of file in bytes + size of unsaved data
pub fn size_with_unsaved(&self) -> u64 {
self.size().unwrap() + self.unflushed_data_size as u64
}
/// Path of the underlying file
pub fn path(&self) -> String {
self.path.clone()
@ -203,10 +202,10 @@ impl AppendOnlyFile {
/// checking of whether a piece of data has been marked for deletion. When the
/// log becomes too long, the MMR backend will actually remove chunks from the
/// MMR data file and truncate the remove log.
struct RemoveLog {
pub struct RemoveLog {
path: String,
// Ordered vector of MMR positions that should get eventually removed.
removed: Vec<(u64, u32)>,
/// Ordered vector of MMR positions that should get eventually removed.
pub removed: Vec<(u64, u32)>,
// Holds positions temporarily until flush is called.
removed_tmp: Vec<(u64, u32)>,
// Holds truncated removed temporarily until discarded or committed
@ -216,7 +215,7 @@ struct RemoveLog {
impl RemoveLog {
/// Open the remove log file. The content of the file will be read in memory
/// for fast checking.
fn open(path: String) -> io::Result<RemoveLog> {
pub fn open(path: String) -> io::Result<RemoveLog> {
let removed = read_ordered_vec(path.clone(), 12)?;
Ok(RemoveLog {
path: path,
@ -227,7 +226,7 @@ impl RemoveLog {
}
/// Truncate and empties the remove log.
fn rewind(&mut self, last_offs: u32) -> io::Result<()> {
pub fn rewind(&mut self, last_offs: u32) -> io::Result<()> {
// simplifying assumption: we always remove older than what's in tmp
self.removed_tmp = vec![];
@ -251,7 +250,7 @@ impl RemoveLog {
/// Append a set of new positions to the remove log. Both adds those
/// positions the ordered in-memory set and to the file.
fn append(&mut self, elmts: Vec<u64>, index: u32) -> io::Result<()> {
pub fn append(&mut self, elmts: Vec<u64>, index: u32) -> io::Result<()> {
for elmt in elmts {
match self.removed_tmp.binary_search(&(elmt, index)) {
Ok(_) => continue,
@ -264,7 +263,7 @@ impl RemoveLog {
}
/// Flush the positions to remove to file.
fn flush(&mut self) -> io::Result<()> {
pub fn flush(&mut self) -> io::Result<()> {
let mut file = File::create(self.path.clone())?;
for elmt in &self.removed_tmp {
match self.removed.binary_search(&elmt) {
@ -283,7 +282,7 @@ impl RemoveLog {
}
/// Discard pending changes
fn discard(&mut self) {
pub fn discard(&mut self) {
if self.removed_bak.len() > 0 {
self.removed = self.removed_bak.clone();
self.removed_bak = vec![];
@ -292,12 +291,30 @@ impl RemoveLog {
}
/// Whether the remove log currently includes the provided position.
fn includes(&self, elmt: u64) -> bool {
pub fn includes(&self, elmt: u64) -> bool {
include_tuple(&self.removed, elmt) || include_tuple(&self.removed_tmp, elmt)
}
/// How many removed positions exist before this particular position
pub fn get_shift(&self, elmt: u64) -> usize {
let mut complete_list = self.removed.clone();
for e in &self.removed_tmp {
match self.removed.binary_search(&e) {
Ok(_) => continue,
Err(idx) => {
complete_list.insert(idx, *e);
}
}
}
let pos = match complete_list.binary_search(&(elmt,0)){
Ok(idx) => idx+1,
Err(idx) => idx,
};
complete_list.split_at(pos).0.len()
}
/// Number of positions stored in the remove log.
fn len(&self) -> usize {
pub fn len(&self) -> usize {
self.removed.len()
}
}
@ -311,216 +328,8 @@ fn include_tuple(v: &Vec<(u64, u32)>, e: u64) -> bool {
false
}
/// PMMR persistent backend implementation. Relies on multiple facilities to
/// handle writing, reading and pruning.
///
/// * A main storage file appends HashSum instances as they come. This
/// AppendOnlyFile is also backed by a mmap for reads.
/// * An in-memory backend buffers the latest batch of writes to ensure the
/// PMMR can always read recent values even if they haven't been flushed to
/// disk yet.
/// * A remove log tracks the positions that need to be pruned from the
/// main storage file.
pub struct PMMRBackend<T>
where
T: Summable + Clone,
{
data_dir: String,
hashsum_file: AppendOnlyFile,
remove_log: RemoveLog,
pruned_nodes: pmmr::PruneList,
phantom: PhantomData<T>,
}
impl<T> Backend<T> for PMMRBackend<T>
where
T: Summable + Clone,
{
/// Append the provided HashSums to the backend storage.
#[allow(unused_variables)]
fn append(&mut self, position: u64, data: Vec<HashSum<T>>) -> Result<(), String> {
for d in data {
self.hashsum_file.append(&mut ser::ser_vec(&d).unwrap());
}
Ok(())
}
/// Get a HashSum by insertion position
fn get(&self, position: u64) -> Option<HashSum<T>> {
// Check if this position has been pruned in the remove log or the
// pruned list
if self.remove_log.includes(position) {
return None;
}
let shift = self.pruned_nodes.get_shift(position);
if let None = shift {
return None;
}
// The MMR starts at 1, our binary backend starts at 0
let pos = position - 1;
// Must be on disk, doing a read at the correct position
let record_len = 32 + T::sum_len();
let file_offset = ((pos - shift.unwrap()) as usize) * record_len;
let data = self.hashsum_file.read(file_offset, record_len);
match ser::deserialize(&mut &data[..]) {
Ok(hashsum) => Some(hashsum),
Err(e) => {
error!(
LOGGER,
"Corrupted storage, could not read an entry from sum tree store: {:?}",
e
);
None
}
}
}
fn rewind(&mut self, position: u64, index: u32) -> Result<(), String> {
self.remove_log
.rewind(index)
.map_err(|e| format!("Could not truncate remove log: {}", e))?;
let shift = self.pruned_nodes.get_shift(position).unwrap_or(0);
let record_len = 32 + T::sum_len();
let file_pos = (position - shift) * (record_len as u64);
self.hashsum_file.rewind(file_pos);
Ok(())
}
/// Remove HashSums by insertion position
fn remove(&mut self, positions: Vec<u64>, index: u32) -> Result<(), String> {
self.remove_log.append(positions, index).map_err(|e| {
format!("Could not write to log storage, disk full? {:?}", e)
})
}
}
impl<T> PMMRBackend<T>
where
T: Summable + Clone,
{
/// Instantiates a new PMMR backend that will use the provided directly to
/// store its files.
pub fn new(data_dir: String) -> io::Result<PMMRBackend<T>> {
let hs_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_DATA_FILE))?;
let rm_log = RemoveLog::open(format!("{}/{}", data_dir, PMMR_RM_LOG_FILE))?;
let prune_list = read_ordered_vec(format!("{}/{}", data_dir, PMMR_PRUNED_FILE), 8)?;
Ok(PMMRBackend {
data_dir: data_dir,
hashsum_file: hs_file,
remove_log: rm_log,
pruned_nodes: pmmr::PruneList {
pruned_nodes: prune_list,
},
phantom: PhantomData,
})
}
/// Total size of the PMMR stored by this backend. Only produces the fully
/// sync'd size.
pub fn unpruned_size(&self) -> io::Result<u64> {
let total_shift = self.pruned_nodes.get_shift(::std::u64::MAX).unwrap();
let record_len = 32 + T::sum_len() as u64;
let sz = self.hashsum_file.size()?;
Ok(sz / record_len + total_shift)
}
/// Syncs all files to disk. A call to sync is required to ensure all the
/// data has been successfully written to disk.
pub fn sync(&mut self) -> io::Result<()> {
if let Err(e) = self.hashsum_file.flush() {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
format!("Could not write to log storage, disk full? {:?}", e),
));
}
self.remove_log.flush()?;
Ok(())
}
/// Discard the current, non synced state of the backend.
pub fn discard(&mut self) {
self.hashsum_file.discard();
self.remove_log.discard();
}
/// Checks the length of the remove log to see if it should get compacted.
/// If so, the remove log is flushed into the pruned list, which itself gets
/// saved, and the main hashsum data file is rewritten, cutting the removed
/// data.
///
/// If a max_len strictly greater than 0 is provided, the value will be used
/// to decide whether the remove log has reached its maximum length,
/// otherwise the RM_LOG_MAX_NODES default value is used.
///
/// TODO whatever is calling this should also clean up the commit to
/// position index in db
pub fn check_compact(&mut self, max_len: usize) -> io::Result<()> {
if !(max_len > 0 && self.remove_log.len() > max_len
|| max_len == 0 && self.remove_log.len() > RM_LOG_MAX_NODES)
{
return Ok(());
}
// 0. validate none of the nodes in the rm log are in the prune list (to
// avoid accidental double compaction)
for pos in &self.remove_log.removed[..] {
if let None = self.pruned_nodes.pruned_pos(pos.0) {
// TODO we likely can recover from this by directly jumping to 3
error!(
LOGGER,
"The remove log contains nodes that are already in the pruned \
list, a previous compaction likely failed."
);
return Ok(());
}
}
// 1. save hashsum file to a compact copy, skipping data that's in the
// remove list
let tmp_prune_file = format!("{}/{}.prune", self.data_dir, PMMR_DATA_FILE);
let record_len = (32 + T::sum_len()) as u64;
let to_rm = self.remove_log
.removed
.iter()
.map(|&(pos, _)| {
let shift = self.pruned_nodes.get_shift(pos);
(pos - 1 - shift.unwrap()) * record_len
})
.collect();
self.hashsum_file
.save_prune(tmp_prune_file.clone(), to_rm, record_len)?;
// 2. update the prune list and save it in place
for &(rm_pos, _) in &self.remove_log.removed[..] {
self.pruned_nodes.add(rm_pos);
}
write_vec(
format!("{}/{}", self.data_dir, PMMR_PRUNED_FILE),
&self.pruned_nodes.pruned_nodes,
)?;
// 3. move the compact copy to the hashsum file and re-open it
fs::rename(
tmp_prune_file.clone(),
format!("{}/{}", self.data_dir, PMMR_DATA_FILE),
)?;
self.hashsum_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE))?;
// 4. truncate the rm log
self.remove_log.rewind(0)?;
self.remove_log.flush()?;
Ok(())
}
}
// Read an ordered vector of scalars from a file.
fn read_ordered_vec<T>(path: String, elmt_len: usize) -> io::Result<Vec<T>>
/// Read an ordered vector of scalars from a file.
pub fn read_ordered_vec<T>(path: String, elmt_len: usize) -> io::Result<Vec<T>>
where
T: ser::Readable + cmp::Ord,
{
@ -557,7 +366,8 @@ where
Ok(ovec)
}
fn write_vec<T>(path: String, v: &Vec<T>) -> io::Result<()>
/// Writes an ordered vector to a file
pub fn write_vec<T>(path: String, v: &Vec<T>) -> io::Result<()>
where
T: ser::Writeable,
{

126
store/tests/sumtree.rs → store/tests/pmmr.rs
View file

@ -20,13 +20,13 @@ extern crate time;
use std::fs;
use core::ser::*;
use core::core::pmmr::{Backend, HashSum, Summable, PMMR};
use core::core::hash::Hashed;
use core::core::pmmr::{PMMR, Backend};
use core::core::hash::{Hash, Hashed};
#[test]
fn sumtree_append() {
fn pmmr_append() {
let (data_dir, elems) = setup("append");
let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
// adding first set of 4 elements and sync
let mut mmr_size = load(0, &elems[0..4], &mut backend);
@ -37,31 +37,22 @@ fn sumtree_append() {
backend.sync().unwrap();
// check the resulting backend store and the computation of the root
let hash = Hashed::hash(&elems[0].clone());
let sum = elems[0].sum();
let node_hash = (1 as u64, &sum, hash).hash();
let node_hash = elems[0].hash();
assert_eq!(
backend.get(1),
Some(HashSum {
hash: node_hash,
sum: sum,
})
backend.get(1, false).expect("").0,
node_hash
);
let sum2 = HashSum::from_summable(1, &elems[0])
+ HashSum::from_summable(2, &elems[1]);
let sum2 = elems[0].hash() + elems[1].hash();
let sum4 = sum2
+ (HashSum::from_summable(4, &elems[2])
+ HashSum::from_summable(5, &elems[3]));
+ (elems[2].hash() + elems[3].hash());
let sum8 = sum4
+ ((HashSum::from_summable(8, &elems[4])
+ HashSum::from_summable(9, &elems[5]))
+ (HashSum::from_summable(11, &elems[6])
+ HashSum::from_summable(12, &elems[7])));
let sum9 = sum8 + HashSum::from_summable(16, &elems[8]);
+ ((elems[4].hash() + elems[5].hash())
+ (elems[6].hash() + elems[7].hash()));
let sum9 = sum8 + elems[8].hash();
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(pmmr.root(), sum9);
}
@ -69,75 +60,81 @@ fn sumtree_append() {
}
#[test]
fn sumtree_prune_compact() {
fn pmmr_prune_compact() {
let (data_dir, elems) = setup("prune_compact");
// setup the mmr store with all elements
let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
// save the root
let root: HashSum<TestElem>;
let root: Hash;
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
root = pmmr.root();
}
// pruning some choice nodes
{
let mut pmmr = PMMR::at(&mut backend, mmr_size);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.prune(1, 1).unwrap();
pmmr.prune(4, 1).unwrap();
pmmr.prune(5, 1).unwrap();
}
backend.sync().unwrap();
// check the root
// check the root and stored data
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(root, pmmr.root());
// check we can still retrieve same element from leaf index 2
assert_eq!(pmmr.get(2, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 2]));
}
// compact
backend.check_compact(2).unwrap();
// recheck the root
// recheck the root and stored data
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(root, pmmr.root());
assert_eq!(pmmr.get(2, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 2]));
assert_eq!(pmmr.get(11, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 7]));
}
teardown(data_dir);
}
#[test]
fn sumtree_reload() {
fn pmmr_reload() {
let (data_dir, elems) = setup("reload");
// set everything up with a first backend
let mmr_size: u64;
let root: HashSum<TestElem>;
let root: Hash;
{
let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
// save the root and prune some nodes so we have prune data
{
let mut pmmr = PMMR::at(&mut backend, mmr_size);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.dump(false);
root = pmmr.root();
pmmr.prune(1, 1).unwrap();
pmmr.prune(4, 1).unwrap();
}
backend.sync().unwrap();
backend.check_compact(1).unwrap();
backend.sync().unwrap();
assert_eq!(backend.unpruned_size().unwrap(), mmr_size);
// prune some more to get rm log data
{
let mut pmmr = PMMR::at(&mut backend, mmr_size);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.prune(5, 1).unwrap();
}
backend.sync().unwrap();
@ -146,37 +143,38 @@ fn sumtree_reload() {
// create a new backend and check everything is kosher
{
let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
let mut backend:store::pmmr::PMMRBackend<TestElem> =
store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
assert_eq!(backend.unpruned_size().unwrap(), mmr_size);
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(root, pmmr.root());
}
assert_eq!(backend.get(5), None);
assert_eq!(backend.get(5, false), None);
}
teardown(data_dir);
}
#[test]
fn sumtree_rewind() {
fn pmmr_rewind() {
let (data_dir, elems) = setup("rewind");
let mut backend = store::sumtree::PMMRBackend::new(data_dir.clone()).unwrap();
let mut backend = store::pmmr::PMMRBackend::new(data_dir.clone()).unwrap();
// adding elements and keeping the corresponding root
let mut mmr_size = load(0, &elems[0..4], &mut backend);
backend.sync().unwrap();
let root1: HashSum<TestElem>;
let root1: Hash;
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
root1 = pmmr.root();
}
mmr_size = load(mmr_size, &elems[4..6], &mut backend);
backend.sync().unwrap();
let root2: HashSum<TestElem>;
let root2: Hash;
{
let pmmr = PMMR::at(&mut backend, mmr_size);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
root2 = pmmr.root();
}
@ -185,7 +183,7 @@ fn sumtree_rewind() {
// prune and compact the 2 first elements to spice things up
{
let mut pmmr = PMMR::at(&mut backend, mmr_size);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.prune(1, 1).unwrap();
pmmr.prune(2, 1).unwrap();
}
@ -194,24 +192,24 @@ fn sumtree_rewind() {
// rewind and check the roots still match
{
let mut pmmr = PMMR::at(&mut backend, mmr_size);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.rewind(9, 3).unwrap();
assert_eq!(pmmr.root(), root2);
}
backend.sync().unwrap();
{
let pmmr = PMMR::at(&mut backend, 10);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, 10);
assert_eq!(pmmr.root(), root2);
}
{
let mut pmmr = PMMR::at(&mut backend, 10);
let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, 10);
pmmr.rewind(5, 3).unwrap();
assert_eq!(pmmr.root(), root1);
}
backend.sync().unwrap();
{
let pmmr = PMMR::at(&mut backend, 7);
let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, 7);
assert_eq!(pmmr.root(), root1);
}
@ -242,7 +240,7 @@ fn teardown(data_dir: String) {
fs::remove_dir_all(data_dir).unwrap();
}
fn load(pos: u64, elems: &[TestElem], backend: &mut store::sumtree::PMMRBackend<TestElem>) -> u64 {
fn load(pos: u64, elems: &[TestElem], backend: &mut store::pmmr::PMMRBackend<TestElem>) -> u64 {
let mut pmmr = PMMR::at(backend, pos);
for elem in elems {
pmmr.push(elem.clone()).unwrap();
@ -252,16 +250,10 @@ fn load(pos: u64, elems: &[TestElem], backend: &mut store::sumtree::PMMRBackend<
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
struct TestElem([u32; 4]);
impl Summable for TestElem {
type Sum = u64;
fn sum(&self) -> u64 {
// sums are not allowed to overflow, so we use this simple
// non-injective "sum" function that will still be homomorphic
self.0[0] as u64 * 0x1000 + self.0[1] as u64 * 0x100 + self.0[2] as u64 * 0x10
+ self.0[3] as u64
}
fn sum_len() -> usize {
8
impl PMMRable for TestElem {
fn len() -> usize {
16
}
}
@ -273,3 +265,15 @@ impl Writeable for TestElem {
writer.write_u32(self.0[3])
}
}
impl Readable for TestElem {
fn read(reader: &mut Reader) -> Result<TestElem, Error> {
Ok(TestElem (
[
reader.read_u32()?,
reader.read_u32()?,
reader.read_u32()?,
reader.read_u32()?,
]
))
}
}