From 05d1c6c8171177d0981da571dcbbd29b2f24e8eb Mon Sep 17 00:00:00 2001
From: Yeastplume <yeastplume@gmail.com>
Date: Thu, 22 Feb 2018 13:45:13 +0000
Subject: [PATCH] Take the 'Sum' out of 'Sumtree' (#702)
* beginning to remove sum
* continuing to remove sumtree sums
* finished removing sums from pmmr core
* renamed sumtree files, and completed changes+test updates in core and store
* updating grin/chain to include removelogs
* integration of flatfile structure, changes to chain/sumtree to start using them
* tests on chain, core and store passing
* cleaning up api and tests
* formatting
* flatfiles stored as part of PMMR backend instead
* all compiling and tests running
* documentation
* added remove + pruning to flatfiles
* remove unneeded enum
* adding sumtree root struct
---
api/src/handlers.rs | 32 +--
api/src/types.rs | 37 +--
chain/src/chain.rs | 31 +-
chain/src/pipe.rs | 12 +-
chain/src/sumtree.rs | 254 +++++++++--------
chain/src/types.rs | 13 +
chain/tests/mine_simple_chain.rs | 11 +
core/src/core/hash.rs | 10 +-
core/src/core/pmmr.rs | 424 +++++++++++-----------------
core/src/core/transaction.rs | 163 +++--------
core/src/ser.rs | 20 +-
grin/tests/api.rs | 26 +-
keychain/src/keychain.rs | 4 +-
store/src/lib.rs | 3 +-
store/src/pmmr.rs | 318 +++++++++++++++++++++
store/src/{sumtree.rs => types.rs} | 288 ++++---------------
store/tests/{sumtree.rs => pmmr.rs} | 126 +++++----
17 files changed, 893 insertions(+), 879 deletions(-)
create mode 100644 store/src/pmmr.rs
rename store/src/{sumtree.rs => types.rs} (54%)
rename store/tests/{sumtree.rs => pmmr.rs} (62%)
diff --git a/api/src/handlers.rs b/api/src/handlers.rs
index e79794064..acefd9ad5 100644
--- a/api/src/handlers.rs
+++ b/api/src/handlers.rs
@@ -206,13 +206,13 @@ impl Handler for UtxoHandler {
}
// Sum tree handler. Retrieve the roots:
-// GET /v1/sumtrees/roots
+// GET /v1/pmmrtrees/roots
//
// Last inserted nodes::
-// GET /v1/sumtrees/lastutxos (gets last 10)
-// GET /v1/sumtrees/lastutxos?n=5
-// GET /v1/sumtrees/lastrangeproofs
-// GET /v1/sumtrees/lastkernels
+// GET /v1/pmmrtrees/lastutxos (gets last 10)
+// GET /v1/pmmrtrees/lastutxos?n=5
+// GET /v1/pmmrtrees/lastrangeproofs
+// GET /v1/pmmrtrees/lastkernels
struct SumTreeHandler {
chain: Weak<chain::Chain>,
}
@@ -224,18 +224,18 @@ impl SumTreeHandler {
}
// gets last n utxos inserted in to the tree
- fn get_last_n_utxo(&self, distance: u64) -> Vec<SumTreeNode> {
- SumTreeNode::get_last_n_utxo(w(&self.chain), distance)
+ fn get_last_n_utxo(&self, distance: u64) -> Vec<PmmrTreeNode> {
+ PmmrTreeNode::get_last_n_utxo(w(&self.chain), distance)
}
// gets last n utxos inserted in to the tree
- fn get_last_n_rangeproof(&self, distance: u64) -> Vec<SumTreeNode> {
- SumTreeNode::get_last_n_rangeproof(w(&self.chain), distance)
+ fn get_last_n_rangeproof(&self, distance: u64) -> Vec<PmmrTreeNode> {
+ PmmrTreeNode::get_last_n_rangeproof(w(&self.chain), distance)
}
// gets last n utxos inserted in to the tree
- fn get_last_n_kernel(&self, distance: u64) -> Vec<SumTreeNode> {
- SumTreeNode::get_last_n_kernel(w(&self.chain), distance)
+ fn get_last_n_kernel(&self, distance: u64) -> Vec<PmmrTreeNode> {
+ PmmrTreeNode::get_last_n_kernel(w(&self.chain), distance)
}
}
@@ -620,10 +620,10 @@ pub fn start_rest_apis<T>(
"get chain".to_string(),
"get chain/utxos".to_string(),
"get status".to_string(),
- "get sumtrees/roots".to_string(),
- "get sumtrees/lastutxos?n=10".to_string(),
- "get sumtrees/lastrangeproofs".to_string(),
- "get sumtrees/lastkernels".to_string(),
+ "get pmmrtrees/roots".to_string(),
+ "get pmmrtrees/lastutxos?n=10".to_string(),
+ "get pmmrtrees/lastrangeproofs".to_string(),
+ "get pmmrtrees/lastkernels".to_string(),
"get pool".to_string(),
"post pool/push".to_string(),
"post peers/a.b.c.d:p/ban".to_string(),
@@ -641,7 +641,7 @@ pub fn start_rest_apis<T>(
chain_tip: get "/chain" => chain_tip_handler,
chain_utxos: get "/chain/utxos/*" => utxo_handler,
status: get "/status" => status_handler,
- sumtree_roots: get "/sumtrees/*" => sumtree_handler,
+ sumtree_roots: get "/pmmrtrees/*" => sumtree_handler,
pool_info: get "/pool" => pool_info_handler,
pool_push: post "/pool/push" => pool_push_handler,
peers_all: get "/peers/all" => peers_all_handler,
diff --git a/api/src/types.rs b/api/src/types.rs
index eecac01ea..59dc7b8a7 100644
--- a/api/src/types.rs
+++ b/api/src/types.rs
@@ -16,7 +16,6 @@ use std::sync::Arc;
use core::{core, ser};
use core::core::hash::Hashed;
-use core::core::SumCommit;
use core::core::SwitchCommitHash;
use chain;
use p2p;
@@ -89,8 +88,6 @@ impl Status {
pub struct SumTrees {
/// UTXO Root Hash
pub utxo_root_hash: String,
- // UTXO Root Sum
- pub utxo_root_sum: String,
// Rangeproof root hash
pub range_proof_root_hash: String,
// Kernel set root hash
@@ -101,10 +98,9 @@ impl SumTrees {
pub fn from_head(head: Arc<chain::Chain>) -> SumTrees {
let roots = head.get_sumtree_roots();
SumTrees {
- utxo_root_hash: roots.0.hash.to_hex(),
- utxo_root_sum: roots.0.sum.to_hex(),
- range_proof_root_hash: roots.1.hash.to_hex(),
- kernel_root_hash: roots.2.hash.to_hex(),
+ utxo_root_hash: roots.0.to_hex(),
+ range_proof_root_hash: roots.1.to_hex(),
+ kernel_root_hash: roots.2.to_hex(),
}
}
}
@@ -112,45 +108,40 @@ impl SumTrees {
/// Wrapper around a list of sumtree nodes, so it can be
/// presented properly via json
#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct SumTreeNode {
+pub struct PmmrTreeNode {
// The hash
pub hash: String,
- // SumCommit (features|commitment), optional (only for utxos)
- pub sum: Option<SumCommit>,
}
-impl SumTreeNode {
- pub fn get_last_n_utxo(chain: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
+impl PmmrTreeNode {
+ pub fn get_last_n_utxo(chain: Arc<chain::Chain>, distance: u64) -> Vec<PmmrTreeNode> {
let mut return_vec = Vec::new();
let last_n = chain.get_last_n_utxo(distance);
for x in last_n {
- return_vec.push(SumTreeNode {
- hash: util::to_hex(x.hash.to_vec()),
- sum: Some(x.sum),
+ return_vec.push(PmmrTreeNode {
+ hash: util::to_hex(x.0.to_vec()),
});
}
return_vec
}
- pub fn get_last_n_rangeproof(head: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
+ pub fn get_last_n_rangeproof(head: Arc<chain::Chain>, distance: u64) -> Vec<PmmrTreeNode> {
let mut return_vec = Vec::new();
let last_n = head.get_last_n_rangeproof(distance);
for elem in last_n {
- return_vec.push(SumTreeNode {
- hash: util::to_hex(elem.hash.to_vec()),
- sum: None,
+ return_vec.push(PmmrTreeNode {
+ hash: util::to_hex(elem.0.to_vec()),
});
}
return_vec
}
- pub fn get_last_n_kernel(head: Arc<chain::Chain>, distance: u64) -> Vec<SumTreeNode> {
+ pub fn get_last_n_kernel(head: Arc<chain::Chain>, distance: u64) -> Vec<PmmrTreeNode> {
let mut return_vec = Vec::new();
let last_n = head.get_last_n_kernel(distance);
for elem in last_n {
- return_vec.push(SumTreeNode {
- hash: util::to_hex(elem.hash.to_vec()),
- sum: None,
+ return_vec.push(PmmrTreeNode {
+ hash: util::to_hex(elem.0.to_vec()),
});
}
return_vec
diff --git a/chain/src/chain.rs b/chain/src/chain.rs
index 5ad68dafd..f4077b54a 100644
--- a/chain/src/chain.rs
+++ b/chain/src/chain.rs
@@ -20,21 +20,18 @@ use std::fs::File;
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
-use util::secp::pedersen::RangeProof;
-
-use core::core::{Input, OutputIdentifier, SumCommit};
-use core::core::hash::Hashed;
-use core::core::pmmr::{HashSum, NoSum};
+use core::core::{Input, OutputIdentifier, OutputStoreable, TxKernel};
+use core::core::hash::{Hash, Hashed};
use core::global;
-use core::core::{Block, BlockHeader, TxKernel};
+use core::core::{Block, BlockHeader};
use core::core::target::Difficulty;
-use core::core::hash::Hash;
use grin_store::Error::NotFoundErr;
use pipe;
use store;
use sumtree;
use types::*;
+use util::secp::pedersen::RangeProof;
use util::LOGGER;
@@ -428,9 +425,9 @@ impl Chain {
Ok(extension.roots())
})?;
- b.header.utxo_root = roots.0.hash;
- b.header.range_proof_root = roots.1.hash;
- b.header.kernel_root = roots.2.hash;
+ b.header.utxo_root = roots.utxo_root;
+ b.header.range_proof_root = roots.rproof_root;
+ b.header.kernel_root = roots.kernel_root;
Ok(())
}
@@ -438,9 +435,9 @@ impl Chain {
pub fn get_sumtree_roots(
&self,
) -> (
- HashSum<SumCommit>,
- HashSum<NoSum<RangeProof>>,
- HashSum<NoSum<TxKernel>>,
+ Hash,
+ Hash,
+ Hash,
) {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.roots()
@@ -507,7 +504,7 @@ impl Chain {
{
let mut head = self.head.lock().unwrap();
*head = Tip::from_block(&header);
- self.store.save_body_head(&head)?;
+ let _ = self.store.save_body_head(&head);
self.store.save_header_height(&header)?;
}
@@ -517,19 +514,19 @@ impl Chain {
}
/// returns the last n nodes inserted into the utxo sum tree
- pub fn get_last_n_utxo(&self, distance: u64) -> Vec<HashSum<SumCommit>> {
+ pub fn get_last_n_utxo(&self, distance: u64) -> Vec<(Hash, Option<OutputStoreable>)> {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.last_n_utxo(distance)
}
/// as above, for rangeproofs
- pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<HashSum<NoSum<RangeProof>>> {
+ pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<(Hash, Option<RangeProof>)> {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.last_n_rangeproof(distance)
}
/// as above, for kernels
- pub fn get_last_n_kernel(&self, distance: u64) -> Vec<HashSum<NoSum<TxKernel>>> {
+ pub fn get_last_n_kernel(&self, distance: u64) -> Vec<(Hash, Option<TxKernel>)> {
let mut sumtrees = self.sumtrees.write().unwrap();
sumtrees.last_n_kernel(distance)
}
diff --git a/chain/src/pipe.rs b/chain/src/pipe.rs
index d8b3aead2..58c3cc5c5 100644
--- a/chain/src/pipe.rs
+++ b/chain/src/pipe.rs
@@ -305,28 +305,28 @@ fn validate_block(
// apply the new block to the MMR trees and check the new root hashes
ext.apply_block(&b)?;
- let (utxo_root, rproof_root, kernel_root) = ext.roots();
- if utxo_root.hash != b.header.utxo_root || rproof_root.hash != b.header.range_proof_root
- || kernel_root.hash != b.header.kernel_root
+ let roots = ext.roots();
+ if roots.utxo_root != b.header.utxo_root || roots.rproof_root != b.header.range_proof_root
+ || roots.kernel_root != b.header.kernel_root
{
ext.dump(false);
debug!(
LOGGER,
"validate_block: utxo roots - {:?}, {:?}",
- utxo_root.hash,
+ roots.utxo_root,
b.header.utxo_root,
);
debug!(
LOGGER,
"validate_block: rproof roots - {:?}, {:?}",
- rproof_root.hash,
+ roots.rproof_root,
b.header.range_proof_root,
);
debug!(
LOGGER,
"validate_block: kernel roots - {:?}, {:?}",
- kernel_root.hash,
+ roots.kernel_root,
b.header.kernel_root,
);
diff --git a/chain/src/sumtree.rs b/chain/src/sumtree.rs
index 3abdceba6..3a86c51be 100644
--- a/chain/src/sumtree.rs
+++ b/chain/src/sumtree.rs
@@ -1,4 +1,3 @@
-
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -26,26 +25,26 @@ use util::static_secp_instance;
use util::secp::pedersen::{RangeProof, Commitment};
use core::consensus::reward;
-use core::core::{Block, BlockHeader, SumCommit, Input, Output, OutputIdentifier, OutputFeatures, TxKernel};
-use core::core::pmmr::{self, HashSum, NoSum, Summable, PMMR};
-use core::core::hash::Hashed;
-use core::ser;
+use core::core::{Block, BlockHeader, Input, Output, OutputIdentifier,
+ OutputFeatures, OutputStoreable, TxKernel};
+use core::core::pmmr::{self, PMMR};
+use core::core::hash::{Hash, Hashed};
+use core::ser::{self, PMMRable};
+
use grin_store;
-use grin_store::sumtree::{PMMRBackend, AppendOnlyFile};
-use types::ChainStore;
-use types::Error;
+use grin_store::pmmr::PMMRBackend;
+use types::{ChainStore, SumTreeRoots, Error};
use util::{LOGGER, zip};
const SUMTREES_SUBDIR: &'static str = "sumtrees";
const UTXO_SUBDIR: &'static str = "utxo";
const RANGE_PROOF_SUBDIR: &'static str = "rangeproof";
const KERNEL_SUBDIR: &'static str = "kernel";
-const KERNEL_FILE: &'static str = "kernel_full_data.bin";
const SUMTREES_ZIP: &'static str = "sumtrees_snapshot.zip";
struct PMMRHandle<T>
where
- T: Summable + Clone,
+ T: PMMRable,
{
backend: PMMRBackend<T>,
last_pos: u64,
@@ -53,7 +52,7 @@ where
impl<T> PMMRHandle<T>
where
- T: Summable + Clone,
+ T: PMMRable,
{
fn new(root_dir: String, file_name: &str) -> Result<PMMRHandle<T>, Error> {
let path = Path::new(&root_dir).join(SUMTREES_SUBDIR).join(file_name);
@@ -70,20 +69,17 @@ where
/// An easy to manipulate structure holding the 3 sum trees necessary to
/// validate blocks and capturing the UTXO set, the range proofs and the
/// kernels. Also handles the index of Commitments to positions in the
-/// output and range proof sum trees.
+/// output and range proof pmmr trees.
///
/// Note that the index is never authoritative, only the trees are
/// guaranteed to indicate whether an output is spent or not. The index
/// may have commitments that have already been spent, even with
/// pruning enabled.
-///
-/// In addition of the sumtrees, this maintains the full list of kernel
-/// data so it can be easily packaged for sync or validation.
+
pub struct SumTrees {
- output_pmmr_h: PMMRHandle<SumCommit>,
- rproof_pmmr_h: PMMRHandle<NoSum<RangeProof>>,
- kernel_pmmr_h: PMMRHandle<NoSum<TxKernel>>,
- kernel_file: AppendOnlyFile,
+ utxo_pmmr_h: PMMRHandle<OutputStoreable>,
+ rproof_pmmr_h: PMMRHandle<RangeProof>,
+ kernel_pmmr_h: PMMRHandle<TxKernel>,
// chain store used as index of commitments to MMR positions
commit_index: Arc<ChainStore>,
@@ -92,16 +88,20 @@ pub struct SumTrees {
impl SumTrees {
/// Open an existing or new set of backends for the SumTrees
pub fn open(root_dir: String, commit_index: Arc<ChainStore>) -> Result<SumTrees, Error> {
- let mut kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect();
+
+ let utxo_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, UTXO_SUBDIR].iter().collect();
+ fs::create_dir_all(utxo_file_path.clone())?;
+
+ let rproof_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, RANGE_PROOF_SUBDIR].iter().collect();
+ fs::create_dir_all(rproof_file_path.clone())?;
+
+ let kernel_file_path: PathBuf = [&root_dir, SUMTREES_SUBDIR, KERNEL_SUBDIR].iter().collect();
fs::create_dir_all(kernel_file_path.clone())?;
- kernel_file_path.push(KERNEL_FILE);
- let kernel_file = AppendOnlyFile::open(kernel_file_path.to_str().unwrap().to_owned())?;
Ok(SumTrees {
- output_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?,
+ utxo_pmmr_h: PMMRHandle::new(root_dir.clone(), UTXO_SUBDIR)?,
rproof_pmmr_h: PMMRHandle::new(root_dir.clone(), RANGE_PROOF_SUBDIR)?,
kernel_pmmr_h: PMMRHandle::new(root_dir.clone(), KERNEL_SUBDIR)?,
- kernel_file: kernel_file,
commit_index: commit_index,
})
}
@@ -109,19 +109,19 @@ impl SumTrees {
/// Check is an output is unspent.
/// We look in the index to find the output MMR pos.
/// Then we check the entry in the output MMR and confirm the hash matches.
- pub fn is_unspent(&mut self, output: &OutputIdentifier) -> Result<(), Error> {
- match self.commit_index.get_output_pos(&output.commit) {
+ pub fn is_unspent(&mut self, output_id: &OutputIdentifier) -> Result<(), Error> {
+ match self.commit_index.get_output_pos(&output_id.commit) {
Ok(pos) => {
- let output_pmmr = PMMR::at(
- &mut self.output_pmmr_h.backend,
- self.output_pmmr_h.last_pos,
+ let output_pmmr:PMMR<OutputStoreable, _> = PMMR::at(
+ &mut self.utxo_pmmr_h.backend,
+ self.utxo_pmmr_h.last_pos,
);
- if let Some(HashSum { hash, sum: _ }) = output_pmmr.get(pos) {
- let sum_commit = output.as_sum_commit();
- let hash_sum = HashSum::from_summable(pos, &sum_commit);
- if hash == hash_sum.hash {
+ if let Some((hash, _)) = output_pmmr.get(pos, false) {
+ println!("Getting output ID hash");
+ if hash == output_id.hash() {
Ok(())
} else {
+ println!("MISMATCH BECAUSE THE BLOODY THING MISMATCHES");
Err(Error::SumTreeErr(format!("sumtree hash mismatch")))
}
} else {
@@ -164,20 +164,21 @@ impl SumTrees {
/// returns the last N nodes inserted into the tree (i.e. the 'bottom'
/// nodes at level 0
- pub fn last_n_utxo(&mut self, distance: u64) -> Vec<HashSum<SumCommit>> {
- let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
- output_pmmr.get_last_n_insertions(distance)
+ /// TODO: These need to return the actual data from the flat-files instead of hashes now
+ pub fn last_n_utxo(&mut self, distance: u64) -> Vec<(Hash, Option<OutputStoreable>)> {
+ let utxo_pmmr:PMMR<OutputStoreable, _> = PMMR::at(&mut self.utxo_pmmr_h.backend, self.utxo_pmmr_h.last_pos);
+ utxo_pmmr.get_last_n_insertions(distance)
}
/// as above, for range proofs
- pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<HashSum<NoSum<RangeProof>>> {
- let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+ pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<(Hash, Option<RangeProof>)> {
+ let rproof_pmmr:PMMR<RangeProof, _> = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
rproof_pmmr.get_last_n_insertions(distance)
}
/// as above, for kernels
- pub fn last_n_kernel(&mut self, distance: u64) -> Vec<HashSum<NoSum<TxKernel>>> {
- let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+ pub fn last_n_kernel(&mut self, distance: u64) -> Vec<(Hash, Option<TxKernel>)> {
+ let kernel_pmmr:PMMR<TxKernel, _> = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
kernel_pmmr.get_last_n_insertions(distance)
}
@@ -186,17 +187,19 @@ impl SumTrees {
indexes_at(block, self.commit_index.deref())
}
+
/// Get sum tree roots
+ /// TODO: Return data instead of hashes
pub fn roots(
&mut self,
) -> (
- HashSum<SumCommit>,
- HashSum<NoSum<RangeProof>>,
- HashSum<NoSum<TxKernel>>,
+ Hash,
+ Hash,
+ Hash,
) {
- let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
- let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
- let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+ let output_pmmr:PMMR<OutputStoreable, _> = PMMR::at(&mut self.utxo_pmmr_h.backend, self.utxo_pmmr_h.last_pos);
+ let rproof_pmmr:PMMR<RangeProof, _> = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+ let kernel_pmmr:PMMR<TxKernel, _> = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
(output_pmmr.root(), rproof_pmmr.root(), kernel_pmmr.root())
}
}
@@ -231,26 +234,23 @@ where
match res {
Err(e) => {
debug!(LOGGER, "Error returned, discarding sumtree extension.");
- trees.output_pmmr_h.backend.discard();
+ trees.utxo_pmmr_h.backend.discard();
trees.rproof_pmmr_h.backend.discard();
trees.kernel_pmmr_h.backend.discard();
- trees.kernel_file.discard();
Err(e)
}
Ok(r) => {
if rollback {
debug!(LOGGER, "Rollbacking sumtree extension.");
- trees.output_pmmr_h.backend.discard();
+ trees.utxo_pmmr_h.backend.discard();
trees.rproof_pmmr_h.backend.discard();
trees.kernel_pmmr_h.backend.discard();
- trees.kernel_file.discard();
} else {
debug!(LOGGER, "Committing sumtree extension.");
- trees.output_pmmr_h.backend.sync()?;
+ trees.utxo_pmmr_h.backend.sync()?;
trees.rproof_pmmr_h.backend.sync()?;
trees.kernel_pmmr_h.backend.sync()?;
- trees.kernel_file.flush()?;
- trees.output_pmmr_h.last_pos = sizes.0;
+ trees.utxo_pmmr_h.last_pos = sizes.0;
trees.rproof_pmmr_h.last_pos = sizes.1;
trees.kernel_pmmr_h.last_pos = sizes.2;
}
@@ -265,11 +265,10 @@ where
/// reversible manner within a unit of work provided by the `extending`
/// function.
pub struct Extension<'a> {
- output_pmmr: PMMR<'a, SumCommit, PMMRBackend<SumCommit>>,
- rproof_pmmr: PMMR<'a, NoSum<RangeProof>, PMMRBackend<NoSum<RangeProof>>>,
- kernel_pmmr: PMMR<'a, NoSum<TxKernel>, PMMRBackend<NoSum<TxKernel>>>,
+ utxo_pmmr: PMMR<'a, OutputStoreable, PMMRBackend<OutputStoreable>>,
+ rproof_pmmr: PMMR<'a, RangeProof, PMMRBackend<RangeProof>>,
+ kernel_pmmr: PMMR<'a, TxKernel, PMMRBackend<TxKernel>>,
- kernel_file: &'a mut AppendOnlyFile,
commit_index: Arc<ChainStore>,
new_output_commits: HashMap<Commitment, u64>,
new_kernel_excesses: HashMap<Commitment, u64>,
@@ -284,9 +283,9 @@ impl<'a> Extension<'a> {
) -> Extension<'a> {
Extension {
- output_pmmr: PMMR::at(
- &mut trees.output_pmmr_h.backend,
- trees.output_pmmr_h.last_pos,
+ utxo_pmmr: PMMR::at(
+ &mut trees.utxo_pmmr_h.backend,
+ trees.utxo_pmmr_h.last_pos,
),
rproof_pmmr: PMMR::at(
&mut trees.rproof_pmmr_h.backend,
@@ -296,10 +295,10 @@ impl<'a> Extension<'a> {
&mut trees.kernel_pmmr_h.backend,
trees.kernel_pmmr_h.last_pos,
),
- kernel_file: &mut trees.kernel_file,
commit_index: commit_index,
new_output_commits: HashMap::new(),
new_kernel_excesses: HashMap::new(),
+
rollback: false,
}
}
@@ -354,15 +353,14 @@ impl<'a> Extension<'a> {
fn apply_input(&mut self, input: &Input, height: u64) -> Result<(), Error> {
let commit = input.commitment();
let pos_res = self.get_output_pos(&commit);
+ let output_id_hash = OutputIdentifier::from_input(input).hash();
if let Ok(pos) = pos_res {
- if let Some(HashSum { hash, sum: _ }) = self.output_pmmr.get(pos) {
- let sum_commit = SumCommit::from_input(&input);
-
- // check hash from pmmr matches hash from input
+ if let Some((read_hash, read_elem)) = self.utxo_pmmr.get(pos, true) {
+ // check hash from pmmr matches hash from input (or corresponding output)
// if not then the input is not being honest about
// what it is attempting to spend...
- let hash_sum = HashSum::from_summable(pos, &sum_commit);
- if hash != hash_sum.hash {
+ if output_id_hash != read_hash ||
+ output_id_hash != read_elem.expect("no output at position").hash() {
return Err(Error::SumTreeErr(format!("output pmmr hash mismatch")));
}
@@ -379,9 +377,10 @@ impl<'a> Extension<'a> {
}
}
- // Now prune the output_pmmr and rproof_pmmr.
+ // Now prune the utxo_pmmr, rproof_pmmr and their storage.
// Input is not valid if we cannot prune successfully (to spend an unspent output).
- match self.output_pmmr.prune(pos, height as u32) {
+ // TODO: rm log, skip list for utxo and range proofs
+ match self.utxo_pmmr.prune(pos, height as u32) {
Ok(true) => {
self.rproof_pmmr
.prune(pos, height as u32)
@@ -398,7 +397,6 @@ impl<'a> Extension<'a> {
fn apply_output(&mut self, out: &Output) -> Result<(), Error> {
let commit = out.commitment();
- let sum_commit = SumCommit::from_output(out);
if let Ok(pos) = self.get_output_pos(&commit) {
// we need to check whether the commitment is in the current MMR view
@@ -406,27 +404,24 @@ impl<'a> Extension<'a> {
// (non-historical node will have a much smaller one)
// note that this doesn't show the commitment *never* existed, just
// that this is not an existing unspent commitment right now
- if let Some(c) = self.output_pmmr.get(pos) {
- let hash_sum = HashSum::from_summable(pos, &sum_commit);
-
+ if let Some((hash, _)) = self.utxo_pmmr.get(pos, false) {
// processing a new fork so we may get a position on the old
// fork that exists but matches a different node
// filtering that case out
- if c.hash == hash_sum.hash {
+ if hash == OutputStoreable::from_output(out).hash() {
return Err(Error::DuplicateCommitment(commit));
}
}
}
- // push new outputs commitments in their MMR and save them in the index
- let pos = self.output_pmmr
- .push(sum_commit)
+ // push new outputs in their MMR and save them in the index
+ let pos = self.utxo_pmmr
+ .push(OutputStoreable::from_output(out))
.map_err(&Error::SumTreeErr)?;
-
self.new_output_commits.insert(out.commitment(), pos);
- // push range proofs in their MMR
+ // push range proofs in their MMR and file
self.rproof_pmmr
- .push(NoSum(out.proof))
+ .push(out.proof)
.map_err(&Error::SumTreeErr)?;
Ok(())
}
@@ -434,9 +429,8 @@ impl<'a> Extension<'a> {
fn apply_kernel(&mut self, kernel: &TxKernel) -> Result<(), Error> {
if let Ok(pos) = self.get_kernel_pos(&kernel.excess) {
// same as outputs
- if let Some(k) = self.kernel_pmmr.get(pos) {
- let hashsum = HashSum::from_summable(pos, &NoSum(kernel));
- if k.hash == hashsum.hash {
+ if let Some((h,_)) = self.kernel_pmmr.get(pos, false) {
+ if h == kernel.hash() {
return Err(Error::DuplicateKernel(kernel.excess.clone()));
}
}
@@ -444,10 +438,9 @@ impl<'a> Extension<'a> {
// push kernels in their MMR and file
let pos = self.kernel_pmmr
- .push(NoSum(kernel.clone()))
+ .push(kernel.clone())
.map_err(&Error::SumTreeErr)?;
self.new_kernel_excesses.insert(kernel.excess, pos);
- self.kernel_file.append(&mut ser::ser_vec(&kernel).unwrap());
Ok(())
}
@@ -465,13 +458,6 @@ impl<'a> Extension<'a> {
// rewind each MMR
let (out_pos_rew, kern_pos_rew) = indexes_at(block, self.commit_index.deref())?;
self.rewind_pos(block.header.height, out_pos_rew, kern_pos_rew)?;
-
- // rewind the kernel file store, the position is the number of kernels
- // multiplied by their size
- // the number of kernels is the number of leaves in the MMR
- let pos = pmmr::n_leaves(kern_pos_rew);
- self.kernel_file.rewind(pos * (TxKernel::size() as u64));
-
Ok(())
}
@@ -485,7 +471,7 @@ impl<'a> Extension<'a> {
kern_pos_rew,
);
- self.output_pmmr
+ self.utxo_pmmr
.rewind(out_pos_rew, height as u32)
.map_err(&Error::SumTreeErr)?;
self.rproof_pmmr
@@ -495,7 +481,6 @@ impl<'a> Extension<'a> {
.rewind(kern_pos_rew, height as u32)
.map_err(&Error::SumTreeErr)?;
- self.dump(true);
Ok(())
}
@@ -515,26 +500,23 @@ impl<'a> Extension<'a> {
}
}
+
/// Current root hashes and sums (if applicable) for the UTXO, range proof
/// and kernel sum trees.
pub fn roots(
&self,
- ) -> (
- HashSum<SumCommit>,
- HashSum<NoSum<RangeProof>>,
- HashSum<NoSum<TxKernel>>,
- ) {
- (
- self.output_pmmr.root(),
- self.rproof_pmmr.root(),
- self.kernel_pmmr.root(),
- )
+ ) -> SumTreeRoots {
+ SumTreeRoots {
+ utxo_root: self.utxo_pmmr.root(),
+ rproof_root: self.rproof_pmmr.root(),
+ kernel_root: self.kernel_pmmr.root(),
+ }
}
/// Validate the current sumtree state against a block header
pub fn validate(&self, header: &BlockHeader) -> Result<(), Error> {
// validate all hashes and sums within the trees
- if let Err(e) = self.output_pmmr.validate() {
+ if let Err(e) = self.utxo_pmmr.validate() {
return Err(Error::InvalidSumtree(e));
}
if let Err(e) = self.rproof_pmmr.validate() {
@@ -545,9 +527,9 @@ impl<'a> Extension<'a> {
}
// validate the tree roots against the block header
- let (utxo_root, rproof_root, kernel_root) = self.roots();
- if utxo_root.hash != header.utxo_root || rproof_root.hash != header.range_proof_root
- || kernel_root.hash != header.kernel_root
+ let roots = self.roots();
+ if roots.utxo_root != header.utxo_root || roots.rproof_root != header.range_proof_root
+ || roots.kernel_root != header.kernel_root
{
return Err(Error::InvalidRoot);
}
@@ -574,11 +556,11 @@ impl<'a> Extension<'a> {
/// by iterating over the whole MMR data. This is a costly operation
/// performed only when we receive a full new chain state.
pub fn rebuild_index(&self) -> Result<(), Error> {
- for n in 1..self.output_pmmr.unpruned_size()+1 {
+ for n in 1..self.utxo_pmmr.unpruned_size()+1 {
// non-pruned leaves only
if pmmr::bintree_postorder_height(n) == 0 {
- if let Some(hs) = self.output_pmmr.get(n) {
- self.commit_index.save_output_pos(&hs.sum.commit, n)?;
+ if let Some((_, out)) = self.utxo_pmmr.get(n, true) {
+ self.commit_index.save_output_pos(&out.expect("not a leaf node").commit, n)?;
}
}
}
@@ -594,7 +576,7 @@ impl<'a> Extension<'a> {
/// version only prints the UTXO tree.
pub fn dump(&self, short: bool) {
debug!(LOGGER, "-- outputs --");
- self.output_pmmr.dump(short);
+ self.utxo_pmmr.dump(short);
if !short {
debug!(LOGGER, "-- range proofs --");
self.rproof_pmmr.dump(short);
@@ -606,7 +588,7 @@ impl<'a> Extension<'a> {
// Sizes of the sum trees, used by `extending` on rollback.
fn sizes(&self) -> (u64, u64, u64) {
(
- self.output_pmmr.unpruned_size(),
+ self.utxo_pmmr.unpruned_size(),
self.rproof_pmmr.unpruned_size(),
self.kernel_pmmr.unpruned_size(),
)
@@ -619,7 +601,7 @@ impl<'a> Extension<'a> {
let mmr_sz = self.kernel_pmmr.unpruned_size();
let count = pmmr::n_leaves(mmr_sz);
- let mut kernel_file = File::open(self.kernel_file.path())?;
+ let mut kernel_file = File::open(self.kernel_pmmr.data_file_path())?;
let first: TxKernel = ser::deserialize(&mut kernel_file)?;
first.verify()?;
let mut sum_kernel = first.excess;
@@ -651,14 +633,14 @@ impl<'a> Extension<'a> {
let mut sum_utxo = None;
let mut utxo_count = 0;
let secp = static_secp_instance();
- for n in 1..self.output_pmmr.unpruned_size()+1 {
+ for n in 1..self.utxo_pmmr.unpruned_size()+1 {
if pmmr::bintree_postorder_height(n) == 0 {
- if let Some(hs) = self.output_pmmr.get(n) {
+ if let Some((_,output)) = self.utxo_pmmr.get(n, true) {
if n == 1 {
- sum_utxo = Some(hs.sum.commit);
+ sum_utxo = Some(output.expect("not a leaf node").commit);
} else {
let secp = secp.lock().unwrap();
- sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), hs.sum.commit], vec![])?);
+ sum_utxo = Some(secp.commit_sum(vec![sum_utxo.unwrap(), output.expect("not a leaf node").commit], vec![])?);
}
utxo_count += 1;
}
@@ -669,6 +651,42 @@ impl<'a> Extension<'a> {
}
}
+/*fn store_element<T>(file_store: &mut FlatFileStore<T>, data: T)
+ -> Result<(), String>
+where
+ T: ser::Readable + ser::Writeable + Clone
+{
+ file_store.append(vec![data])
+}
+
+fn read_element_at_pmmr_index<T>(file_store: &FlatFileStore<T>, pos: u64) -> Option<T>
+where
+ T: ser::Readable + ser::Writeable + Clone
+{
+ let leaf_index = pmmr::leaf_index(pos);
+ // flat files are zero indexed
+ file_store.get(leaf_index - 1)
+}
+
+fn _remove_element_at_pmmr_index<T>(file_store: &mut FlatFileStore<T>, pos: u64)
+ -> Result<(), String>
+where
+ T: ser::Readable + ser::Writeable + Clone
+{
+ let leaf_index = pmmr::leaf_index(pos);
+ // flat files are zero indexed
+ file_store.remove(vec![leaf_index - 1])
+}
+
+fn rewind_to_pmmr_index<T>(file_store: &mut FlatFileStore<T>, pos: u64) -> Result<(), String>
+where
+ T: ser::Readable + ser::Writeable + Clone
+{
+ let leaf_index = pmmr::leaf_index(pos);
+ // flat files are zero indexed
+ file_store.rewind(leaf_index - 1)
+}*/
+
/// Output and kernel MMR indexes at the end of the provided block
fn indexes_at(block: &Block, commit_index: &ChainStore) -> Result<(u64, u64), Error> {
let out_idx = match block.outputs.last() {
diff --git a/chain/src/types.rs b/chain/src/types.rs
index 43ab71fbd..a16a64512 100644
--- a/chain/src/types.rs
+++ b/chain/src/types.rs
@@ -40,6 +40,17 @@ bitflags! {
}
}
+/// A helper to hold the roots of the sumtrees in order to keep them
+/// readable
+pub struct SumTreeRoots {
+ /// UTXO root
+ pub utxo_root: Hash,
+ /// Range Proof root
+ pub rproof_root: Hash,
+ /// Kernel root
+ pub kernel_root: Hash,
+}
+
/// Errors
#[derive(Debug)]
pub enum Error {
@@ -81,6 +92,8 @@ pub enum Error {
InvalidSumtree(String),
/// Internal issue when trying to save or load data from store
StoreErr(grin_store::Error, String),
+ /// Internal issue when trying to save or load data from append only files
+ FileReadErr(String),
/// Error serializing or deserializing a type
SerErr(ser::Error),
/// Error with the sumtrees
diff --git a/chain/tests/mine_simple_chain.rs b/chain/tests/mine_simple_chain.rs
index 8fff8b7a6..6cb4ce579 100644
--- a/chain/tests/mine_simple_chain.rs
+++ b/chain/tests/mine_simple_chain.rs
@@ -253,6 +253,8 @@ fn spend_in_fork() {
fork_head = b.header.clone();
chain.process_block(b, chain::Options::SKIP_POW).unwrap();
+ println!("First block");
+
// now mine three further blocks
for n in 3..6 {
let b = prepare_block(&kc, &fork_head, &chain, n);
@@ -263,6 +265,8 @@ fn spend_in_fork() {
let lock_height = 1 + global::coinbase_maturity();
assert_eq!(lock_height, 4);
+ println!("3 Further Blocks: should have 4 blocks or 264 bytes in file ");
+
let tx1 = build::transaction(
vec![
build::coinbase_input(consensus::REWARD, block_hash, kc.derive_key_id(2).unwrap()),
@@ -272,10 +276,14 @@ fn spend_in_fork() {
&kc,
).unwrap();
+ println!("Built coinbase input and output");
+
let next = prepare_block_tx(&kc, &fork_head, &chain, 7, vec![&tx1]);
let prev_main = next.header.clone();
chain.process_block(next.clone(), chain::Options::SKIP_POW).unwrap();
+ println!("tx 1 processed, should have 6 outputs or 396 bytes in file, first skipped");
+
let tx2 = build::transaction(
vec![
build::input(consensus::REWARD - 20000, next.hash(), kc.derive_key_id(30).unwrap()),
@@ -289,6 +297,9 @@ fn spend_in_fork() {
let prev_main = next.header.clone();
chain.process_block(next, chain::Options::SKIP_POW).unwrap();
+ println!("tx 2 processed");
+ /*panic!("Stop");*/
+
// mine 2 forked blocks from the first
let fork = prepare_fork_block_tx(&kc, &fork_head, &chain, 6, vec![&tx1]);
let prev_fork = fork.header.clone();
diff --git a/core/src/core/hash.rs b/core/src/core/hash.rs
index 7b3131ec2..27d1918ff 100644
--- a/core/src/core/hash.rs
+++ b/core/src/core/hash.rs
@@ -20,13 +20,13 @@
use std::cmp::min;
use std::{fmt, ops};
use std::convert::AsRef;
+use std::ops::Add;
use blake2::blake2b::Blake2b;
use consensus;
use ser::{self, AsFixedBytes, Error, Readable, Reader, Writeable, Writer};
use util;
-use util::LOGGER;
/// A hash consisting of all zeroes, used as a sentinel. No known preimage.
pub const ZERO_HASH: Hash = Hash([0; 32]);
@@ -147,6 +147,13 @@ impl Writeable for Hash {
}
}
+impl Add for Hash {
+ type Output = Hash;
+ fn add(self, other: Hash) -> Hash {
+ self.hash_with(other)
+ }
+}
+
/// Serializer that outputs a hash of the serialized object
pub struct HashWriter {
state: Blake2b,
@@ -205,7 +212,6 @@ impl<W: ser::Writeable> Hashed for W {
fn hash_with<T: Writeable>(&self, other: T) -> Hash {
let mut hasher = HashWriter::default();
ser::Writeable::write(self, &mut hasher).unwrap();
- trace!(LOGGER, "Hashing with additional data");
ser::Writeable::write(&other, &mut hasher).unwrap();
let mut ret = [0; 32];
hasher.finalize(&mut ret);
diff --git a/core/src/core/pmmr.rs b/core/src/core/pmmr.rs
index b7fe85bb8..1a4b29214 100644
--- a/core/src/core/pmmr.rs
+++ b/core/src/core/pmmr.rs
@@ -29,169 +29,30 @@
//! position of siblings, parents, etc. As all those functions only rely on
//! binary operations, they're extremely fast. For more information, see the
//! doc on bintree_jump_left_sibling.
-//! 2. The implementation of a prunable MMR sum tree using the above. Each leaf
-//! is required to be Summable and Hashed. Tree roots can be trivially and
+//! 2. The implementation of a prunable MMR tree using the above. Each leaf
+//! is required to be Writeable (which implements Hashed). Tree roots can be trivially and
//! efficiently calculated without materializing the full tree. The underlying
-//! (Hash, Sum) pais are stored in a Backend implementation that can either be
+//! Hashes are stored in a Backend implementation that can either be
//! a simple Vec or a database.
use std::clone::Clone;
+use std::ops::Deref;
use std::marker::PhantomData;
-use std::ops::{self, Deref};
use core::hash::{Hash, Hashed};
-use ser::{self, Readable, Reader, Writeable, Writer};
+use ser::PMMRable;
use util::LOGGER;
-/// Trait for an element of the tree that has a well-defined sum and hash that
-/// the tree can sum over
-pub trait Summable {
- /// The type of the sum
- type Sum: Clone + ops::Add<Output = Self::Sum> + Readable + Writeable + PartialEq;
-
- /// Obtain the sum of the element
- fn sum(&self) -> Self::Sum;
-
- /// Length of the Sum type when serialized. Can be used as a hint by
- /// underlying storages.
- fn sum_len() -> usize;
-}
-
-/// An empty sum that takes no space, to store elements that do not need summing
-/// but can still leverage the hierarchical hashing.
-#[derive(Copy, Clone, Debug)]
-pub struct NullSum;
-impl ops::Add for NullSum {
- type Output = NullSum;
- fn add(self, _: NullSum) -> NullSum {
- NullSum
- }
-}
-
-impl Readable for NullSum {
- fn read(_: &mut Reader) -> Result<NullSum, ser::Error> {
- Ok(NullSum)
- }
-}
-
-impl Writeable for NullSum {
- fn write<W: Writer>(&self, _: &mut W) -> Result<(), ser::Error> {
- Ok(())
- }
-}
-
-impl PartialEq for NullSum {
- fn eq(&self, _other: &NullSum) -> bool {
- true
- }
-}
-
-/// Wrapper for a type that allows it to be inserted in a tree without summing
-#[derive(Clone, Debug)]
-pub struct NoSum<T>(pub T);
-impl<T> Summable for NoSum<T> {
- type Sum = NullSum;
- fn sum(&self) -> NullSum {
- NullSum
- }
- fn sum_len() -> usize {
- return 0;
- }
-}
-impl<T> Writeable for NoSum<T>
-where
- T: Writeable,
-{
- fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
- self.0.write(writer)
- }
-}
-
-/// A utility type to handle (Hash, Sum) pairs more conveniently. The addition
-/// of two HashSums is the (Hash(h1|h2), h1 + h2) HashSum.
-#[derive(Debug, Clone, Eq)]
-pub struct HashSum<T>
-where
- T: Summable,
-{
- /// The hash
- pub hash: Hash,
- /// The sum
- pub sum: T::Sum,
-}
-
-impl<T> HashSum<T>
-where
- T: Summable + Hashed,
-{
- /// Create a hash sum from a summable
- pub fn from_summable(idx: u64, elmt: &T) -> HashSum<T> {
- let hash = elmt.hash();
- let sum = elmt.sum();
- let node_hash = (idx, &sum, hash).hash();
- HashSum {
- hash: node_hash,
- sum: sum,
- }
- }
-}
-
-impl<T> PartialEq for HashSum<T>
-where
- T: Summable,
-{
- fn eq(&self, other: &HashSum<T>) -> bool {
- self.hash == other.hash && self.sum == other.sum
- }
-}
-
-impl<T> Readable for HashSum<T>
-where
- T: Summable,
-{
- fn read(r: &mut Reader) -> Result<HashSum<T>, ser::Error> {
- Ok(HashSum {
- hash: Hash::read(r)?,
- sum: T::Sum::read(r)?,
- })
- }
-}
-
-impl<T> Writeable for HashSum<T>
-where
- T: Summable,
-{
- fn write<W: Writer>(&self, w: &mut W) -> Result<(), ser::Error> {
- self.hash.write(w)?;
- self.sum.write(w)
- }
-}
-
-impl<T> ops::Add for HashSum<T>
-where
- T: Summable,
-{
- type Output = HashSum<T>;
- fn add(self, other: HashSum<T>) -> HashSum<T> {
- HashSum {
- hash: (self.hash, other.hash).hash(),
- sum: self.sum + other.sum,
- }
- }
-}
-
/// Storage backend for the MMR, just needs to be indexed by order of insertion.
/// The PMMR itself does not need the Backend to be accurate on the existence
/// of an element (i.e. remove could be a no-op) but layers above can
/// depend on an accurate Backend to check existence.
-pub trait Backend<T>
-where
- T: Summable,
-{
- /// Append the provided HashSums to the backend storage. The position of the
- /// first element of the Vec in the MMR is provided to help the
- /// implementation.
- fn append(&mut self, position: u64, data: Vec<HashSum<T>>) -> Result<(), String>;
+pub trait Backend<T> where
+ T:PMMRable {
+ /// Append the provided Hashes to the backend storage, and optionally an associated
+ /// data element to flatfile storage (for leaf nodes only). The position of the
+ /// first element of the Vec in the MMR is provided to help the implementation.
+ fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String>;
/// Rewind the backend state to a previous position, as if all append
/// operations after that had been canceled. Expects a position in the PMMR
@@ -199,14 +60,20 @@ where
/// occurred (see remove).
fn rewind(&mut self, position: u64, index: u32) -> Result<(), String>;
- /// Get a HashSum by insertion position
- fn get(&self, position: u64) -> Option<HashSum<T>>;
+ /// Get a Hash/Element by insertion position. If include_data is true, will
+ /// also return the associated data element
+ fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option<T>)>;
- /// Remove HashSums by insertion position. An index is also provided so the
+ /// Remove Hashes/Data by insertion position. An index is also provided so the
/// underlying backend can implement some rollback of positions up to a
/// given index (practically the index is a the height of a block that
/// triggered removal).
fn remove(&mut self, positions: Vec<u64>, index: u32) -> Result<(), String>;
+
+ /// Returns the data file path.. this is a bit of a hack now that doesn't
+ /// sit well with the design, but TxKernels have to be summed and the
+ /// fastest way to to be able to allow direct access to the file
+ fn get_data_file_path(&self) -> String;
}
/// Prunable Merkle Mountain Range implementation. All positions within the tree
@@ -218,18 +85,18 @@ where
/// we are in the sequence of nodes making up the MMR.
pub struct PMMR<'a, T, B>
where
- T: Summable,
+ T: PMMRable,
B: 'a + Backend<T>,
{
last_pos: u64,
backend: &'a mut B,
// only needed for parameterizing Backend
- summable: PhantomData<T>,
+ writeable: PhantomData<T>,
}
impl<'a, T, B> PMMR<'a, T, B>
where
- T: Summable + Hashed + Clone,
+ T: PMMRable,
B: 'a + Backend<T>,
{
/// Build a new prunable Merkle Mountain Range using the provided backend.
@@ -237,7 +104,7 @@ where
PMMR {
last_pos: 0,
backend: backend,
- summable: PhantomData,
+ writeable: PhantomData,
}
}
@@ -248,48 +115,51 @@ where
PMMR {
last_pos: last_pos,
backend: backend,
- summable: PhantomData,
+ writeable: PhantomData,
}
}
/// Computes the root of the MMR. Find all the peaks in the current
/// tree and "bags" them to get a single peak.
- pub fn root(&self) -> HashSum<T> {
+ pub fn root(&self) -> Hash {
let peaks_pos = peaks(self.last_pos);
- let peaks: Vec<Option<HashSum<T>>> = map_vec!(peaks_pos, |&pi| self.backend.get(pi));
+ let peaks: Vec<Option<(Hash, Option<T>)>> = peaks_pos.into_iter()
+ .map(|pi| self.backend.get(pi, false))
+ .collect();
+
let mut ret = None;
for peak in peaks {
ret = match (ret, peak) {
(None, x) => x,
- (Some(hsum), None) => Some(hsum),
- (Some(lhsum), Some(rhsum)) => Some(lhsum + rhsum),
+ (Some(hash), None) => Some(hash),
+ (Some(lhash), Some(rhash)) => Some((lhash.0.hash_with(rhash.0), None)),
}
}
- ret.expect("no root, invalid tree")
+ ret.expect("no root, invalid tree").0
}
- /// Push a new Summable element in the MMR. Computes new related peaks at
+ /// Push a new element into the MMR. Computes new related peaks at
/// the same time if applicable.
pub fn push(&mut self, elmt: T) -> Result<u64, String> {
let elmt_pos = self.last_pos + 1;
- let mut current_hashsum = HashSum::from_summable(elmt_pos, &elmt);
- let mut to_append = vec![current_hashsum.clone()];
+ let mut current_hash = elmt.hash();
+ let mut to_append = vec![(current_hash, Some(elmt))];
let mut height = 0;
let mut pos = elmt_pos;
// we look ahead one position in the MMR, if the expected node has a higher
- // height it means we have to build a higher peak by summing with a previous
+ // height it means we have to build a higher peak by hashing with a previous
// sibling. we do it iteratively in case the new peak itself allows the
// creation of another parent.
while bintree_postorder_height(pos + 1) > height {
let left_sibling = bintree_jump_left_sibling(pos);
- let left_hashsum = self.backend.get(left_sibling).expect(
+ let left_elem = self.backend.get(left_sibling, false).expect(
"missing left sibling in tree, should not have been pruned",
);
- current_hashsum = left_hashsum + current_hashsum;
+ current_hash = left_elem.0 + current_hash;
- to_append.push(current_hashsum.clone());
+ to_append.push((current_hash.clone(), None));
height += 1;
pos += 1;
}
@@ -322,7 +192,7 @@ where
/// to keep an index of elements to positions in the tree. Prunes parent
/// nodes as well when they become childless.
pub fn prune(&mut self, position: u64, index: u32) -> Result<bool, String> {
- if let None = self.backend.get(position) {
+ if let None = self.backend.get(position, false) {
return Ok(false);
}
let prunable_height = bintree_postorder_height(position);
@@ -345,7 +215,7 @@ where
// if we have a pruned sibling, we can continue up the tree
// otherwise we're done
- if let None = self.backend.get(sibling) {
+ if let None = self.backend.get(sibling, false) {
current = parent;
} else {
break;
@@ -356,26 +226,27 @@ where
Ok(true)
}
- /// Helper function to get the HashSum of a node at a given position from
+ /// Helper function to get a node at a given position from
/// the backend.
- pub fn get(&self, position: u64) -> Option<HashSum<T>> {
+ pub fn get(&self, position: u64, include_data: bool) -> Option<(Hash, Option<T>)> {
if position > self.last_pos {
None
} else {
- self.backend.get(position)
+ self.backend.get(position, include_data)
}
}
+
/// Helper function to get the last N nodes inserted, i.e. the last
/// n nodes along the bottom of the tree
- pub fn get_last_n_insertions(&self, n: u64) -> Vec<HashSum<T>> {
+ pub fn get_last_n_insertions(&self, n: u64) -> Vec<(Hash, Option<T>)> {
let mut return_vec = Vec::new();
let mut last_leaf = self.last_pos;
let size = self.unpruned_size();
// Special case that causes issues in bintree functions,
// just return
if size == 1 {
- return_vec.push(self.backend.get(last_leaf).unwrap());
+ return_vec.push(self.backend.get(last_leaf, true).unwrap());
return return_vec;
}
// if size is even, we're already at the bottom, otherwise
@@ -390,7 +261,7 @@ where
if bintree_postorder_height(last_leaf) > 0 {
last_leaf = bintree_rightmost(last_leaf);
}
- return_vec.push(self.backend.get(last_leaf).unwrap());
+ return_vec.push(self.backend.get(last_leaf, true).unwrap());
last_leaf = bintree_jump_left_sibling(last_leaf);
}
@@ -398,21 +269,20 @@ where
}
/// Walks all unpruned nodes in the MMR and revalidate all parent hashes
- /// and sums.
pub fn validate(&self) -> Result<(), String> {
// iterate on all parent nodes
for n in 1..(self.last_pos + 1) {
if bintree_postorder_height(n) > 0 {
- if let Some(hs) = self.get(n) {
+ if let Some(hs) = self.get(n, false) {
// take the left and right children, if they exist
let left_pos = bintree_move_down_left(n).unwrap();
let right_pos = bintree_jump_right_sibling(left_pos);
- if let Some(left_child_hs) = self.get(left_pos) {
- if let Some(right_child_hs) = self.get(right_pos) {
- // sum and compare
- if left_child_hs + right_child_hs != hs {
- return Err(format!("Invalid MMR, hashsum of parent at {} does \
+ if let Some(left_child_hs) = self.get(left_pos, false) {
+ if let Some(right_child_hs) = self.get(right_pos, false) {
+ // add hashes and compare
+ if left_child_hs.0+right_child_hs.0 != hs.0 {
+ return Err(format!("Invalid MMR, hash of parent at {} does \
not match children.", n));
}
}
@@ -429,6 +299,11 @@ where
self.last_pos
}
+ /// Return the path of the data file (needed to sum kernels efficiently)
+ pub fn data_file_path(&self) -> String {
+ self.backend.get_data_file_path()
+ }
+
/// Debugging utility to print information about the MMRs. Short version
/// only prints the last 8 nodes.
pub fn dump(&self, short: bool) {
@@ -445,40 +320,36 @@ where
break;
}
idx.push_str(&format!("{:>8} ", m + 1));
- let ohs = self.get(m + 1);
+ let ohs = self.get(m + 1, false);
match ohs {
- Some(hs) => hashes.push_str(&format!("{} ", hs.hash)),
+ Some(hs) => hashes.push_str(&format!("{} ", hs.0)),
None => hashes.push_str(&format!("{:>8} ", "??")),
}
}
- debug!(LOGGER, "{}", idx);
- debug!(LOGGER, "{}", hashes);
+ trace!(LOGGER, "{}", idx);
+ trace!(LOGGER, "{}", hashes);
}
}
}
/// Simple MMR backend implementation based on a Vector. Pruning does not
/// compact the Vector itself but still frees the reference to the
-/// underlying HashSum.
+/// underlying Hash.
#[derive(Clone)]
pub struct VecBackend<T>
-where
- T: Summable + Clone,
-{
+ where T:PMMRable {
/// Backend elements
- pub elems: Vec<Option<HashSum<T>>>,
+ pub elems: Vec<Option<(Hash, Option<T>)>>,
}
-impl<T> Backend<T> for VecBackend<T>
-where
- T: Summable + Clone,
-{
+impl <T> Backend <T> for VecBackend<T>
+ where T: PMMRable {
#[allow(unused_variables)]
- fn append(&mut self, position: u64, data: Vec<HashSum<T>>) -> Result<(), String> {
+ fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String> {
self.elems.append(&mut map_vec!(data, |d| Some(d.clone())));
Ok(())
}
- fn get(&self, position: u64) -> Option<HashSum<T>> {
+ fn get(&self, position: u64, _include_data:bool) -> Option<(Hash, Option<T>)> {
self.elems[(position - 1) as usize].clone()
}
#[allow(unused_variables)]
@@ -493,18 +364,19 @@ where
self.elems = self.elems[0..(position as usize) + 1].to_vec();
Ok(())
}
+ fn get_data_file_path(&self) -> String {
+ "".to_string()
+ }
}
-impl<T> VecBackend<T>
-where
- T: Summable + Clone,
-{
+impl <T> VecBackend <T>
+ where T:PMMRable {
/// Instantiates a new VecBackend<T>
pub fn new() -> VecBackend<T> {
VecBackend { elems: vec![] }
}
- /// Current number of HashSum elements in the underlying Vec.
+ /// Current number of elements in the underlying Vec.
pub fn used_size(&self) -> usize {
let mut usz = self.elems.len();
for elem in self.elems.deref() {
@@ -568,6 +440,28 @@ impl PruneList {
}
}
+ /// As above, but only returning the number of leaf nodes to skip for a
+ /// given leaf. Helpful if, for instance, data for each leaf is being stored
+ /// separately in a continuous flat-file
+ pub fn get_leaf_shift(&self, pos: u64) -> Option<u64> {
+
+ // get the position where the node at pos would fit in the pruned list, if
+ // it's already pruned, nothing to skip
+ match self.pruned_pos(pos) {
+ None => None,
+ Some(idx) => {
+ // skip by the number of leaf nodes pruned in the preceeding subtrees
+ // which just 2^height
+ Some(
+ self.pruned_nodes[0..(idx as usize)]
+ .iter()
+ .map(|n| 1 << bintree_postorder_height(*n))
+ .sum(),
+ )
+ }
+ }
+ }
+
/// Push the node at the provided position in the prune list. Compacts the
/// list if pruning the additional node means a parent can get pruned as
/// well.
@@ -591,7 +485,7 @@ impl PruneList {
}
/// Gets the position a new pruned node should take in the prune list.
- /// If the node has already bee pruned, either directly or through one of
+ /// If the node has already been pruned, either directly or through one of
/// its parents contained in the prune list, returns None.
pub fn pruned_pos(&self, pos: u64) -> Option<usize> {
match self.pruned_nodes.binary_search(&pos) {
@@ -833,7 +727,20 @@ fn most_significant_pos(num: u64) -> u64 {
#[cfg(test)]
mod test {
use super::*;
- use core::hash::Hashed;
+ use ser::{Writeable, Readable, Error};
+ use core::{Writer, Reader};
+ use core::hash::{Hash};
+
+ #[test]
+ fn test_leaf_index(){
+ assert_eq!(n_leaves(1),1);
+ assert_eq!(n_leaves(2),2);
+ assert_eq!(n_leaves(4),3);
+ assert_eq!(n_leaves(5),4);
+ assert_eq!(n_leaves(8),5);
+ assert_eq!(n_leaves(9),6);
+
+ }
#[test]
fn some_all_ones() {
@@ -890,23 +797,17 @@ mod test {
assert_eq!(peaks(42), vec![31, 38, 41, 42]);
}
- #[derive(Copy, Clone, Debug, PartialEq, Eq)]
+ #[derive(Copy, Clone, Debug, PartialEq, Eq)]
struct TestElem([u32; 4]);
- impl Summable for TestElem {
- type Sum = u64;
- fn sum(&self) -> u64 {
- // sums are not allowed to overflow, so we use this simple
- // non-injective "sum" function that will still be homomorphic
- self.0[0] as u64 * 0x1000 + self.0[1] as u64 * 0x100 + self.0[2] as u64 * 0x10 +
- self.0[3] as u64
- }
- fn sum_len() -> usize {
- 8
+
+ impl PMMRable for TestElem {
+ fn len() -> usize {
+ 16
}
}
impl Writeable for TestElem {
- fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
+ fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
try!(writer.write_u32(self.0[0]));
try!(writer.write_u32(self.0[1]));
try!(writer.write_u32(self.0[2]));
@@ -914,6 +815,19 @@ mod test {
}
}
+ impl Readable for TestElem {
+ fn read(reader: &mut Reader) -> Result<TestElem, Error> {
+ Ok(TestElem (
+ [
+ reader.read_u32()?,
+ reader.read_u32()?,
+ reader.read_u32()?,
+ reader.read_u32()?,
+ ]
+ ))
+ }
+ }
+
#[test]
#[allow(unused_variables)]
fn pmmr_push_root() {
@@ -934,72 +848,67 @@ mod test {
// one element
pmmr.push(elems[0]).unwrap();
- let hash = Hashed::hash(&elems[0]);
- let sum = elems[0].sum();
- let node_hash = (1 as u64, &sum, hash).hash();
+ let node_hash = elems[0].hash();
assert_eq!(
pmmr.root(),
- HashSum {
- hash: node_hash,
- sum: sum,
- }
+ node_hash,
);
assert_eq!(pmmr.unpruned_size(), 1);
+ pmmr.dump(false);
// two elements
pmmr.push(elems[1]).unwrap();
- let sum2 = HashSum::from_summable(1, &elems[0]) +
- HashSum::from_summable(2, &elems[1]);
+ let sum2 = elems[0].hash() + elems[1].hash();
+ pmmr.dump(false);
assert_eq!(pmmr.root(), sum2);
assert_eq!(pmmr.unpruned_size(), 3);
// three elements
pmmr.push(elems[2]).unwrap();
- let sum3 = sum2.clone() + HashSum::from_summable(4, &elems[2]);
+ let sum3 = sum2 + elems[2].hash();
+ pmmr.dump(false);
assert_eq!(pmmr.root(), sum3);
assert_eq!(pmmr.unpruned_size(), 4);
// four elements
pmmr.push(elems[3]).unwrap();
- let sum4 = sum2 +
- (HashSum::from_summable(4, &elems[2]) +
- HashSum::from_summable(5, &elems[3]));
+ let sum_one = elems[2].hash() + elems[3].hash();
+ let sum4 = sum2 + sum_one;
+ pmmr.dump(false);
assert_eq!(pmmr.root(), sum4);
assert_eq!(pmmr.unpruned_size(), 7);
// five elements
pmmr.push(elems[4]).unwrap();
- let sum5 = sum4.clone() + HashSum::from_summable(8, &elems[4]);
- assert_eq!(pmmr.root(), sum5);
+ let sum3 = sum4 + elems[4].hash();
+ pmmr.dump(false);
+ assert_eq!(pmmr.root(), sum3);
assert_eq!(pmmr.unpruned_size(), 8);
// six elements
pmmr.push(elems[5]).unwrap();
- let sum6 = sum4.clone() +
- (HashSum::from_summable(8, &elems[4]) +
- HashSum::from_summable(9, &elems[5]));
+ let sum6 = sum4 +
+ (elems[4].hash() + elems[5].hash());
assert_eq!(pmmr.root(), sum6.clone());
assert_eq!(pmmr.unpruned_size(), 10);
// seven elements
pmmr.push(elems[6]).unwrap();
- let sum7 = sum6 + HashSum::from_summable(11, &elems[6]);
+ let sum7 = sum6 + elems[6].hash();
assert_eq!(pmmr.root(), sum7);
assert_eq!(pmmr.unpruned_size(), 11);
// eight elements
pmmr.push(elems[7]).unwrap();
let sum8 = sum4 +
- ((HashSum::from_summable(8, &elems[4]) +
- HashSum::from_summable(9, &elems[5])) +
- (HashSum::from_summable(11, &elems[6]) +
- HashSum::from_summable(12, &elems[7])));
+ ((elems[4].hash() + elems[5].hash()) +
+ (elems[6].hash() + elems[7].hash()));
assert_eq!(pmmr.root(), sum8);
assert_eq!(pmmr.unpruned_size(), 15);
// nine elements
pmmr.push(elems[8]).unwrap();
- let sum9 = sum8 + HashSum::from_summable(16, &elems[8]);
+ let sum9 = sum8 + elems[8].hash();
assert_eq!(pmmr.root(), sum9);
assert_eq!(pmmr.unpruned_size(), 16);
}
@@ -1015,8 +924,9 @@ mod test {
TestElem([0, 0, 0, 6]),
TestElem([0, 0, 0, 7]),
TestElem([0, 0, 0, 8]),
- TestElem([0, 0, 0, 9]),
+ TestElem([1, 0, 0, 0]),
];
+
let mut ba = VecBackend::new();
let mut pmmr = PMMR::new(&mut ba);
@@ -1026,23 +936,23 @@ mod test {
pmmr.push(elems[0]).unwrap();
let res = pmmr.get_last_n_insertions(19);
- assert!(res.len() == 1 && res[0].sum == 1);
+ assert!(res.len() == 1);
pmmr.push(elems[1]).unwrap();
let res = pmmr.get_last_n_insertions(12);
- assert!(res[0].sum == 2 && res[1].sum == 1);
+ assert!(res.len() == 2);
pmmr.push(elems[2]).unwrap();
let res = pmmr.get_last_n_insertions(2);
- assert!(res[0].sum == 3 && res[1].sum == 2);
+ assert!(res.len() == 2);
pmmr.push(elems[3]).unwrap();
let res = pmmr.get_last_n_insertions(19);
assert!(
- res[0].sum == 4 && res[1].sum == 3 && res[2].sum == 2 && res[3].sum == 1 && res.len() == 4
+ res.len() == 4
);
pmmr.push(elems[5]).unwrap();
@@ -1052,7 +962,7 @@ mod test {
let res = pmmr.get_last_n_insertions(7);
assert!(
- res[0].sum == 9 && res[1].sum == 8 && res[2].sum == 7 && res[3].sum == 6 && res.len() == 7
+ res.len() == 7
);
}
@@ -1071,7 +981,7 @@ mod test {
TestElem([1, 0, 0, 0]),
];
- let orig_root: HashSum<TestElem>;
+ let orig_root: Hash;
let sz: u64;
let mut ba = VecBackend::new();
{
@@ -1085,7 +995,7 @@ mod test {
// pruning a leaf with no parent should do nothing
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(16, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@@ -1093,14 +1003,14 @@ mod test {
// pruning leaves with no shared parent just removes 1 element
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(2, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
assert_eq!(ba.used_size(), 15);
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(4, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@@ -1108,7 +1018,7 @@ mod test {
// pruning a non-leaf node has no effect
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(3, 0).unwrap_err();
assert_eq!(orig_root, pmmr.root());
}
@@ -1116,7 +1026,7 @@ mod test {
// pruning sibling removes subtree
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(5, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@@ -1124,7 +1034,7 @@ mod test {
// pruning all leaves under level >1 removes all subtree
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
pmmr.prune(1, 0).unwrap();
assert_eq!(orig_root, pmmr.root());
}
@@ -1132,7 +1042,7 @@ mod test {
// pruning everything should only leave us the peaks
{
- let mut pmmr = PMMR::at(&mut ba, sz);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut ba, sz);
for n in 1..16 {
let _ = pmmr.prune(n, 0);
}
diff --git a/core/src/core/transaction.rs b/core/src/core/transaction.rs
index 0c24f1854..5e42f718d 100644
--- a/core/src/core/transaction.rs
+++ b/core/src/core/transaction.rs
@@ -19,16 +19,14 @@ use util::{static_secp_instance, kernel_sig_msg};
use util::secp::pedersen::{Commitment, RangeProof};
use std::cmp::{min, max};
use std::cmp::Ordering;
-use std::ops;
use consensus;
use consensus::VerifySortOrder;
use core::Committed;
use core::hash::{Hash, Hashed, ZERO_HASH};
-use core::pmmr::Summable;
-use keychain;
use keychain::{Identifier, Keychain, BlindingFactor};
-use ser::{self, read_and_verify_sorted, Readable, Reader, Writeable, WriteableSorted, Writer};
+use keychain;
+use ser::{self, read_and_verify_sorted, PMMRable, Readable, Reader, Writeable, WriteableSorted, Writer};
use util;
/// The size of the blake2 hash of a switch commitment (256 bits)
@@ -217,9 +215,10 @@ impl TxKernel {
..self
}
}
+}
- /// Size in bytes of a kernel, necessary for binary storage
- pub fn size() -> usize {
+impl PMMRable for TxKernel {
+ fn len() -> usize {
17 + // features plus fee and lock_height
secp::constants::PEDERSEN_COMMITMENT_SIZE +
secp::constants::AGG_SIGNATURE_SIZE
@@ -671,17 +670,13 @@ impl SwitchCommitHash {
/// provides future-proofing against quantum-based attacks, as well as providing
/// wallet implementations with a way to identify their outputs for wallet
/// reconstruction.
-///
-/// The hash of an output only covers its features, commitment,
-/// and switch commitment. The range proof is expected to have its own hash
-/// and is stored and committed to separately.
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub struct Output {
/// Options for an output's structure or use
pub features: OutputFeatures,
/// The homomorphic commitment representing the output amount
pub commit: Commitment,
- /// The switch commitment hash, a 160 bit length blake2 hash of blind*J
+ /// The switch commitment hash, a 256 bit length blake2 hash of blind*J
pub switch_commit_hash: SwitchCommitHash,
/// A proof that the commitment is in the right range
pub proof: RangeProof,
@@ -704,9 +699,13 @@ impl Writeable for Output {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
writer.write_u8(self.features.bits())?;
writer.write_fixed_bytes(&self.commit)?;
- writer.write_fixed_bytes(&self.switch_commit_hash)?;
-
- // The hash of an output doesn't include the range proof
+ // Hash of an output doesn't cover the switch commit, it should
+ // be wound into the range proof separately
+ if writer.serialization_mode() != ser::SerializationMode::Hash {
+ writer.write_fixed_bytes(&self.switch_commit_hash)?;
+ }
+ // The hash of an output doesn't include the range proof, which
+ // is commit to separately
if writer.serialization_mode() == ser::SerializationMode::Full {
writer.write_bytes(&self.proof)?
}
@@ -818,21 +817,6 @@ impl OutputIdentifier {
util::to_hex(self.commit.0.to_vec()),
)
}
-
- /// Convert an output_indentifier to a sum_commit representation
- /// so we can use it to query the the output MMR
- pub fn as_sum_commit(&self) -> SumCommit {
- SumCommit {
- features: self.features,
- commit: self.commit,
- switch_commit_hash: SwitchCommitHash::zero(),
- }
- }
-
- /// Convert a sum_commit back to an output_identifier.
- pub fn from_sum_commit(sum_commit: &SumCommit) -> OutputIdentifier {
- OutputIdentifier::new(sum_commit.features, &sum_commit.commit)
- }
}
impl Writeable for OutputIdentifier {
@@ -855,140 +839,73 @@ impl Readable for OutputIdentifier {
}
}
-/// Wrapper to Output commitments to provide the Summable trait.
+/// Yet another output version to read/write from disk. Ends up being far too awkward
+/// to use the write serialisation property to do this
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-pub struct SumCommit {
+pub struct OutputStoreable {
/// Output features (coinbase vs. regular transaction output)
/// We need to include this when hashing to ensure coinbase maturity can be enforced.
pub features: OutputFeatures,
/// Output commitment
pub commit: Commitment,
- /// The corresponding switch commit hash
+ /// Switch commit hash
pub switch_commit_hash: SwitchCommitHash,
}
-impl SumCommit {
- /// Build a new sum_commit.
- pub fn new(
- features: OutputFeatures,
- commit: &Commitment,
- switch_commit_hash: &SwitchCommitHash,
- ) -> SumCommit {
- SumCommit {
- features: features.clone(),
- commit: commit.clone(),
- switch_commit_hash: switch_commit_hash.clone(),
- }
- }
-
- /// Build a new sum_commit from an existing output.
- pub fn from_output(output: &Output) -> SumCommit {
- SumCommit {
+impl OutputStoreable {
+ /// Build a StoreableOutput from an existing output.
+ pub fn from_output(output: &Output) -> OutputStoreable {
+ OutputStoreable {
features: output.features,
commit: output.commit,
switch_commit_hash: output.switch_commit_hash,
}
}
- /// Build a new sum_commit from an existing input.
- pub fn from_input(input: &Input) -> SumCommit {
- SumCommit {
- features: input.features,
- commit: input.commit,
- switch_commit_hash: SwitchCommitHash::zero(),
+ /// Return a regular output
+ pub fn to_output(self) -> Output {
+ Output{
+ features: self.features,
+ commit: self.commit,
+ switch_commit_hash: self.switch_commit_hash,
+ proof: RangeProof{
+ proof:[0; secp::constants::MAX_PROOF_SIZE],
+ plen: 0,
+ },
}
}
-
- /// Hex string representation of a sum_commit.
- pub fn to_hex(&self) -> String {
- format!(
- "{:b}{}{}",
- self.features.bits(),
- util::to_hex(self.commit.0.to_vec()),
- self.switch_commit_hash.to_hex(),
- )
- }
}
-/// Outputs get summed through their commitments.
-impl Summable for SumCommit {
- type Sum = SumCommit;
-
- fn sum(&self) -> SumCommit {
- SumCommit {
- commit: self.commit.clone(),
- features: self.features.clone(),
- switch_commit_hash: self.switch_commit_hash.clone(),
- }
- }
-
- fn sum_len() -> usize {
- secp::constants::PEDERSEN_COMMITMENT_SIZE + SWITCH_COMMIT_HASH_SIZE + 1
+impl PMMRable for OutputStoreable {
+ fn len() -> usize {
+ 1 + secp::constants::PEDERSEN_COMMITMENT_SIZE + SWITCH_COMMIT_HASH_SIZE
}
}
-impl Writeable for SumCommit {
+impl Writeable for OutputStoreable {
fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
writer.write_u8(self.features.bits())?;
self.commit.write(writer)?;
- if writer.serialization_mode() == ser::SerializationMode::Full {
+ if writer.serialization_mode() != ser::SerializationMode::Hash {
self.switch_commit_hash.write(writer)?;
}
Ok(())
}
}
-impl Readable for SumCommit {
- fn read(reader: &mut Reader) -> Result<SumCommit, ser::Error> {
+impl Readable for OutputStoreable {
+ fn read(reader: &mut Reader) -> Result<OutputStoreable, ser::Error> {
let features = OutputFeatures::from_bits(reader.read_u8()?).ok_or(
ser::Error::CorruptedData,
)?;
- Ok(SumCommit {
- features: features,
+ Ok(OutputStoreable {
commit: Commitment::read(reader)?,
switch_commit_hash: SwitchCommitHash::read(reader)?,
+ features: features,
})
}
}
-impl ops::Add for SumCommit {
- type Output = SumCommit;
-
- fn add(self, other: SumCommit) -> SumCommit {
- // Build a new commitment by summing the two commitments.
- let secp = static_secp_instance();
- let sum = match secp.lock().unwrap().commit_sum(
- vec![
- self.commit.clone(),
- other.commit.clone(),
- ],
- vec![],
- ) {
- Ok(s) => s,
- Err(_) => Commitment::from_vec(vec![1; 33]),
- };
-
- // Now build a new switch_commit_hash by concatenating the two switch_commit_hash value
- // and hashing the result.
- let mut bytes = self.switch_commit_hash.0.to_vec();
- bytes.extend(other.switch_commit_hash.0.iter().cloned());
- let key = SwitchCommitHashKey::zero();
- let hash = blake2b(SWITCH_COMMIT_HASH_SIZE, &key.0, &bytes);
- let hash = hash.as_bytes();
- let mut h = [0; SWITCH_COMMIT_HASH_SIZE];
- for i in 0..SWITCH_COMMIT_HASH_SIZE {
- h[i] = hash[i];
- }
- let switch_commit_hash_sum = SwitchCommitHash(h);
-
- SumCommit {
- features: self.features | other.features,
- commit: sum,
- switch_commit_hash: switch_commit_hash_sum,
- }
- }
-}
-
#[cfg(test)]
mod test {
use super::*;
diff --git a/core/src/ser.rs b/core/src/ser.rs
index 5f3c662c2..0d7f0bb8c 100644
--- a/core/src/ser.rs
+++ b/core/src/ser.rs
@@ -37,6 +37,8 @@ use util::secp::constants::{
SECRET_KEY_SIZE,
};
+const BULLET_PROOF_SIZE:usize = 674;
+
/// Possible errors deriving from serializing or deserializing.
#[derive(Debug)]
pub enum Error {
@@ -119,6 +121,9 @@ pub enum SerializationMode {
Hash,
/// Serialize everything that a signer of the object should know
SigHash,
+ /// Serialize for local storage, for instance in the case where
+ /// an output doesn't wish to store its range proof
+ Storage,
}
/// Implementations defined how different numbers and binary structures are
@@ -255,6 +260,7 @@ pub fn ser_vec<W: Writeable>(thing: &W) -> Result<Vec<u8>, Error> {
Ok(vec)
}
+/// Utility to read from a binary source
struct BinReader<'a> {
source: &'a mut Read,
}
@@ -364,7 +370,7 @@ impl Writeable for RangeProof {
impl Readable for RangeProof {
fn read(reader: &mut Reader) -> Result<RangeProof, Error> {
- let p = try!(reader.read_limited_vec(MAX_PROOF_SIZE));
+ let p = try!(reader.read_limited_vec(BULLET_PROOF_SIZE));
let mut a = [0; MAX_PROOF_SIZE];
for i in 0..p.len() {
a[i] = p[i];
@@ -376,6 +382,12 @@ impl Readable for RangeProof {
}
}
+impl PMMRable for RangeProof {
+ fn len() -> usize {
+ BULLET_PROOF_SIZE
+ }
+}
+
impl Readable for Signature {
fn read(reader: &mut Reader) -> Result<Signature, Error> {
let a = try!(reader.read_fixed_bytes(AGG_SIGNATURE_SIZE));
@@ -542,6 +554,12 @@ impl Writeable for [u8; 4] {
}
}
+/// Trait for types that can serialize and report their size
+pub trait PMMRable: Readable + Writeable + Hashed + Clone {
+ /// Length in bytes
+ fn len() -> usize;
+}
+
/// Useful marker trait on types that can be sized byte slices
pub trait AsFixedBytes: Sized + AsRef<[u8]> {
/// The length in bytes
diff --git a/grin/tests/api.rs b/grin/tests/api.rs
index 022a7af09..24dac9543 100644
--- a/grin/tests/api.rs
+++ b/grin/tests/api.rs
@@ -283,38 +283,38 @@ fn get_utxos_by_height(base_addr: &String, api_server_port: u16, start_height: u
// Sumtree handler functions
fn get_sumtree_roots(base_addr: &String, api_server_port: u16) -> Result<api::SumTrees, Error> {
- let url = format!("http://{}:{}/v1/sumtrees/roots", base_addr, api_server_port);
+ let url = format!("http://{}:{}/v1/pmmrtrees/roots", base_addr, api_server_port);
api::client::get::<api::SumTrees>(url.as_str()).map_err(|e| Error::API(e))
}
-fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
+fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::PmmrTreeNode>, Error> {
let url: String;
if n == 0 {
- url = format!("http://{}:{}/v1/sumtrees/lastutxos", base_addr, api_server_port);
+ url = format!("http://{}:{}/v1/pmmrtrees/lastutxos", base_addr, api_server_port);
} else {
- url = format!("http://{}:{}/v1/sumtrees/lastutxos?n={}", base_addr, api_server_port, n);
+ url = format!("http://{}:{}/v1/pmmrtrees/lastutxos?n={}", base_addr, api_server_port, n);
}
- api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
+ api::client::get::<Vec<api::PmmrTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
}
-fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
+fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::PmmrTreeNode>, Error> {
let url: String;
if n == 0 {
- url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs", base_addr, api_server_port);
+ url = format!("http://{}:{}/v1/pmmrtrees/lastrangeproofs", base_addr, api_server_port);
} else {
- url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs?n={}", base_addr, api_server_port, n);
+ url = format!("http://{}:{}/v1/pmmrtrees/lastrangeproofs?n={}", base_addr, api_server_port, n);
}
- api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
+ api::client::get::<Vec<api::PmmrTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
}
-fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
+fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::PmmrTreeNode>, Error> {
let url: String;
if n == 0 {
- url = format!("http://{}:{}/v1/sumtrees/lastkernels", base_addr, api_server_port);
+ url = format!("http://{}:{}/v1/pmmrtrees/lastkernels", base_addr, api_server_port);
} else {
- url = format!("http://{}:{}/v1/sumtrees/lastkernels?n={}", base_addr, api_server_port, n);
+ url = format!("http://{}:{}/v1/pmmrtrees/lastkernels?n={}", base_addr, api_server_port, n);
}
- api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
+ api::client::get::<Vec<api::PmmrTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
}
// Helper function to get a vec of commitment output ids from a vec of block outputs
diff --git a/keychain/src/keychain.rs b/keychain/src/keychain.rs
index 27e9f210b..39a5e622e 100644
--- a/keychain/src/keychain.rs
+++ b/keychain/src/keychain.rs
@@ -29,9 +29,9 @@ use blind::{BlindSum, BlindingFactor};
use extkey::{self, Identifier};
#[cfg(feature = "use-bullet-proofs")]
-const USE_BULLET_PROOFS:bool = true;
+pub const USE_BULLET_PROOFS:bool = true;
#[cfg(not(feature = "use-bullet-proofs"))]
-const USE_BULLET_PROOFS:bool = false;
+pub const USE_BULLET_PROOFS:bool = false;
#[derive(PartialEq, Eq, Clone, Debug)]
pub enum Error {
diff --git a/store/src/lib.rs b/store/src/lib.rs
index a3abdce49..745982d48 100644
--- a/store/src/lib.rs
+++ b/store/src/lib.rs
@@ -30,7 +30,8 @@ extern crate rocksdb;
#[macro_use]
extern crate slog;
-pub mod sumtree;
+pub mod pmmr;
+pub mod types;
const SEP: u8 = ':' as u8;
diff --git a/store/src/pmmr.rs b/store/src/pmmr.rs
new file mode 100644
index 000000000..df4e0213a
--- /dev/null
+++ b/store/src/pmmr.rs
@@ -0,0 +1,318 @@
+// Copyright 2017 The Grin Developers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Implementation of the persistent Backend for the prunable MMR tree.
+
+use std::fs::{self};
+use std::io::{self};
+use std::marker::PhantomData;
+
+use core::core::pmmr::{self, Backend};
+use core::ser::{self, PMMRable};
+use core::core::hash::Hash;
+use util::LOGGER;
+use types::{AppendOnlyFile, RemoveLog, read_ordered_vec, write_vec};
+
+const PMMR_HASH_FILE: &'static str = "pmmr_hash.bin";
+const PMMR_DATA_FILE: &'static str = "pmmr_data.bin";
+const PMMR_RM_LOG_FILE: &'static str = "pmmr_rm_log.bin";
+const PMMR_PRUNED_FILE: &'static str = "pmmr_pruned.bin";
+
+/// Maximum number of nodes in the remove log before it gets flushed
+pub const RM_LOG_MAX_NODES: usize = 10000;
+
+/// PMMR persistent backend implementation. Relies on multiple facilities to
+/// handle writing, reading and pruning.
+///
+/// * A main storage file appends Hash instances as they come. This
+/// AppendOnlyFile is also backed by a mmap for reads.
+/// * An in-memory backend buffers the latest batch of writes to ensure the
+/// PMMR can always read recent values even if they haven't been flushed to
+/// disk yet.
+/// * A remove log tracks the positions that need to be pruned from the
+/// main storage file.
+pub struct PMMRBackend<T>
+where
+ T: PMMRable,
+{
+ data_dir: String,
+ hash_file: AppendOnlyFile,
+ data_file: AppendOnlyFile,
+ rm_log: RemoveLog,
+ pruned_nodes: pmmr::PruneList,
+ phantom: PhantomData<T>,
+}
+
+impl<T> Backend<T> for PMMRBackend<T>
+where
+ T: PMMRable,
+{
+ /// Append the provided Hashes to the backend storage.
+ #[allow(unused_variables)]
+ fn append(&mut self, position: u64, data: Vec<(Hash, Option<T>)>) -> Result<(), String> {
+ for d in data {
+ self.hash_file.append(&mut ser::ser_vec(&d.0).unwrap());
+ if let Some(elem) = d.1 {
+ self.data_file.append(&mut ser::ser_vec(&elem).unwrap());
+ }
+ }
+ Ok(())
+ }
+
+ /// Get a Hash by insertion position
+ fn get(&self, position: u64, include_data:bool) -> Option<(Hash, Option<T>)> {
+ // Check if this position has been pruned in the remove log or the
+ // pruned list
+ if self.rm_log.includes(position) {
+ return None;
+ }
+ let shift = self.pruned_nodes.get_shift(position);
+ if let None = shift {
+ return None;
+ }
+
+ // Read PMMR
+ // The MMR starts at 1, our binary backend starts at 0
+ let pos = position - 1;
+
+ // Must be on disk, doing a read at the correct position
+ let hash_record_len = 32;
+ let file_offset = ((pos - shift.unwrap()) as usize) * hash_record_len;
+ let data = self.hash_file.read(file_offset, hash_record_len);
+ let hash_val = match ser::deserialize(&mut &data[..]) {
+ Ok(h) => h,
+ Err(e) => {
+ error!(
+ LOGGER,
+ "Corrupted storage, could not read an entry from hash store: {:?}",
+ e
+ );
+ return None;
+ }
+ };
+
+ if !include_data {
+ return Some(((hash_val), None));
+ }
+
+ // Optionally read flatfile storage to get data element
+ let flatfile_pos = pmmr::n_leaves(position)
+ - 1 - self.pruned_nodes.get_leaf_shift(position).unwrap();
+ let record_len = T::len();
+ let file_offset = flatfile_pos as usize * T::len();
+ let data = self.data_file.read(file_offset, record_len);
+ let data = match ser::deserialize(&mut &data[..]) {
+ Ok(elem) => Some(elem),
+ Err(e) => {
+ error!(
+ LOGGER,
+ "Corrupted storage, could not read an entry from backend flatfile store: {:?}",
+ e
+ );
+ None
+ }
+ };
+
+ Some((hash_val, data))
+ }
+
+ fn rewind(&mut self, position: u64, index: u32) -> Result<(), String> {
+ self.rm_log
+ .rewind(index)
+ .map_err(|e| format!("Could not truncate remove log: {}", e))?;
+
+ let shift = self.pruned_nodes.get_shift(position).unwrap_or(0);
+ let record_len = 32;
+ let file_pos = (position - shift) * (record_len as u64);
+ self.hash_file.rewind(file_pos);
+
+ //Data file
+ let flatfile_pos = pmmr::n_leaves(position) - 1;
+ let file_pos = (flatfile_pos as usize + 1) * T::len();
+ self.data_file.rewind(file_pos as u64);
+ Ok(())
+ }
+
+ /// Remove Hashes by insertion position
+ fn remove(&mut self, positions: Vec<u64>, index: u32) -> Result<(), String> {
+ self.rm_log.append(positions, index).map_err(|e| {
+ format!("Could not write to log storage, disk full? {:?}", e)
+ })
+ }
+
+ /// Return data file path
+ fn get_data_file_path(&self) -> String {
+ self.data_file.path()
+ }
+}
+
+impl<T> PMMRBackend<T>
+where
+ T: PMMRable,
+{
+ /// Instantiates a new PMMR backend that will use the provided directly to
+ /// store its files.
+ pub fn new(data_dir: String) -> io::Result<PMMRBackend<T>> {
+ let hash_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_HASH_FILE))?;
+ let rm_log = RemoveLog::open(format!("{}/{}", data_dir, PMMR_RM_LOG_FILE))?;
+ let prune_list = read_ordered_vec(format!("{}/{}", data_dir, PMMR_PRUNED_FILE), 8)?;
+ let data_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_DATA_FILE))?;
+
+ Ok(PMMRBackend {
+ data_dir: data_dir,
+ hash_file: hash_file,
+ data_file: data_file,
+ rm_log: rm_log,
+ pruned_nodes: pmmr::PruneList {
+ pruned_nodes: prune_list,
+ },
+ phantom: PhantomData,
+ })
+ }
+
+ /// Total size of the PMMR stored by this backend. Only produces the fully
+ /// sync'd size.
+ pub fn unpruned_size(&self) -> io::Result<u64> {
+ let total_shift = self.pruned_nodes.get_shift(::std::u64::MAX).unwrap();
+ let record_len = 32;
+ let sz = self.hash_file.size()?;
+ Ok(sz / record_len + total_shift)
+ }
+
+ /// Syncs all files to disk. A call to sync is required to ensure all the
+ /// data has been successfully written to disk.
+ pub fn sync(&mut self) -> io::Result<()> {
+ if let Err(e) = self.hash_file.flush() {
+ return Err(io::Error::new(
+ io::ErrorKind::Interrupted,
+ format!("Could not write to log hash storage, disk full? {:?}", e),
+ ));
+ }
+ if let Err(e) = self.data_file.flush() {
+ return Err(io::Error::new(
+ io::ErrorKind::Interrupted,
+ format!("Could not write to log data storage, disk full? {:?}", e),
+ ));
+ }
+ self.rm_log.flush()?;
+ Ok(())
+ }
+
+ /// Discard the current, non synced state of the backend.
+ pub fn discard(&mut self) {
+ self.hash_file.discard();
+ self.rm_log.discard();
+ self.data_file.discard();
+ }
+
+ /// Return the data file path
+ pub fn data_file_path(&self) -> String {
+ self.get_data_file_path()
+ }
+
+ /// Checks the length of the remove log to see if it should get compacted.
+ /// If so, the remove log is flushed into the pruned list, which itself gets
+ /// saved, and the main hashsum data file is rewritten, cutting the removed
+ /// data.
+ ///
+ /// If a max_len strictly greater than 0 is provided, the value will be used
+ /// to decide whether the remove log has reached its maximum length,
+ /// otherwise the RM_LOG_MAX_NODES default value is used.
+ ///
+ /// TODO whatever is calling this should also clean up the commit to
+ /// position index in db
+ pub fn check_compact(&mut self, max_len: usize) -> io::Result<()> {
+ if !(max_len > 0 && self.rm_log.len() > max_len
+ || max_len == 0 && self.rm_log.len() > RM_LOG_MAX_NODES)
+ {
+ return Ok(());
+ }
+
+ // 0. validate none of the nodes in the rm log are in the prune list (to
+ // avoid accidental double compaction)
+ for pos in &self.rm_log.removed[..] {
+ if let None = self.pruned_nodes.pruned_pos(pos.0) {
+ // TODO we likely can recover from this by directly jumping to 3
+ error!(
+ LOGGER,
+ "The remove log contains nodes that are already in the pruned \
+ list, a previous compaction likely failed."
+ );
+ return Ok(());
+ }
+ }
+
+ // 1. save hashsum file to a compact copy, skipping data that's in the
+ // remove list
+ let tmp_prune_file_hash = format!("{}/{}.hashprune", self.data_dir, PMMR_HASH_FILE);
+ let record_len = 32;
+ let to_rm = self.rm_log
+ .removed
+ .iter()
+ .map(|&(pos, _)| {
+ let shift = self.pruned_nodes.get_shift(pos);
+ (pos - 1 - shift.unwrap()) * record_len
+ })
+ .collect();
+ self.hash_file
+ .save_prune(tmp_prune_file_hash.clone(), to_rm, record_len)?;
+
+ // 2. And the same with the data file
+ let tmp_prune_file_data = format!("{}/{}.dataprune", self.data_dir, PMMR_DATA_FILE);
+ let record_len = T::len() as u64;
+ let to_rm = self.rm_log
+ .removed.clone()
+ .into_iter()
+ .filter(|&(pos, _)| pmmr::bintree_postorder_height(pos) == 0)
+ .map(|(pos, _)| {
+ let shift = self.pruned_nodes.get_leaf_shift(pos).unwrap();
+ let pos = pmmr::n_leaves(pos as u64);
+ (pos - 1 - shift) * record_len
+ })
+ .collect();
+
+ self.data_file
+ .save_prune(tmp_prune_file_data.clone(), to_rm, record_len)?;
+
+ // 3. update the prune list and save it in place
+ for &(rm_pos, _) in &self.rm_log.removed[..] {
+ self.pruned_nodes.add(rm_pos);
+ }
+ write_vec(
+ format!("{}/{}", self.data_dir, PMMR_PRUNED_FILE),
+ &self.pruned_nodes.pruned_nodes,
+ )?;
+
+ // 4. move the compact copy of hashes to the hashsum file and re-open it
+ fs::rename(
+ tmp_prune_file_hash.clone(),
+ format!("{}/{}", self.data_dir, PMMR_HASH_FILE),
+ )?;
+ self.hash_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_HASH_FILE))?;
+
+ // 5. and the same with the data file
+ fs::rename(
+ tmp_prune_file_data.clone(),
+ format!("{}/{}", self.data_dir, PMMR_DATA_FILE),
+ )?;
+ self.data_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE))?;
+
+ // 6. truncate the rm log
+ self.rm_log.rewind(0)?;
+ self.rm_log.flush()?;
+
+ Ok(())
+ }
+}
+
+
diff --git a/store/src/sumtree.rs b/store/src/types.rs
similarity index 54%
rename from store/src/sumtree.rs
rename to store/src/types.rs
index 2f9368105..f2bbff371 100644
--- a/store/src/sumtree.rs
+++ b/store/src/types.rs
@@ -1,4 +1,4 @@
-// Copyright 2017 The Grin Developers
+// Copyright 2018 The Grin Developers
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -11,33 +11,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-//! Implementation of the persistent Backend for the prunable MMR sum-tree.
-
+//! Common storage-related types
use memmap;
use std::cmp;
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufRead, BufReader, ErrorKind, Write};
-use std::marker::PhantomData;
use std::os::unix::io::AsRawFd;
-use std::path::Path;
use std::io::Read;
+use std::path::Path;
#[cfg(any(target_os = "linux"))]
use libc::{ftruncate64, off64_t};
#[cfg(not(any(target_os = "linux", target_os = "android")))]
use libc::{ftruncate as ftruncate64, off_t as off64_t};
-use core::core::pmmr::{self, Backend, HashSum, Summable};
use core::ser;
-use util::LOGGER;
-
-const PMMR_DATA_FILE: &'static str = "pmmr_dat.bin";
-const PMMR_RM_LOG_FILE: &'static str = "pmmr_rm_log.bin";
-const PMMR_PRUNED_FILE: &'static str = "pmmr_pruned.bin";
-
-/// Maximum number of nodes in the remove log before it gets flushed
-pub const RM_LOG_MAX_NODES: usize = 10000;
/// Wrapper for a file that can be read at any position (random read) but for
/// which writes are append only. Reads are backed by a memory map (mmap(2)),
@@ -54,6 +43,7 @@ pub struct AppendOnlyFile {
buffer_start: usize,
buffer: Vec<u8>,
buffer_start_bak: usize,
+ unflushed_data_size: usize,
}
impl AppendOnlyFile {
@@ -71,6 +61,7 @@ impl AppendOnlyFile {
buffer_start: 0,
buffer: vec![],
buffer_start_bak: 0,
+ unflushed_data_size: 0,
};
if let Ok(sz) = aof.size() {
if sz > 0 {
@@ -84,6 +75,7 @@ impl AppendOnlyFile {
/// Append data to the file. Until the append-only file is synced, data is
/// only written to memory.
pub fn append(&mut self, buf: &mut Vec<u8>) {
+ self.unflushed_data_size += buf.len();
self.buffer.append(buf);
}
@@ -110,6 +102,7 @@ impl AppendOnlyFile {
self.file.sync_data()?;
self.buffer = vec![];
self.mmap = Some(unsafe { memmap::Mmap::map(&self.file)? });
+ self.unflushed_data_size = 0;
Ok(())
}
@@ -121,11 +114,12 @@ impl AppendOnlyFile {
self.buffer_start_bak = 0;
}
self.buffer = vec![];
+ self.unflushed_data_size = 0;
}
/// Read length bytes of data at offset from the file. Leverages the memory
/// map.
- fn read(&self, offset: usize, length: usize) -> Vec<u8> {
+ pub fn read(&self, offset: usize, length: usize) -> Vec<u8> {
if offset >= self.buffer_start {
let offset = offset - self.buffer_start;
return self.buffer[offset..(offset+length)].to_vec();
@@ -138,7 +132,7 @@ impl AppendOnlyFile {
}
/// Truncates the underlying file to the provided offset
- fn truncate(&self, offs: usize) -> io::Result<()> {
+ pub fn truncate(&self, offs: usize) -> io::Result<()> {
let fd = self.file.as_raw_fd();
let res = unsafe { ftruncate64(fd, offs as off64_t) };
if res == -1 {
@@ -150,7 +144,7 @@ impl AppendOnlyFile {
/// Saves a copy of the current file content, skipping data at the provided
/// prune indices. The prune Vec must be ordered.
- fn save_prune(&self, target: String, prune_offs: Vec<u64>, prune_len: u64) -> io::Result<()> {
+ pub fn save_prune(&self, target: String, prune_offs: Vec<u64>, prune_len: u64) -> io::Result<()> {
let mut reader = File::open(self.path.clone())?;
let mut writer = File::create(target)?;
@@ -188,10 +182,15 @@ impl AppendOnlyFile {
}
/// Current size of the file in bytes.
- fn size(&self) -> io::Result<u64> {
+ pub fn size(&self) -> io::Result<u64> {
fs::metadata(&self.path).map(|md| md.len())
}
+ /// Current size of file in bytes + size of unsaved data
+ pub fn size_with_unsaved(&self) -> u64 {
+ self.size().unwrap() + self.unflushed_data_size as u64
+ }
+
/// Path of the underlying file
pub fn path(&self) -> String {
self.path.clone()
@@ -203,10 +202,10 @@ impl AppendOnlyFile {
/// checking of whether a piece of data has been marked for deletion. When the
/// log becomes too long, the MMR backend will actually remove chunks from the
/// MMR data file and truncate the remove log.
-struct RemoveLog {
+pub struct RemoveLog {
path: String,
- // Ordered vector of MMR positions that should get eventually removed.
- removed: Vec<(u64, u32)>,
+ /// Ordered vector of MMR positions that should get eventually removed.
+ pub removed: Vec<(u64, u32)>,
// Holds positions temporarily until flush is called.
removed_tmp: Vec<(u64, u32)>,
// Holds truncated removed temporarily until discarded or committed
@@ -216,7 +215,7 @@ struct RemoveLog {
impl RemoveLog {
/// Open the remove log file. The content of the file will be read in memory
/// for fast checking.
- fn open(path: String) -> io::Result<RemoveLog> {
+ pub fn open(path: String) -> io::Result<RemoveLog> {
let removed = read_ordered_vec(path.clone(), 12)?;
Ok(RemoveLog {
path: path,
@@ -227,7 +226,7 @@ impl RemoveLog {
}
/// Truncate and empties the remove log.
- fn rewind(&mut self, last_offs: u32) -> io::Result<()> {
+ pub fn rewind(&mut self, last_offs: u32) -> io::Result<()> {
// simplifying assumption: we always remove older than what's in tmp
self.removed_tmp = vec![];
@@ -251,7 +250,7 @@ impl RemoveLog {
/// Append a set of new positions to the remove log. Both adds those
/// positions the ordered in-memory set and to the file.
- fn append(&mut self, elmts: Vec<u64>, index: u32) -> io::Result<()> {
+ pub fn append(&mut self, elmts: Vec<u64>, index: u32) -> io::Result<()> {
for elmt in elmts {
match self.removed_tmp.binary_search(&(elmt, index)) {
Ok(_) => continue,
@@ -264,7 +263,7 @@ impl RemoveLog {
}
/// Flush the positions to remove to file.
- fn flush(&mut self) -> io::Result<()> {
+ pub fn flush(&mut self) -> io::Result<()> {
let mut file = File::create(self.path.clone())?;
for elmt in &self.removed_tmp {
match self.removed.binary_search(&elmt) {
@@ -283,7 +282,7 @@ impl RemoveLog {
}
/// Discard pending changes
- fn discard(&mut self) {
+ pub fn discard(&mut self) {
if self.removed_bak.len() > 0 {
self.removed = self.removed_bak.clone();
self.removed_bak = vec![];
@@ -292,12 +291,30 @@ impl RemoveLog {
}
/// Whether the remove log currently includes the provided position.
- fn includes(&self, elmt: u64) -> bool {
+ pub fn includes(&self, elmt: u64) -> bool {
include_tuple(&self.removed, elmt) || include_tuple(&self.removed_tmp, elmt)
}
+ /// How many removed positions exist before this particular position
+ pub fn get_shift(&self, elmt: u64) -> usize {
+ let mut complete_list = self.removed.clone();
+ for e in &self.removed_tmp {
+ match self.removed.binary_search(&e) {
+ Ok(_) => continue,
+ Err(idx) => {
+ complete_list.insert(idx, *e);
+ }
+ }
+ }
+ let pos = match complete_list.binary_search(&(elmt,0)){
+ Ok(idx) => idx+1,
+ Err(idx) => idx,
+ };
+ complete_list.split_at(pos).0.len()
+ }
+
/// Number of positions stored in the remove log.
- fn len(&self) -> usize {
+ pub fn len(&self) -> usize {
self.removed.len()
}
}
@@ -311,216 +328,8 @@ fn include_tuple(v: &Vec<(u64, u32)>, e: u64) -> bool {
false
}
-/// PMMR persistent backend implementation. Relies on multiple facilities to
-/// handle writing, reading and pruning.
-///
-/// * A main storage file appends HashSum instances as they come. This
-/// AppendOnlyFile is also backed by a mmap for reads.
-/// * An in-memory backend buffers the latest batch of writes to ensure the
-/// PMMR can always read recent values even if they haven't been flushed to
-/// disk yet.
-/// * A remove log tracks the positions that need to be pruned from the
-/// main storage file.
-pub struct PMMRBackend<T>
-where
- T: Summable + Clone,
-{
- data_dir: String,
- hashsum_file: AppendOnlyFile,
- remove_log: RemoveLog,
- pruned_nodes: pmmr::PruneList,
- phantom: PhantomData<T>,
-}
-
-impl<T> Backend<T> for PMMRBackend<T>
-where
- T: Summable + Clone,
-{
- /// Append the provided HashSums to the backend storage.
- #[allow(unused_variables)]
- fn append(&mut self, position: u64, data: Vec<HashSum<T>>) -> Result<(), String> {
- for d in data {
- self.hashsum_file.append(&mut ser::ser_vec(&d).unwrap());
- }
- Ok(())
- }
-
- /// Get a HashSum by insertion position
- fn get(&self, position: u64) -> Option<HashSum<T>> {
- // Check if this position has been pruned in the remove log or the
- // pruned list
- if self.remove_log.includes(position) {
- return None;
- }
- let shift = self.pruned_nodes.get_shift(position);
- if let None = shift {
- return None;
- }
-
- // The MMR starts at 1, our binary backend starts at 0
- let pos = position - 1;
-
- // Must be on disk, doing a read at the correct position
- let record_len = 32 + T::sum_len();
- let file_offset = ((pos - shift.unwrap()) as usize) * record_len;
- let data = self.hashsum_file.read(file_offset, record_len);
- match ser::deserialize(&mut &data[..]) {
- Ok(hashsum) => Some(hashsum),
- Err(e) => {
- error!(
- LOGGER,
- "Corrupted storage, could not read an entry from sum tree store: {:?}",
- e
- );
- None
- }
- }
- }
-
- fn rewind(&mut self, position: u64, index: u32) -> Result<(), String> {
- self.remove_log
- .rewind(index)
- .map_err(|e| format!("Could not truncate remove log: {}", e))?;
-
- let shift = self.pruned_nodes.get_shift(position).unwrap_or(0);
- let record_len = 32 + T::sum_len();
- let file_pos = (position - shift) * (record_len as u64);
- self.hashsum_file.rewind(file_pos);
- Ok(())
- }
-
- /// Remove HashSums by insertion position
- fn remove(&mut self, positions: Vec<u64>, index: u32) -> Result<(), String> {
- self.remove_log.append(positions, index).map_err(|e| {
- format!("Could not write to log storage, disk full? {:?}", e)
- })
- }
-}
-
-impl<T> PMMRBackend<T>
-where
- T: Summable + Clone,
-{
- /// Instantiates a new PMMR backend that will use the provided directly to
- /// store its files.
- pub fn new(data_dir: String) -> io::Result<PMMRBackend<T>> {
- let hs_file = AppendOnlyFile::open(format!("{}/{}", data_dir, PMMR_DATA_FILE))?;
- let rm_log = RemoveLog::open(format!("{}/{}", data_dir, PMMR_RM_LOG_FILE))?;
- let prune_list = read_ordered_vec(format!("{}/{}", data_dir, PMMR_PRUNED_FILE), 8)?;
-
- Ok(PMMRBackend {
- data_dir: data_dir,
- hashsum_file: hs_file,
- remove_log: rm_log,
- pruned_nodes: pmmr::PruneList {
- pruned_nodes: prune_list,
- },
- phantom: PhantomData,
- })
- }
-
- /// Total size of the PMMR stored by this backend. Only produces the fully
- /// sync'd size.
- pub fn unpruned_size(&self) -> io::Result<u64> {
- let total_shift = self.pruned_nodes.get_shift(::std::u64::MAX).unwrap();
- let record_len = 32 + T::sum_len() as u64;
- let sz = self.hashsum_file.size()?;
- Ok(sz / record_len + total_shift)
- }
-
- /// Syncs all files to disk. A call to sync is required to ensure all the
- /// data has been successfully written to disk.
- pub fn sync(&mut self) -> io::Result<()> {
- if let Err(e) = self.hashsum_file.flush() {
- return Err(io::Error::new(
- io::ErrorKind::Interrupted,
- format!("Could not write to log storage, disk full? {:?}", e),
- ));
- }
-
- self.remove_log.flush()?;
- Ok(())
- }
-
- /// Discard the current, non synced state of the backend.
- pub fn discard(&mut self) {
- self.hashsum_file.discard();
- self.remove_log.discard();
- }
-
- /// Checks the length of the remove log to see if it should get compacted.
- /// If so, the remove log is flushed into the pruned list, which itself gets
- /// saved, and the main hashsum data file is rewritten, cutting the removed
- /// data.
- ///
- /// If a max_len strictly greater than 0 is provided, the value will be used
- /// to decide whether the remove log has reached its maximum length,
- /// otherwise the RM_LOG_MAX_NODES default value is used.
- ///
- /// TODO whatever is calling this should also clean up the commit to
- /// position index in db
- pub fn check_compact(&mut self, max_len: usize) -> io::Result<()> {
- if !(max_len > 0 && self.remove_log.len() > max_len
- || max_len == 0 && self.remove_log.len() > RM_LOG_MAX_NODES)
- {
- return Ok(());
- }
-
- // 0. validate none of the nodes in the rm log are in the prune list (to
- // avoid accidental double compaction)
- for pos in &self.remove_log.removed[..] {
- if let None = self.pruned_nodes.pruned_pos(pos.0) {
- // TODO we likely can recover from this by directly jumping to 3
- error!(
- LOGGER,
- "The remove log contains nodes that are already in the pruned \
- list, a previous compaction likely failed."
- );
- return Ok(());
- }
- }
-
- // 1. save hashsum file to a compact copy, skipping data that's in the
- // remove list
- let tmp_prune_file = format!("{}/{}.prune", self.data_dir, PMMR_DATA_FILE);
- let record_len = (32 + T::sum_len()) as u64;
- let to_rm = self.remove_log
- .removed
- .iter()
- .map(|&(pos, _)| {
- let shift = self.pruned_nodes.get_shift(pos);
- (pos - 1 - shift.unwrap()) * record_len
- })
- .collect();
- self.hashsum_file
- .save_prune(tmp_prune_file.clone(), to_rm, record_len)?;
-
- // 2. update the prune list and save it in place
- for &(rm_pos, _) in &self.remove_log.removed[..] {
- self.pruned_nodes.add(rm_pos);
- }
- write_vec(
- format!("{}/{}", self.data_dir, PMMR_PRUNED_FILE),
- &self.pruned_nodes.pruned_nodes,
- )?;
-
- // 3. move the compact copy to the hashsum file and re-open it
- fs::rename(
- tmp_prune_file.clone(),
- format!("{}/{}", self.data_dir, PMMR_DATA_FILE),
- )?;
- self.hashsum_file = AppendOnlyFile::open(format!("{}/{}", self.data_dir, PMMR_DATA_FILE))?;
-
- // 4. truncate the rm log
- self.remove_log.rewind(0)?;
- self.remove_log.flush()?;
-
- Ok(())
- }
-}
-
-// Read an ordered vector of scalars from a file.
-fn read_ordered_vec<T>(path: String, elmt_len: usize) -> io::Result<Vec<T>>
+/// Read an ordered vector of scalars from a file.
+pub fn read_ordered_vec<T>(path: String, elmt_len: usize) -> io::Result<Vec<T>>
where
T: ser::Readable + cmp::Ord,
{
@@ -557,7 +366,8 @@ where
Ok(ovec)
}
-fn write_vec<T>(path: String, v: &Vec<T>) -> io::Result<()>
+/// Writes an ordered vector to a file
+pub fn write_vec<T>(path: String, v: &Vec<T>) -> io::Result<()>
where
T: ser::Writeable,
{
diff --git a/store/tests/sumtree.rs b/store/tests/pmmr.rs
similarity index 62%
rename from store/tests/sumtree.rs
rename to store/tests/pmmr.rs
index 7b4139cce..00c4becff 100644
--- a/store/tests/sumtree.rs
+++ b/store/tests/pmmr.rs
@@ -20,13 +20,13 @@ extern crate time;
use std::fs;
use core::ser::*;
-use core::core::pmmr::{Backend, HashSum, Summable, PMMR};
-use core::core::hash::Hashed;
+use core::core::pmmr::{PMMR, Backend};
+use core::core::hash::{Hash, Hashed};
#[test]
-fn sumtree_append() {
+fn pmmr_append() {
let (data_dir, elems) = setup("append");
- let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
+ let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
// adding first set of 4 elements and sync
let mut mmr_size = load(0, &elems[0..4], &mut backend);
@@ -37,31 +37,22 @@ fn sumtree_append() {
backend.sync().unwrap();
// check the resulting backend store and the computation of the root
- let hash = Hashed::hash(&elems[0].clone());
- let sum = elems[0].sum();
- let node_hash = (1 as u64, &sum, hash).hash();
+ let node_hash = elems[0].hash();
assert_eq!(
- backend.get(1),
- Some(HashSum {
- hash: node_hash,
- sum: sum,
- })
+ backend.get(1, false).expect("").0,
+ node_hash
);
- let sum2 = HashSum::from_summable(1, &elems[0])
- + HashSum::from_summable(2, &elems[1]);
+ let sum2 = elems[0].hash() + elems[1].hash();
let sum4 = sum2
- + (HashSum::from_summable(4, &elems[2])
- + HashSum::from_summable(5, &elems[3]));
+ + (elems[2].hash() + elems[3].hash());
let sum8 = sum4
- + ((HashSum::from_summable(8, &elems[4])
- + HashSum::from_summable(9, &elems[5]))
- + (HashSum::from_summable(11, &elems[6])
- + HashSum::from_summable(12, &elems[7])));
- let sum9 = sum8 + HashSum::from_summable(16, &elems[8]);
+ + ((elems[4].hash() + elems[5].hash())
+ + (elems[6].hash() + elems[7].hash()));
+ let sum9 = sum8 + elems[8].hash();
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(pmmr.root(), sum9);
}
@@ -69,75 +60,81 @@ fn sumtree_append() {
}
#[test]
-fn sumtree_prune_compact() {
+fn pmmr_prune_compact() {
let (data_dir, elems) = setup("prune_compact");
// setup the mmr store with all elements
- let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
+ let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
let mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
// save the root
- let root: HashSum<TestElem>;
+ let root: Hash;
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
root = pmmr.root();
}
// pruning some choice nodes
{
- let mut pmmr = PMMR::at(&mut backend, mmr_size);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.prune(1, 1).unwrap();
pmmr.prune(4, 1).unwrap();
pmmr.prune(5, 1).unwrap();
}
backend.sync().unwrap();
- // check the root
+ // check the root and stored data
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(root, pmmr.root());
+ // check we can still retrieve same element from leaf index 2
+ assert_eq!(pmmr.get(2, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 2]));
}
// compact
backend.check_compact(2).unwrap();
- // recheck the root
+ // recheck the root and stored data
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(root, pmmr.root());
+ assert_eq!(pmmr.get(2, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 2]));
+ assert_eq!(pmmr.get(11, true).unwrap().1.unwrap(), TestElem([0, 0, 0, 7]));
}
teardown(data_dir);
}
#[test]
-fn sumtree_reload() {
+fn pmmr_reload() {
let (data_dir, elems) = setup("reload");
// set everything up with a first backend
let mmr_size: u64;
- let root: HashSum<TestElem>;
+ let root: Hash;
{
- let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
+ let mut backend = store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
mmr_size = load(0, &elems[..], &mut backend);
backend.sync().unwrap();
// save the root and prune some nodes so we have prune data
{
- let mut pmmr = PMMR::at(&mut backend, mmr_size);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
+ pmmr.dump(false);
root = pmmr.root();
pmmr.prune(1, 1).unwrap();
pmmr.prune(4, 1).unwrap();
}
backend.sync().unwrap();
+
backend.check_compact(1).unwrap();
backend.sync().unwrap();
assert_eq!(backend.unpruned_size().unwrap(), mmr_size);
// prune some more to get rm log data
{
- let mut pmmr = PMMR::at(&mut backend, mmr_size);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.prune(5, 1).unwrap();
}
backend.sync().unwrap();
@@ -146,37 +143,38 @@ fn sumtree_reload() {
// create a new backend and check everything is kosher
{
- let mut backend = store::sumtree::PMMRBackend::new(data_dir.to_string()).unwrap();
+ let mut backend:store::pmmr::PMMRBackend<TestElem> =
+ store::pmmr::PMMRBackend::new(data_dir.to_string()).unwrap();
assert_eq!(backend.unpruned_size().unwrap(), mmr_size);
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
assert_eq!(root, pmmr.root());
}
- assert_eq!(backend.get(5), None);
+ assert_eq!(backend.get(5, false), None);
}
teardown(data_dir);
}
#[test]
-fn sumtree_rewind() {
+fn pmmr_rewind() {
let (data_dir, elems) = setup("rewind");
- let mut backend = store::sumtree::PMMRBackend::new(data_dir.clone()).unwrap();
+ let mut backend = store::pmmr::PMMRBackend::new(data_dir.clone()).unwrap();
// adding elements and keeping the corresponding root
let mut mmr_size = load(0, &elems[0..4], &mut backend);
backend.sync().unwrap();
- let root1: HashSum<TestElem>;
+ let root1: Hash;
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
root1 = pmmr.root();
}
mmr_size = load(mmr_size, &elems[4..6], &mut backend);
backend.sync().unwrap();
- let root2: HashSum<TestElem>;
+ let root2: Hash;
{
- let pmmr = PMMR::at(&mut backend, mmr_size);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
root2 = pmmr.root();
}
@@ -185,7 +183,7 @@ fn sumtree_rewind() {
// prune and compact the 2 first elements to spice things up
{
- let mut pmmr = PMMR::at(&mut backend, mmr_size);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.prune(1, 1).unwrap();
pmmr.prune(2, 1).unwrap();
}
@@ -194,24 +192,24 @@ fn sumtree_rewind() {
// rewind and check the roots still match
{
- let mut pmmr = PMMR::at(&mut backend, mmr_size);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, mmr_size);
pmmr.rewind(9, 3).unwrap();
assert_eq!(pmmr.root(), root2);
}
backend.sync().unwrap();
{
- let pmmr = PMMR::at(&mut backend, 10);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, 10);
assert_eq!(pmmr.root(), root2);
}
{
- let mut pmmr = PMMR::at(&mut backend, 10);
+ let mut pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, 10);
pmmr.rewind(5, 3).unwrap();
assert_eq!(pmmr.root(), root1);
}
backend.sync().unwrap();
{
- let pmmr = PMMR::at(&mut backend, 7);
+ let pmmr:PMMR<TestElem, _> = PMMR::at(&mut backend, 7);
assert_eq!(pmmr.root(), root1);
}
@@ -242,7 +240,7 @@ fn teardown(data_dir: String) {
fs::remove_dir_all(data_dir).unwrap();
}
-fn load(pos: u64, elems: &[TestElem], backend: &mut store::sumtree::PMMRBackend<TestElem>) -> u64 {
+fn load(pos: u64, elems: &[TestElem], backend: &mut store::pmmr::PMMRBackend<TestElem>) -> u64 {
let mut pmmr = PMMR::at(backend, pos);
for elem in elems {
pmmr.push(elem.clone()).unwrap();
@@ -252,16 +250,10 @@ fn load(pos: u64, elems: &[TestElem], backend: &mut store::sumtree::PMMRBackend<
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
struct TestElem([u32; 4]);
-impl Summable for TestElem {
- type Sum = u64;
- fn sum(&self) -> u64 {
- // sums are not allowed to overflow, so we use this simple
- // non-injective "sum" function that will still be homomorphic
- self.0[0] as u64 * 0x1000 + self.0[1] as u64 * 0x100 + self.0[2] as u64 * 0x10
- + self.0[3] as u64
- }
- fn sum_len() -> usize {
- 8
+
+impl PMMRable for TestElem {
+ fn len() -> usize {
+ 16
}
}
@@ -273,3 +265,15 @@ impl Writeable for TestElem {
writer.write_u32(self.0[3])
}
}
+impl Readable for TestElem {
+ fn read(reader: &mut Reader) -> Result<TestElem, Error> {
+ Ok(TestElem (
+ [
+ reader.read_u32()?,
+ reader.read_u32()?,
+ reader.read_u32()?,
+ reader.read_u32()?,
+ ]
+ ))
+ }
+}