Kernel sum and MMR sizes in block header (#1163)

* Add kernel commitments sum and kernel and output MMR sizes to block header
* Sum a block without including previous sums, cleanup. Blocks are now summed and validated based on their own totals and not the totals since genesis. This allows to get rid of BlockSum and simplified the setting of a new block's roots, kernel sum and MMR sizes. Fixes #116
* Additional kernel MMR validation to check all prior header roots successively
* Wallet tests fix

api/src/rest.rs

@@ -24,9 +24,9 @@ use std::net::ToSocketAddrs;
 use std::string::ToString;
 
 use failure::{Backtrace, Context, Fail};
+use iron::Listening;
 use iron::middleware::Handler;
 use iron::prelude::Iron;
-use iron::Listening;
 use mount::Mount;
 use router::Router;
 

chain/src/chain.rs

@@ -20,19 +20,19 @@ use std::fs::File;
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};
 
+use core::core::Committed;
 use core::core::hash::{Hash, Hashed};
 use core::core::merkle_proof::MerkleProof;
 use core::core::target::Difficulty;
-use core::core::Committed;
 use core::core::{Block, BlockHeader, Output, OutputIdentifier, Transaction, TxKernel};
 use core::global;
 use grin_store::Error::NotFoundErr;
 use pipe;
 use store;
 use txhashset;
-use types::{BlockMarker, BlockSums, ChainAdapter, ChainStore, Error, Options, Tip};
+use types::{BlockMarker, ChainAdapter, ChainStore, Error, Options, Tip};
-use util::secp::pedersen::{Commitment, RangeProof};
 use util::LOGGER;
+use util::secp::pedersen::{Commitment, RangeProof};
 
 /// Orphan pool size is limited by MAX_ORPHAN_SIZE
 pub const MAX_ORPHAN_SIZE: usize = 200;
@@ -191,36 +191,8 @@ impl Chain {
 						);
 
 						extension.rewind(&header, &head_header)?;
-
 						extension.validate_roots(&header)?;
 
-						// now check we have the "block sums" for the block in question
-						// if we have no sums (migrating an existing node) we need to go
-						// back to the txhashset and sum the outputs and kernels
-						if header.height > 0 && store.get_block_sums(&header.hash()).is_err() {
-							debug!(
-								LOGGER,
-								"chain: init: building (missing) block sums for {} @ {}",
-								header.height,
-								header.hash()
-							);
-
-							let (output_sum, kernel_sum) = extension.verify_kernel_sums(
-								header.total_overage(),
-								header.total_kernel_offset(),
-								None,
-								None,
-							)?;
-
-							store.save_block_sums(
-								&header.hash(),
-								&BlockSums {
-									output_sum,
-									kernel_sum,
-								},
-							)?;
-						}
-
 						Ok(())
 					});
 
@@ -539,21 +511,23 @@ impl Chain {
 
 	/// Sets the txhashset roots on a brand new block by applying the block on
 	/// the current txhashset state.
-	pub fn set_txhashset_roots(&self, b: &mut Block, is_fork: bool) -> Result<(), Error> {
+	pub fn set_block_roots(&self, b: &mut Block, is_fork: bool) -> Result<(), Error> {
 		let mut txhashset = self.txhashset.write().unwrap();
 		let store = self.store.clone();
 
-		let roots = txhashset::extending_readonly(&mut txhashset, |extension| {
+		let (roots, sizes) = txhashset::extending_readonly(&mut txhashset, |extension| {
 			if is_fork {
 				pipe::rewind_and_apply_fork(b, store, extension)?;
 			}
 			extension.apply_block(b)?;
-			Ok(extension.roots())
+			Ok((extension.roots(), extension.sizes()))
 		})?;
 
 		b.header.output_root = roots.output_root;
 		b.header.range_proof_root = roots.rproof_root;
 		b.header.kernel_root = roots.kernel_root;
+		b.header.output_mmr_size = sizes.0;
+		b.header.kernel_mmr_size = sizes.2;
 		Ok(())
 	}
 
@@ -654,13 +628,18 @@ impl Chain {
 		let mut txhashset =
 			txhashset::TxHashSet::open(self.db_root.clone(), self.store.clone(), Some(&header))?;
 
-		// Note: we are validating against a writeable extension.
+		// first read-only extension, for validation only
+		txhashset::extending_readonly(&mut txhashset, |extension| {
+			extension.rewind(&header)?;
+			extension.validate(&header, false)?;
+			Ok(())
+		})?;
+		// second real extension to commit the rewind and indexes
 		txhashset::extending(&mut txhashset, |extension| {
 			// TODO do we need to rewind here? We have no blocks to rewind
 			// (and we need them for the pos to unremove)
 			extension.rewind(&header, &header)?;
-			let (output_sum, kernel_sum) = extension.validate(&header, false)?;
+			extension.validate(&header, false)?;
-			extension.save_latest_block_sums(&header, output_sum, kernel_sum)?;
 			extension.rebuild_index()?;
 			Ok(())
 		})?;
@@ -739,7 +718,6 @@ impl Chain {
 					// TODO - consider wrapping these up in a single fn call?
 					self.store.delete_block(&b.hash())?;
 					self.store.delete_block_marker(&b.hash())?;
-					self.store.delete_block_sums(&b.hash())?;
 					self.store.delete_block_input_bitmap(&b.hash())?;
 				}
 				Err(NotFoundErr) => {
@@ -859,13 +837,6 @@ impl Chain {
 			.map_err(|e| Error::StoreErr(e, "chain get block marker".to_owned()))
 	}
 
-	/// Get the blocks sums for the specified block hash.
-	pub fn get_block_sums(&self, bh: &Hash) -> Result<BlockSums, Error> {
-		self.store
-			.get_block_sums(bh)
-			.map_err(|e| Error::StoreErr(e, "chain get block sums".to_owned()))
-	}
-
 	/// Gets the block header at the provided height
 	pub fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> {
 		self.store

chain/src/lib.rs

@@ -46,4 +46,4 @@ pub mod types;
 // Re-export the base interface
 
 pub use chain::{Chain, MAX_ORPHAN_SIZE};
-pub use types::{BlockSums, ChainAdapter, ChainStore, Error, Options, Tip};
+pub use types::{ChainAdapter, ChainStore, Error, Options, Tip};

chain/src/pipe.rs

@@ -26,7 +26,7 @@ use core::global;
 use grin_store;
 use store;
 use txhashset;
-use types::{BlockSums, ChainStore, Error, Options, Tip};
+use types::{ChainStore, Error, Options, Tip};
 use util::LOGGER;
 
 /// Contextual information required to process a new block and either reject or
@@ -308,25 +308,9 @@ fn validate_header(header: &BlockHeader, ctx: &mut BlockContext) -> Result<(), E
 }
 
 fn validate_block(b: &Block, ctx: &mut BlockContext) -> Result<(), Error> {
-	// If this is the first block then we have no previous block sums stored.
+	let prev = ctx.store.get_block_header(&b.header.previous)?;
-	let block_sums = if b.header.height == 1 {
+	b.validate(&prev.total_kernel_offset, &prev.total_kernel_sum)
-		BlockSums::default()
+		.map_err(&Error::InvalidBlockProof)?;
-	} else {
-		ctx.store.get_block_sums(&b.header.previous)?
-	};
-
-	let (new_output_sum, new_kernel_sum) =
-		b.validate(&block_sums.output_sum, &block_sums.kernel_sum)
-			.map_err(&Error::InvalidBlockProof)?;
-
-	ctx.store.save_block_sums(
-		&b.hash(),
-		&BlockSums {
-			output_sum: new_output_sum,
-			kernel_sum: new_kernel_sum,
-		},
-	)?;
-
 	Ok(())
 }
 
@@ -369,6 +353,10 @@ fn validate_block_via_txhashset(b: &Block, ext: &mut txhashset::Extension) -> Re
 
 		return Err(Error::InvalidRoot);
 	}
+	let sizes = ext.sizes();
+	if b.header.output_mmr_size != sizes.0 || b.header.kernel_mmr_size != sizes.2 {
+		return Err(Error::InvalidMMRSize);
+	}
 
 	Ok(())
 }

chain/src/store.rs

@@ -25,8 +25,8 @@ use core::consensus::TargetError;
 use core::core::hash::{Hash, Hashed};
 use core::core::target::Difficulty;
 use core::core::{Block, BlockHeader};
-use grin_store::{self, option_to_not_found, to_key, u64_to_key, Error};
+use grin_store::{self, option_to_not_found, to_key, Error, u64_to_key};
-use types::{BlockMarker, BlockSums, ChainStore, Tip};
+use types::{BlockMarker, ChainStore, Tip};
 
 const STORE_SUBPATH: &'static str = "chain";
 
@@ -38,7 +38,6 @@ const SYNC_HEAD_PREFIX: u8 = 's' as u8;
 const HEADER_HEIGHT_PREFIX: u8 = '8' as u8;
 const COMMIT_POS_PREFIX: u8 = 'c' as u8;
 const BLOCK_MARKER_PREFIX: u8 = 'm' as u8;
-const BLOCK_SUMS_PREFIX: u8 = 'M' as u8;
 const BLOCK_INPUT_BITMAP_PREFIX: u8 = 'B' as u8;
 
 /// An implementation of the ChainStore trait backed by a simple key-value
@@ -279,22 +278,6 @@ impl ChainStore for ChainKVStore {
 			.delete(&to_key(BLOCK_MARKER_PREFIX, &mut bh.to_vec()))
 	}
 
-	fn save_block_sums(&self, bh: &Hash, marker: &BlockSums) -> Result<(), Error> {
-		self.db
-			.put_ser(&to_key(BLOCK_SUMS_PREFIX, &mut bh.to_vec())[..], &marker)
-	}
-
-	fn get_block_sums(&self, bh: &Hash) -> Result<BlockSums, Error> {
-		option_to_not_found(
-			self.db
-				.get_ser(&to_key(BLOCK_SUMS_PREFIX, &mut bh.to_vec())),
-		)
-	}
-
-	fn delete_block_sums(&self, bh: &Hash) -> Result<(), Error> {
-		self.db.delete(&to_key(BLOCK_SUMS_PREFIX, &mut bh.to_vec()))
-	}
-
 	fn get_block_input_bitmap(&self, bh: &Hash) -> Result<Bitmap, Error> {
 		{
 			let mut cache = self.block_input_bitmap_cache.write().unwrap();

chain/src/txhashset.rs

@@ -39,8 +39,8 @@ use core::ser::{PMMRIndexHashable, PMMRable};
 use grin_store;
 use grin_store::pmmr::PMMRBackend;
 use grin_store::types::prune_noop;
-use types::{BlockMarker, BlockSums, ChainStore, Error, TxHashSetRoots};
+use types::{BlockMarker, ChainStore, Error, TxHashSetRoots};
-use util::{secp_static, zip, LOGGER};
+use util::{zip, LOGGER};
 
 const TXHASHSET_SUBDIR: &'static str = "txhashset";
 const OUTPUT_SUBDIR: &'static str = "output";
@@ -825,9 +825,8 @@ impl<'a> Extension<'a> {
 
 	/// Validate the various MMR roots against the block header.
 	pub fn validate_roots(&self, header: &BlockHeader) -> Result<(), Error> {
-		// If we are validating the genesis block then
+		// If we are validating the genesis block then we have no outputs or
-		// we have no outputs or kernels.
+		// kernels. So we are done here.
-		// So we are done here.
 		if header.height == 0 {
 			return Ok(());
 		}
@@ -865,28 +864,18 @@ impl<'a> Extension<'a> {
 	}
 
 	/// Validate the txhashset state against the provided block header.
-	pub fn validate(
+	pub fn validate(&mut self, header: &BlockHeader, skip_rproofs: bool) -> Result<(), Error> {
-		&mut self,
-		header: &BlockHeader,
-		skip_rproofs: bool,
-	) -> Result<((Commitment, Commitment)), Error> {
 		self.validate_mmrs()?;
 		self.validate_roots(header)?;
 
 		if header.height == 0 {
-			let zero_commit = secp_static::commit_to_zero_value();
+			return Ok(());
-			return Ok((zero_commit.clone(), zero_commit.clone()));
 		}
 
 		// The real magicking happens here.
 		// Sum of kernel excesses should equal sum of
 		// unspent outputs minus total supply.
-		let (output_sum, kernel_sum) = self.verify_kernel_sums(
+		self.verify_kernel_sums(header.total_overage(), header.total_kernel_offset())?;
-			header.total_overage(),
-			header.total_kernel_offset(),
-			None,
-			None,
-		)?;
 
 		// This is an expensive verification step.
 		self.verify_kernel_signatures()?;
@@ -897,24 +886,10 @@ impl<'a> Extension<'a> {
 			self.verify_rangeproofs()?;
 		}
 
-		Ok((output_sum, kernel_sum))
+		// Verify kernel roots for all past headers, need to be last as it rewinds
-	}
+		// a lot without resetting
+		self.verify_kernel_history(header)?;
 
-	/// Save blocks sums (the output_sum and kernel_sum) for the given block
-	/// header.
-	pub fn save_latest_block_sums(
-		&self,
-		header: &BlockHeader,
-		output_sum: Commitment,
-		kernel_sum: Commitment,
-	) -> Result<(), Error> {
-		self.commit_index.save_block_sums(
-			&header.hash(),
-			&BlockSums {
-				output_sum,
-				kernel_sum,
-			},
-		)?;
 		Ok(())
 	}
 
@@ -961,8 +936,8 @@ impl<'a> Extension<'a> {
 		}
 	}
 
-	// Sizes of the sum trees, used by `extending` on rollback.
+	/// Sizes of each of the sum trees
-	fn sizes(&self) -> (u64, u64, u64) {
+	pub fn sizes(&self) -> (u64, u64, u64) {
 		(
 			self.output_pmmr.unpruned_size(),
 			self.rproof_pmmr.unpruned_size(),
@@ -1027,6 +1002,32 @@ impl<'a> Extension<'a> {
 		);
 		Ok(())
 	}
+
+	fn verify_kernel_history(&mut self, header: &BlockHeader) -> Result<(), Error> {
+		// Special handling to make sure the whole kernel set matches each of its
+		// roots in each block header, without truncation. We go back header by
+		// header, rewind and check each root. This fixes a potential weakness in
+		// fast sync where a reorg past the horizon could allow a whole rewrite of
+		// the kernel set.
+		let mut current = header.clone();
+		loop {
+			current = self.commit_index.get_block_header(&current.previous)?;
+			if current.height == 0 {
+				break;
+			}
+			// rewinding further and further back
+			self.kernel_pmmr
+				.rewind(current.kernel_mmr_size, current.height as u32)
+				.map_err(&Error::TxHashSetErr)?;
+			if self.kernel_pmmr.root() != current.kernel_root {
+				return Err(Error::InvalidTxHashSet(format!(
+					"Kernel root at {} does not match",
+					current.height
+				)));
+			}
+		}
+		Ok(())
+	}
 }
 
 /// Packages the txhashset data files into a zip and returns a Read to the

chain/src/types.rs

@@ -20,7 +20,6 @@ use croaring::Bitmap;
 
 use util::secp;
 use util::secp::pedersen::Commitment;
-use util::secp_static;
 
 use core::core::committed;
 use core::core::hash::{Hash, Hashed};
@@ -77,6 +76,8 @@ pub enum Error {
 	InvalidBlockHeight,
 	/// One of the root hashes in the block is invalid
 	InvalidRoot,
+	/// One of the MMR sizes in the block header is invalid
+	InvalidMMRSize,
 	/// Error from underlying keychain impl
 	Keychain(keychain::Error),
 	/// Error from underlying secp lib
@@ -343,15 +344,6 @@ pub trait ChainStore: Send + Sync {
 	/// Deletes a block marker associated with the provided hash
 	fn delete_block_marker(&self, bh: &Hash) -> Result<(), store::Error>;
 
-	/// Save block sums for the given block hash.
-	fn save_block_sums(&self, bh: &Hash, marker: &BlockSums) -> Result<(), store::Error>;
-
-	/// Get block sums for the given block hash.
-	fn get_block_sums(&self, bh: &Hash) -> Result<BlockSums, store::Error>;
-
-	/// Delete block sums for the given block hash.
-	fn delete_block_sums(&self, bh: &Hash) -> Result<(), store::Error>;
-
 	/// Get the bitmap representing the inputs for the specified block.
 	fn get_block_input_bitmap(&self, bh: &Hash) -> Result<Bitmap, store::Error>;
 
@@ -422,42 +414,3 @@ impl Default for BlockMarker {
 		}
 	}
 }
-
-/// The output_sum and kernel_sum for a given block.
-/// This is used to validate the next block being processed by applying
-/// the inputs, outputs, kernels and kernel_offset from the new block
-/// and checking everything sums correctly.
-#[derive(Debug, Clone)]
-pub struct BlockSums {
-	/// The total output sum so far.
-	pub output_sum: Commitment,
-	/// The total kernel sum so far.
-	pub kernel_sum: Commitment,
-}
-
-impl Writeable for BlockSums {
-	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
-		writer.write_fixed_bytes(&self.output_sum)?;
-		writer.write_fixed_bytes(&self.kernel_sum)?;
-		Ok(())
-	}
-}
-
-impl Readable for BlockSums {
-	fn read(reader: &mut Reader) -> Result<BlockSums, ser::Error> {
-		Ok(BlockSums {
-			output_sum: Commitment::read(reader)?,
-			kernel_sum: Commitment::read(reader)?,
-		})
-	}
-}
-
-impl Default for BlockSums {
-	fn default() -> BlockSums {
-		let zero_commit = secp_static::commit_to_zero_value();
-		BlockSums {
-			output_sum: zero_commit.clone(),
-			kernel_sum: zero_commit.clone(),
-		}
-	}
-}

chain/tests/data_file_integrity.rs

@@ -24,8 +24,8 @@ extern crate time;
 use std::fs;
 use std::sync::Arc;
 
-use chain::types::{NoopAdapter, Tip};
 use chain::Chain;
+use chain::types::{NoopAdapter, Tip};
 use core::core::target::Difficulty;
 use core::core::{Block, BlockHeader, Transaction};
 use core::global::{self, ChainTypes};
@@ -76,7 +76,7 @@ fn data_files() {
 			let mut b = core::core::Block::new(&prev, vec![], difficulty.clone(), reward).unwrap();
 			b.header.timestamp = prev.timestamp + time::Duration::seconds(60);
 
-			chain.set_txhashset_roots(&mut b, false).unwrap();
+			chain.set_block_roots(&mut b, false).unwrap();
 
 			pow::pow_size(
 				&mut b.header,
@@ -93,14 +93,13 @@ fn data_files() {
 			let head = Tip::from_block(&b.header);
 
 			// Check we have block markers for the last block and the block previous
-			let cur_pmmr_md = chain
+			let _cur_pmmr_md = chain
 				.get_block_marker(&head.last_block_h)
 				.expect("block marker does not exist");
 			chain
 				.get_block_marker(&head.prev_block_h)
 				.expect("prev block marker does not exist");
 
-			println!("Cur_pmmr_md: {:?}", cur_pmmr_md);
 			chain.validate(false).unwrap();
 		}
 	}
@@ -113,7 +112,7 @@ fn data_files() {
 
 fn _prepare_block(kc: &ExtKeychain, prev: &BlockHeader, chain: &Chain, diff: u64) -> Block {
 	let mut b = _prepare_block_nosum(kc, prev, diff, vec![]);
-	chain.set_txhashset_roots(&mut b, false).unwrap();
+	chain.set_block_roots(&mut b, false).unwrap();
 	b
 }
 
@@ -125,13 +124,13 @@ fn _prepare_block_tx(
 	txs: Vec<&Transaction>,
 ) -> Block {
 	let mut b = _prepare_block_nosum(kc, prev, diff, txs);
-	chain.set_txhashset_roots(&mut b, false).unwrap();
+	chain.set_block_roots(&mut b, false).unwrap();
 	b
 }
 
 fn _prepare_fork_block(kc: &ExtKeychain, prev: &BlockHeader, chain: &Chain, diff: u64) -> Block {
 	let mut b = _prepare_block_nosum(kc, prev, diff, vec![]);
-	chain.set_txhashset_roots(&mut b, true).unwrap();
+	chain.set_block_roots(&mut b, true).unwrap();
 	b
 }
 
@@ -143,7 +142,7 @@ fn _prepare_fork_block_tx(
 	txs: Vec<&Transaction>,
 ) -> Block {
 	let mut b = _prepare_block_nosum(kc, prev, diff, txs);
-	chain.set_txhashset_roots(&mut b, true).unwrap();
+	chain.set_block_roots(&mut b, true).unwrap();
 	b
 }
 

chain/tests/mine_simple_chain.rs

@@ -24,8 +24,8 @@ extern crate time;
 use std::fs;
 use std::sync::Arc;
 
-use chain::types::NoopAdapter;
 use chain::Chain;
+use chain::types::NoopAdapter;
 use core::core::hash::Hashed;
 use core::core::target::Difficulty;
 use core::core::{Block, BlockHeader, OutputFeatures, OutputIdentifier, Transaction};
@@ -67,7 +67,7 @@ fn mine_empty_chain() {
 		let mut b = core::core::Block::new(&prev, vec![], difficulty.clone(), reward).unwrap();
 		b.header.timestamp = prev.timestamp + time::Duration::seconds(60);
 
-		chain.set_txhashset_roots(&mut b, false).unwrap();
+		chain.set_block_roots(&mut b, false).unwrap();
 
 		pow::pow_size(
 			&mut b.header,
@@ -363,7 +363,7 @@ where
 	K: Keychain,
 {
 	let mut b = prepare_block_nosum(kc, prev, diff, vec![]);
-	chain.set_txhashset_roots(&mut b, false).unwrap();
+	chain.set_block_roots(&mut b, false).unwrap();
 	b
 }
 
@@ -378,7 +378,7 @@ where
 	K: Keychain,
 {
 	let mut b = prepare_block_nosum(kc, prev, diff, txs);
-	chain.set_txhashset_roots(&mut b, false).unwrap();
+	chain.set_block_roots(&mut b, false).unwrap();
 	b
 }
 
@@ -387,7 +387,7 @@ where
 	K: Keychain,
 {
 	let mut b = prepare_block_nosum(kc, prev, diff, vec![]);
-	chain.set_txhashset_roots(&mut b, true).unwrap();
+	chain.set_block_roots(&mut b, true).unwrap();
 	b
 }
 
@@ -402,7 +402,7 @@ where
 	K: Keychain,
 {
 	let mut b = prepare_block_nosum(kc, prev, diff, txs);
-	chain.set_txhashset_roots(&mut b, true).unwrap();
+	chain.set_block_roots(&mut b, true).unwrap();
 	b
 }
 

chain/tests/test_coinbase_maturity.rs

@@ -64,7 +64,7 @@ fn test_coinbase_maturity() {
 
 	let difficulty = consensus::next_difficulty(chain.difficulty_iter()).unwrap();
 
-	chain.set_txhashset_roots(&mut block, false).unwrap();
+	chain.set_block_roots(&mut block, false).unwrap();
 
 	pow::pow_size(
 		&mut block.header,
@@ -118,7 +118,7 @@ fn test_coinbase_maturity() {
 
 	let difficulty = consensus::next_difficulty(chain.difficulty_iter()).unwrap();
 
-	chain.set_txhashset_roots(&mut block, false).unwrap();
+	chain.set_block_roots(&mut block, false).unwrap();
 
 	// Confirm the tx attempting to spend the coinbase output
 	// is not valid at the current block height given the current chain state.
@@ -148,7 +148,7 @@ fn test_coinbase_maturity() {
 
 		let difficulty = consensus::next_difficulty(chain.difficulty_iter()).unwrap();
 
-		chain.set_txhashset_roots(&mut block, false).unwrap();
+		chain.set_block_roots(&mut block, false).unwrap();
 
 		pow::pow_size(
 			&mut block.header,
@@ -175,7 +175,7 @@ fn test_coinbase_maturity() {
 
 	let difficulty = consensus::next_difficulty(chain.difficulty_iter()).unwrap();
 
-	chain.set_txhashset_roots(&mut block, false).unwrap();
+	chain.set_block_roots(&mut block, false).unwrap();
 
 	pow::pow_size(
 		&mut block.header,

chain/tests/test_txhashset_raw_txs.rs

@@ -20,10 +20,10 @@ extern crate grin_wallet as wallet;
 use std::fs;
 use std::sync::Arc;
 
+use chain::ChainStore;
 use chain::store::ChainKVStore;
 use chain::txhashset::{self, TxHashSet};
 use chain::types::Tip;
-use chain::ChainStore;
 use core::core::merkle_proof::MerkleProof;
 use core::core::target::Difficulty;
 use core::core::{Block, BlockHeader};

core/src/core/block.rs

@@ -29,7 +29,7 @@ use core::{transaction, Commitment, Input, KernelFeatures, Output, OutputFeature
 use global;
 use keychain::{self, BlindingFactor};
 use ser::{self, read_and_verify_sorted, Readable, Reader, Writeable, WriteableSorted, Writer};
-use util::{secp, static_secp_instance, LOGGER};
+use util::{secp, secp_static, static_secp_instance, LOGGER};
 
 /// Errors thrown by Block validation
 #[derive(Debug, Clone, PartialEq)]
@@ -37,6 +37,8 @@ pub enum Error {
 	/// The sum of output minus input commitments does not
 	/// match the sum of kernel commitments
 	KernelSumMismatch,
+	/// The total kernel sum on the block header is wrong
+	InvalidTotalKernelSum,
 	/// Same as above but for the coinbase part of a block, including reward
 	CoinbaseSumMismatch,
 	/// Too many inputs, outputs or kernels in the block
@@ -116,6 +118,13 @@ pub struct BlockHeader {
 	/// We can derive the kernel offset sum for *this* block from
 	/// the total kernel offset of the previous block header.
 	pub total_kernel_offset: BlindingFactor,
+	/// Total accumulated sum of kernel commitments since genesis block.
+	/// Should always equal the UTXO commitment sum minus supply.
+	pub total_kernel_sum: Commitment,
+	/// Total size of the output MMR after applying this block
+	pub output_mmr_size: u64,
+	/// Total size of the kernel MMR after applying this block
+	pub kernel_mmr_size: u64,
 	/// Nonce increment used to mine this block.
 	pub nonce: u64,
 	/// Proof of work data.
@@ -135,6 +144,9 @@ impl Default for BlockHeader {
 			range_proof_root: ZERO_HASH,
 			kernel_root: ZERO_HASH,
 			total_kernel_offset: BlindingFactor::zero(),
+			total_kernel_sum: Commitment::from_vec(vec![0; 33]),
+			output_mmr_size: 0,
+			kernel_mmr_size: 0,
 			nonce: 0,
 			pow: Proof::zero(proof_size),
 		}
@@ -161,10 +173,12 @@ impl Readable for BlockHeader {
 		let timestamp = reader.read_i64()?;
 		let total_difficulty = Difficulty::read(reader)?;
 		let output_root = Hash::read(reader)?;
-		let rproof_root = Hash::read(reader)?;
+		let range_proof_root = Hash::read(reader)?;
 		let kernel_root = Hash::read(reader)?;
 		let total_kernel_offset = BlindingFactor::read(reader)?;
-		let nonce = reader.read_u64()?;
+		let total_kernel_sum = Commitment::read(reader)?;
+		let (output_mmr_size, kernel_mmr_size, nonce) =
+			ser_multiread!(reader, read_u64, read_u64, read_u64);
 		let pow = Proof::read(reader)?;
 
 		if timestamp > (1 << 55) || timestamp < -(1 << 55) {
@@ -172,20 +186,23 @@ impl Readable for BlockHeader {
 		}
 
 		Ok(BlockHeader {
-			version: version,
+			version,
-			height: height,
+			height,
-			previous: previous,
+			previous,
 			timestamp: time::at_utc(time::Timespec {
 				sec: timestamp,
 				nsec: 0,
 			}),
-			total_difficulty: total_difficulty,
+			total_difficulty,
-			output_root: output_root,
+			output_root,
-			range_proof_root: rproof_root,
+			range_proof_root,
-			kernel_root: kernel_root,
+			kernel_root,
-			total_kernel_offset: total_kernel_offset,
+			total_kernel_offset,
-			nonce: nonce,
+			total_kernel_sum,
-			pow: pow,
+			output_mmr_size,
+			kernel_mmr_size,
+			nonce,
+			pow,
 		})
 	}
 }
@@ -204,6 +221,9 @@ impl BlockHeader {
 			[write_fixed_bytes, &self.range_proof_root],
 			[write_fixed_bytes, &self.kernel_root],
 			[write_fixed_bytes, &self.total_kernel_offset],
+			[write_fixed_bytes, &self.total_kernel_sum],
+			[write_u64, self.output_mmr_size],
+			[write_u64, self.kernel_mmr_size],
 			[write_u64, self.nonce]
 		);
 		Ok(())
@@ -567,24 +587,17 @@ impl Block {
 
 		// now sum the kernel_offsets up to give us
 		// an aggregate offset for the entire block
-		let total_kernel_offset = {
+		kernel_offsets.push(prev.total_kernel_offset);
+		let total_kernel_offset = committed::sum_kernel_offsets(kernel_offsets, vec![])?;
+
+		let total_kernel_sum = {
+			let zero_commit = secp_static::commit_to_zero_value();
 			let secp = static_secp_instance();
 			let secp = secp.lock().unwrap();
-			let mut keys = kernel_offsets
+			let mut excesses = map_vec!(kernels, |x| x.excess());
-				.into_iter()
+			excesses.push(prev.total_kernel_sum);
-				.filter(|x| *x != BlindingFactor::zero())
+			excesses.retain(|x| *x != zero_commit);
-				.filter_map(|x| x.secret_key(&secp).ok())
+			secp.commit_sum(excesses, vec![])?
-				.collect::<Vec<_>>();
-			if prev.total_kernel_offset != BlindingFactor::zero() {
-				keys.push(prev.total_kernel_offset.secret_key(&secp)?);
-			}
-
-			if keys.is_empty() {
-				BlindingFactor::zero()
-			} else {
-				let sum = secp.blind_sum(keys, vec![])?;
-				BlindingFactor::from_secret_key(sum)
-			}
 		};
 
 		Ok(Block {
@@ -596,12 +609,13 @@ impl Block {
 				},
 				previous: prev.hash(),
 				total_difficulty: difficulty + prev.total_difficulty,
-				total_kernel_offset: total_kernel_offset,
+				total_kernel_offset,
+				total_kernel_sum,
 				..Default::default()
 			},
-			inputs: inputs,
+			inputs,
-			outputs: outputs,
+			outputs,
-			kernels: kernels,
+			kernels,
 		}.cut_through())
 	}
 
@@ -666,25 +680,39 @@ impl Block {
 	/// trees, reward, etc.
 	pub fn validate(
 		&self,
-		prev_output_sum: &Commitment,
+		prev_kernel_offset: &BlindingFactor,
 		prev_kernel_sum: &Commitment,
-	) -> Result<((Commitment, Commitment)), Error> {
+	) -> Result<(Commitment), Error> {
 		self.verify_weight()?;
 		self.verify_sorted()?;
 		self.verify_cut_through()?;
 		self.verify_coinbase()?;
 		self.verify_kernel_lock_heights()?;
 
-		let sums = self.verify_kernel_sums(
+		// take the kernel offset for this block (block offset minus previous) and
-			self.header.overage(),
+		// verify outputs and kernel sums
-			self.header.total_kernel_offset(),
+		let block_kernel_offset = if self.header.total_kernel_offset() == prev_kernel_offset.clone()
-			Some(prev_output_sum),
+		{
-			Some(prev_kernel_sum),
+			// special case when the sum hasn't changed (typically an empty block),
-		)?;
+			// zero isn't a valid private key but it's a valid blinding factor
+			BlindingFactor::zero()
+		} else {
+			committed::sum_kernel_offsets(
+				vec![self.header.total_kernel_offset()],
+				vec![prev_kernel_offset.clone()],
+			)?
+		};
+		let sum = self.verify_kernel_sums(self.header.overage(), block_kernel_offset)?;
+
+		// check the block header's total kernel sum
+		let total_sum = committed::sum_commits(vec![sum, prev_kernel_sum.clone()], vec![])?;
+		if total_sum != self.header.total_kernel_sum {
+			return Err(Error::InvalidTotalKernelSum);
+		}
 
 		self.verify_rangeproofs()?;
 		self.verify_kernel_signatures()?;
-		Ok(sums)
+		Ok(sum)
 	}
 
 	fn verify_weight(&self) -> Result<(), Error> {

core/src/core/committed.rs

@@ -17,6 +17,7 @@
 use keychain;
 use keychain::BlindingFactor;
 
+use util::secp::key::SecretKey;
 use util::secp::pedersen::Commitment;
 use util::{secp, secp_static, static_secp_instance};
 
@@ -52,26 +53,12 @@ pub trait Committed {
 	fn sum_kernel_excesses(
 		&self,
 		offset: &BlindingFactor,
-		extra_excess: Option<&Commitment>,
 	) -> Result<(Commitment, Commitment), Error> {
-		let zero_commit = secp_static::commit_to_zero_value();
-
 		// then gather the kernel excess commitments
-		let mut kernel_commits = self.kernels_committed();
+		let kernel_commits = self.kernels_committed();
-
-		if let Some(extra) = extra_excess {
-			kernel_commits.push(*extra);
-		}
-
-		// handle "zero commit" values by filtering them out here
-		kernel_commits.retain(|x| *x != zero_commit);
 
 		// sum the commitments
-		let kernel_sum = {
+		let kernel_sum = sum_commits(kernel_commits, vec![])?;
-			let secp = static_secp_instance();
-			let secp = secp.lock().unwrap();
-			secp.commit_sum(kernel_commits, vec![])?
-		};
 
 		// sum the commitments along with the
 		// commit to zero built from the offset
@@ -91,14 +78,8 @@ pub trait Committed {
 	}
 
 	/// Gathers commitments and sum them.
-	fn sum_commitments(
+	fn sum_commitments(&self, overage: i64) -> Result<Commitment, Error> {
-		&self,
+		// gather the commitments
-		overage: i64,
-		extra_commit: Option<&Commitment>,
-	) -> Result<Commitment, Error> {
-		let zero_commit = secp_static::commit_to_zero_value();
-
-		// then gather the commitments
 		let mut input_commits = self.inputs_committed();
 		let mut output_commits = self.outputs_committed();
 
@@ -117,21 +98,7 @@ pub trait Committed {
 			}
 		}
 
-		if let Some(extra) = extra_commit {
+		sum_commits(output_commits, input_commits)
-			output_commits.push(*extra);
-		}
-
-		// handle "zero commit" values by filtering them out here
-		output_commits.retain(|x| *x != zero_commit);
-		input_commits.retain(|x| *x != zero_commit);
-
-		// sum all that stuff
-		{
-			let secp = static_secp_instance();
-			let secp = secp.lock().unwrap();
-			let res = secp.commit_sum(output_commits, input_commits)?;
-			Ok(res)
-		}
 	}
 
 	/// Vector of input commitments to verify.
@@ -150,20 +117,57 @@ pub trait Committed {
 		&self,
 		overage: i64,
 		kernel_offset: BlindingFactor,
-		prev_output_sum: Option<&Commitment>,
+	) -> Result<(Commitment), Error> {
-		prev_kernel_sum: Option<&Commitment>,
-	) -> Result<((Commitment, Commitment)), Error> {
 		// Sum all input|output|overage commitments.
-		let utxo_sum = self.sum_commitments(overage, prev_output_sum)?;
+		let utxo_sum = self.sum_commitments(overage)?;
 
 		// Sum the kernel excesses accounting for the kernel offset.
-		let (kernel_sum, kernel_sum_plus_offset) =
+		let (kernel_sum, kernel_sum_plus_offset) = self.sum_kernel_excesses(&kernel_offset)?;
-			self.sum_kernel_excesses(&kernel_offset, prev_kernel_sum)?;
 
 		if utxo_sum != kernel_sum_plus_offset {
 			return Err(Error::KernelSumMismatch);
 		}
 
-		Ok((utxo_sum, kernel_sum))
+		Ok(kernel_sum)
 	}
 }
+
+/// Utility to sum positive and negative commitments, eliminating zero values
+pub fn sum_commits(
+	mut positive: Vec<Commitment>,
+	mut negative: Vec<Commitment>,
+) -> Result<Commitment, Error> {
+	let zero_commit = secp_static::commit_to_zero_value();
+	positive.retain(|x| *x != zero_commit);
+	negative.retain(|x| *x != zero_commit);
+	let secp = static_secp_instance();
+	let secp = secp.lock().unwrap();
+	Ok(secp.commit_sum(positive, negative)?)
+}
+
+/// Utility function to take sets of positive and negative kernel offsets as
+/// blinding factors, convert them to private key filtering zero values and
+/// summing all of them. Useful to build blocks.
+pub fn sum_kernel_offsets(
+	positive: Vec<BlindingFactor>,
+	negative: Vec<BlindingFactor>,
+) -> Result<BlindingFactor, Error> {
+	let secp = static_secp_instance();
+	let secp = secp.lock().unwrap();
+	let positive = to_secrets(positive, &secp);
+	let negative = to_secrets(negative, &secp);
+
+	if positive.is_empty() {
+		Ok(BlindingFactor::zero())
+	} else {
+		let sum = secp.blind_sum(positive, negative)?;
+		Ok(BlindingFactor::from_secret_key(sum))
+	}
+}
+
+fn to_secrets(bf: Vec<BlindingFactor>, secp: &secp::Secp256k1) -> Vec<SecretKey> {
+	bf.into_iter()
+		.filter(|x| *x != BlindingFactor::zero())
+		.filter_map(|x| x.secret_key(&secp).ok())
+		.collect::<Vec<_>>()
+}

core/src/core/id.rs

@@ -14,8 +14,8 @@
 
 //! short ids for compact blocks
 
-use std::cmp::min;
 use std::cmp::Ordering;
+use std::cmp::min;
 
 use byteorder::{ByteOrder, LittleEndian};
 use siphasher::sip::SipHasher24;

core/src/core/pmmr.rs

@@ -39,9 +39,9 @@ use std::marker;
 
 use croaring::Bitmap;
 
+use core::BlockHeader;
 use core::hash::Hash;
 use core::merkle_proof::MerkleProof;
-use core::BlockHeader;
 use ser::{PMMRIndexHashable, PMMRable};
 use util::LOGGER;
 

core/src/core/transaction.rs

@@ -14,8 +14,8 @@
 
 //! Transactions
 
-use std::cmp::max;
 use std::cmp::Ordering;
+use std::cmp::max;
 use std::collections::HashSet;
 use std::io::Cursor;
 use std::{error, fmt};
@@ -429,7 +429,7 @@ impl Transaction {
 		}
 		self.verify_sorted()?;
 		self.verify_cut_through()?;
-		self.verify_kernel_sums(self.overage(), self.offset, None, None)?;
+		self.verify_kernel_sums(self.overage(), self.offset)?;
 		self.verify_rangeproofs()?;
 		self.verify_kernel_signatures()?;
 
@@ -556,7 +556,7 @@ pub fn aggregate(transactions: Vec<Transaction>) -> Result<Transaction, Error> {
 
 	// We need to check sums here as aggregation/cut-through
 	// may have created an invalid tx.
-	tx.verify_kernel_sums(tx.overage(), tx.offset, None, None)?;
+	tx.verify_kernel_sums(tx.overage(), tx.offset)?;
 
 	Ok(tx)
 }

core/src/ser.rs

@@ -25,10 +25,10 @@ use core::hash::{Hash, Hashed};
 use keychain::{BlindingFactor, Identifier, IDENTIFIER_SIZE};
 use std::io::{self, Read, Write};
 use std::{cmp, error, fmt, mem};
+use util::secp::Signature;
 use util::secp::constants::{AGG_SIGNATURE_SIZE, MAX_PROOF_SIZE, PEDERSEN_COMMITMENT_SIZE,
                             SECRET_KEY_SIZE};
 use util::secp::pedersen::{Commitment, RangeProof};
-use util::secp::Signature;
 
 /// Possible errors deriving from serializing or deserializing.
 #[derive(Debug)]

core/tests/block.rs

@@ -21,13 +21,13 @@ pub mod common;
 
 use common::{new_block, tx1i2o, tx2i1o, txspend1i1o};
 use grin_core::consensus::{BLOCK_OUTPUT_WEIGHT, MAX_BLOCK_WEIGHT};
+use grin_core::core::Committed;
 use grin_core::core::block::Error;
 use grin_core::core::hash::Hashed;
 use grin_core::core::id::{ShortId, ShortIdentifiable};
-use grin_core::core::Committed;
 use grin_core::core::{Block, BlockHeader, CompactBlock, KernelFeatures, OutputFeatures};
 use grin_core::{global, ser};
-use keychain::{ExtKeychain, Keychain};
+use keychain::{BlindingFactor, ExtKeychain, Keychain};
 use std::time::Instant;
 use util::{secp, secp_static};
 use wallet::libtx::build::{self, input, output, with_fee};
@@ -59,7 +59,7 @@ fn too_large_block() {
 	let prev = BlockHeader::default();
 	let key_id = keychain.derive_key_id(1).unwrap();
 	let b = new_block(vec![&tx], &keychain, &prev, &key_id);
-	assert!(b.validate(&zero_commit, &zero_commit).is_err());
+	assert!(b.validate(&BlindingFactor::zero(), &zero_commit).is_err());
 }
 
 #[test]
@@ -109,9 +109,11 @@ fn block_with_cut_through() {
 
 	// block should have been automatically compacted (including reward
 	// output) and should still be valid
-	b.validate(&zero_commit, &zero_commit).unwrap();
+	println!("3");
+	b.validate(&BlindingFactor::zero(), &zero_commit).unwrap();
 	assert_eq!(b.inputs.len(), 3);
 	assert_eq!(b.outputs.len(), 3);
+	println!("4");
 }
 
 #[test]
@@ -142,7 +144,7 @@ fn empty_block_with_coinbase_is_valid() {
 
 	// the block should be valid here (single coinbase output with corresponding
 	// txn kernel)
-	assert!(b.validate(&zero_commit, &zero_commit).is_ok());
+	assert!(b.validate(&BlindingFactor::zero(), &zero_commit).is_ok());
 }
 
 #[test]
@@ -166,14 +168,12 @@ fn remove_coinbase_output_flag() {
 		.remove(OutputFeatures::COINBASE_OUTPUT);
 
 	assert_eq!(b.verify_coinbase(), Err(Error::CoinbaseSumMismatch));
-	assert!(b.verify_kernel_sums(
+	assert!(
-		b.header.overage(),
+		b.verify_kernel_sums(b.header.overage(), b.header.total_kernel_offset())
-		b.header.total_kernel_offset(),
+			.is_ok()
-		None,
+	);
-		None
-	).is_ok());
 	assert_eq!(
-		b.validate(&zero_commit, &zero_commit),
+		b.validate(&BlindingFactor::zero(), &zero_commit),
 		Err(Error::CoinbaseSumMismatch)
 	);
 }
@@ -203,7 +203,7 @@ fn remove_coinbase_kernel_flag() {
 	);
 
 	assert_eq!(
-		b.validate(&zero_commit, &zero_commit),
+		b.validate(&BlindingFactor::zero(), &zero_commit),
 		Err(Error::Secp(secp::Error::IncorrectCommitSum))
 	);
 }
@@ -233,7 +233,7 @@ fn empty_block_serialized_size() {
 	let b = new_block(vec![], &keychain, &prev, &key_id);
 	let mut vec = Vec::new();
 	ser::serialize(&mut vec, &b).expect("serialization failed");
-	let target_len = 1_216;
+	let target_len = 1_265;
 	assert_eq!(vec.len(), target_len,);
 }
 
@@ -246,7 +246,7 @@ fn block_single_tx_serialized_size() {
 	let b = new_block(vec![&tx1], &keychain, &prev, &key_id);
 	let mut vec = Vec::new();
 	ser::serialize(&mut vec, &b).expect("serialization failed");
-	let target_len = 2_796;
+	let target_len = 2_845;
 	assert_eq!(vec.len(), target_len);
 }
 
@@ -258,7 +258,7 @@ fn empty_compact_block_serialized_size() {
 	let b = new_block(vec![], &keychain, &prev, &key_id);
 	let mut vec = Vec::new();
 	ser::serialize(&mut vec, &b.as_compact_block()).expect("serialization failed");
-	let target_len = 1_224;
+	let target_len = 1_273;
 	assert_eq!(vec.len(), target_len,);
 }
 
@@ -271,7 +271,7 @@ fn compact_block_single_tx_serialized_size() {
 	let b = new_block(vec![&tx1], &keychain, &prev, &key_id);
 	let mut vec = Vec::new();
 	ser::serialize(&mut vec, &b.as_compact_block()).expect("serialization failed");
-	let target_len = 1_230;
+	let target_len = 1_279;
 	assert_eq!(vec.len(), target_len,);
 }
 
@@ -290,7 +290,7 @@ fn block_10_tx_serialized_size() {
 	let b = new_block(txs.iter().collect(), &keychain, &prev, &key_id);
 	let mut vec = Vec::new();
 	ser::serialize(&mut vec, &b).expect("serialization failed");
-	let target_len = 17_016;
+	let target_len = 17_065;
 	assert_eq!(vec.len(), target_len,);
 }
 
@@ -308,7 +308,7 @@ fn compact_block_10_tx_serialized_size() {
 	let b = new_block(txs.iter().collect(), &keychain, &prev, &key_id);
 	let mut vec = Vec::new();
 	ser::serialize(&mut vec, &b.as_compact_block()).expect("serialization failed");
-	let target_len = 1_284;
+	let target_len = 1_333;
 	assert_eq!(vec.len(), target_len,);
 }
 

core/tests/common/mod.rs

@@ -19,9 +19,9 @@ extern crate grin_keychain as keychain;
 extern crate grin_util as util;
 extern crate grin_wallet as wallet;
 
+use grin_core::core::Transaction;
 use grin_core::core::block::{Block, BlockHeader};
 use grin_core::core::target::Difficulty;
-use grin_core::core::Transaction;
 use keychain::{Identifier, Keychain};
 use wallet::libtx::build::{self, input, output, with_fee};
 use wallet::libtx::reward;

core/tests/consensus.rs

@@ -50,10 +50,9 @@ fn create_chain_sim(diff: u64) -> Vec<Result<(u64, Difficulty), TargetError>> {
 		time::get_time().sec,
 		Difficulty::from_num(diff)
 	);
-	vec![Ok((
+	vec![
-		time::get_time().sec as u64,
+		Ok((time::get_time().sec as u64, Difficulty::from_num(diff))),
-		Difficulty::from_num(diff),
+	]
-	))]
 }
 
 // Adds another 'block' to the iterator, so to speak, with difficulty calculated

core/tests/core.rs

@@ -26,7 +26,7 @@ use grin_core::core::block::Error::KernelLockHeight;
 use grin_core::core::hash::{Hashed, ZERO_HASH};
 use grin_core::core::{aggregate, deaggregate, KernelFeatures, Output, Transaction};
 use grin_core::ser;
-use keychain::{ExtKeychain, Keychain};
+use keychain::{BlindingFactor, ExtKeychain, Keychain};
 use util::{secp_static, static_secp_instance};
 use wallet::libtx::build::{self, initial_tx, input, output, with_excess, with_fee,
                            with_lock_height};
@@ -396,7 +396,7 @@ fn reward_empty_block() {
 	let b = new_block(vec![], &keychain, &previous_header, &key_id);
 
 	b.cut_through()
-		.validate(&zero_commit, &zero_commit)
+		.validate(&BlindingFactor::zero(), &zero_commit)
 		.unwrap();
 }
 
@@ -415,7 +415,7 @@ fn reward_with_tx_block() {
 	let block = new_block(vec![&mut tx1], &keychain, &previous_header, &key_id);
 	block
 		.cut_through()
-		.validate(&zero_commit, &zero_commit)
+		.validate(&BlindingFactor::zero(), &zero_commit)
 		.unwrap();
 }
 
@@ -437,7 +437,7 @@ fn simple_block() {
 		&key_id,
 	);
 
-	b.validate(&zero_commit, &zero_commit).unwrap();
+	b.validate(&BlindingFactor::zero(), &zero_commit).unwrap();
 }
 
 #[test]
@@ -465,7 +465,7 @@ fn test_block_with_timelocked_tx() {
 	let previous_header = BlockHeader::default();
 
 	let b = new_block(vec![&tx1], &keychain, &previous_header, &key_id3.clone());
-	b.validate(&zero_commit, &zero_commit).unwrap();
+	b.validate(&BlindingFactor::zero(), &zero_commit).unwrap();
 
 	// now try adding a timelocked tx where lock height is greater than current
 	// block height
@@ -482,7 +482,7 @@ fn test_block_with_timelocked_tx() {
 	let previous_header = BlockHeader::default();
 	let b = new_block(vec![&tx1], &keychain, &previous_header, &key_id3.clone());
 
-	match b.validate(&zero_commit, &zero_commit) {
+	match b.validate(&BlindingFactor::zero(), &zero_commit) {
 		Err(KernelLockHeight(height)) => {
 			assert_eq!(height, 2);
 		}

core/tests/vec_backend/mod.rs

@@ -16,9 +16,9 @@ extern crate croaring;
 
 use croaring::Bitmap;
 
+use core::core::BlockHeader;
 use core::core::hash::Hash;
 use core::core::pmmr::Backend;
-use core::core::BlockHeader;
 use core::ser;
 use core::ser::{PMMRable, Readable, Reader, Writeable, Writer};
 

keychain/src/extkey.rs

@@ -15,8 +15,8 @@
 use blake2::blake2b::blake2b;
 use byteorder::{BigEndian, ByteOrder};
 use types::{Error, Identifier};
-use util::secp::key::SecretKey;
 use util::secp::Secp256k1;
+use util::secp::key::SecretKey;
 
 #[derive(Debug, Clone)]
 pub struct ChildKey {
@@ -119,8 +119,8 @@ mod test {
 
 	use super::{ExtendedKey, Identifier};
 	use util;
-	use util::secp::key::SecretKey;
 	use util::secp::Secp256k1;
+	use util::secp::key::SecretKey;
 
 	fn from_hex(hex_str: &str) -> Vec<u8> {
 		util::from_hex(hex_str.to_string()).unwrap()

keychain/src/types.rs

@@ -297,8 +297,8 @@ mod test {
 	use rand::thread_rng;
 
 	use types::BlindingFactor;
-	use util::secp::key::{SecretKey, ZERO_KEY};
 	use util::secp::Secp256k1;
+	use util::secp::key::{SecretKey, ZERO_KEY};
 
 	#[test]
 	fn split_blinding_factor() {

p2p/src/handshake.rs

@@ -16,8 +16,8 @@ use std::collections::VecDeque;
 use std::net::{SocketAddr, TcpStream};
 use std::sync::{Arc, RwLock};
 
-use rand::os::OsRng;
 use rand::Rng;
+use rand::os::OsRng;
 
 use core::core::hash::Hash;
 use core::core::target::Difficulty;

p2p/src/msg.rs

@@ -20,9 +20,9 @@ use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, TcpSt
 use std::{thread, time};
 
 use core::consensus::{MAX_MSG_LEN, MAX_TX_INPUTS, MAX_TX_KERNELS, MAX_TX_OUTPUTS};
+use core::core::BlockHeader;
 use core::core::hash::Hash;
 use core::core::target::Difficulty;
-use core::core::BlockHeader;
 use core::ser::{self, Readable, Reader, Writeable, Writer};
 
 use types::{Capabilities, Error, ReasonForBan, MAX_BLOCK_HEADERS, MAX_LOCATORS, MAX_PEER_ADDRS};

p2p/tests/peer_handshake.rs

@@ -17,8 +17,8 @@ extern crate grin_p2p as p2p;
 extern crate grin_util as util;
 
 use std::net::{SocketAddr, TcpListener, TcpStream};
-use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 use std::{thread, time};
 
 use core::core::hash::Hash;

pool/tests/coinbase_maturity.rs

@@ -30,8 +30,8 @@ use std::sync::{Arc, RwLock};
 use common::{test_source, test_transaction};
 use core::core::Transaction;
 use keychain::{ExtKeychain, Keychain};
-use pool::types::{BlockChain, NoopAdapter, PoolConfig, PoolError};
 use pool::TransactionPool;
+use pool::types::{BlockChain, NoopAdapter, PoolConfig, PoolError};
 
 pub fn test_setup(
 	chain: &Arc<CoinbaseMaturityErrorChainAdapter>,

pool/tests/common/mod.rs

@@ -40,8 +40,8 @@ use pool::*;
 use keychain::Keychain;
 use wallet::libtx;
 
-use pool::types::*;
 use pool::TransactionPool;
+use pool::types::*;
 
 #[derive(Clone)]
 pub struct ChainAdapter {

servers/src/common/adapters.rs

@@ -140,8 +140,11 @@ impl p2p::ChainAdapter for NetToChainAdapter {
 				.upgrade()
 				.expect("failed to upgrade weak ref to chain");
 
-			if let Ok(sums) = chain.get_block_sums(&cb.header.previous) {
+			if let Ok(prev) = chain.get_block_header(&cb.header.previous) {
-				if block.validate(&sums.output_sum, &sums.kernel_sum).is_ok() {
+				if block
+					.validate(&prev.total_kernel_offset, &prev.total_kernel_sum)
+					.is_ok()
+				{
 					debug!(LOGGER, "adapter: successfully hydrated block from tx pool!");
 					self.process_block(block, addr)
 				} else {

servers/src/grin/sync.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Duration;
 use std::{cmp, thread};
 use time;
@@ -457,7 +457,7 @@ mod test {
 		assert_eq!(
 			get_locator_heights(10000),
 			vec![
-				10000, 9998, 9994, 9986, 9970, 9938, 9874, 9746, 9490, 8978, 7954, 5906, 1810, 0,
+				10000, 9998, 9994, 9986, 9970, 9938, 9874, 9746, 9490, 8978, 7954, 5906, 1810, 0
 			]
 		);
 	}

servers/src/mining/mine_block.rs

@@ -22,10 +22,9 @@ use std::thread;
 use std::time::Duration;
 use time;
 
-use chain::{self, types::BlockSums};
+use chain;
 use common::adapters::PoolToChainAdapter;
 use common::types::Error;
-use core::core::hash::Hashed;
 use core::ser::{self, AsFixedBytes};
 use core::{consensus, core};
 use keychain::{ExtKeychain, Identifier, Keychain};
@@ -135,12 +134,6 @@ fn build_block(
 	// prepare the block header timestamp
 	let head = chain.head_header()?;
 
-	let prev_sums = if head.height == 0 {
-		BlockSums::default()
-	} else {
-		chain.get_block_sums(&head.hash())?
-	};
-
 	let mut now_sec = time::get_time().sec;
 	let head_sec = head.timestamp.to_timespec().sec;
 	if now_sec <= head_sec {
@@ -170,7 +163,7 @@ fn build_block(
 	let mut b = core::Block::with_reward(&head, txs, output, kernel, difficulty.clone())?;
 
 	// making sure we're not spending time mining a useless block
-	b.validate(&prev_sums.output_sum, &prev_sums.kernel_sum)?;
+	b.validate(&head.total_kernel_offset, &head.total_kernel_sum)?;
 
 	let mut rng = rand::OsRng::new().unwrap();
 	b.header.nonce = rng.gen();
@@ -186,7 +179,7 @@ fn build_block(
 		b.header.clone().total_difficulty.to_num(),
 	);
 
-	let roots_result = chain.set_txhashset_roots(&mut b, false);
+	let roots_result = chain.set_block_roots(&mut b, false);
 
 	match roots_result {
 		Ok(_) => Ok((b, block_fees)),

src/bin/tui/menu.rs

@@ -14,13 +14,13 @@
 
 //! Main Menu definition
 
+use cursive::Cursive;
 use cursive::align::HAlign;
 use cursive::direction::Orientation;
 use cursive::event::{EventResult, Key};
 use cursive::view::Identifiable;
 use cursive::view::View;
 use cursive::views::{BoxView, LinearLayout, OnEventView, SelectView, StackView, TextView, ViewRef};
-use cursive::Cursive;
 
 use tui::constants::{MAIN_MENU, ROOT_STACK, SUBMENU_MINING_BUTTON, VIEW_BASIC_STATUS, VIEW_MINING,
                      VIEW_PEER_SYNC, VIEW_VERSION};

src/bin/tui/mining.rs

@@ -16,13 +16,13 @@
 
 use std::cmp::Ordering;
 
+use cursive::Cursive;
 use cursive::direction::Orientation;
 use cursive::event::Key;
 use cursive::traits::{Boxable, Identifiable};
 use cursive::view::View;
 use cursive::views::{BoxView, Button, Dialog, LinearLayout, OnEventView, Panel, StackView,
                      TextView};
-use cursive::Cursive;
 use std::time;
 use tui::chrono::prelude::{DateTime, NaiveDateTime, Utc};
 

src/bin/tui/peers.rs

@@ -18,11 +18,11 @@ use std::cmp::Ordering;
 
 use servers::{PeerStats, ServerStats};
 
+use cursive::Cursive;
 use cursive::direction::Orientation;
 use cursive::traits::{Boxable, Identifiable};
 use cursive::view::View;
 use cursive::views::{BoxView, Dialog, LinearLayout, TextView};
-use cursive::Cursive;
 
 use tui::constants::{TABLE_PEER_STATUS, VIEW_PEER_SYNC};
 use tui::table::{TableView, TableViewItem};

src/bin/tui/status.rs

@@ -14,11 +14,11 @@
 
 //! Basic status view definition
 
+use cursive::Cursive;
 use cursive::direction::Orientation;
 use cursive::traits::Identifiable;
 use cursive::view::View;
 use cursive::views::{BoxView, LinearLayout, TextView};
-use cursive::Cursive;
 
 use tui::constants::VIEW_BASIC_STATUS;
 use tui::types::TUIStatusListener;

src/bin/tui/table.rs

@@ -41,10 +41,8 @@
 //! Adapted from https://github.com/behnam/rust-cursive-table-view
 //! A basic table view implementation for [cursive](https://crates.io/crates/cursive).
 
-#![deny(
+#![deny(missing_docs, missing_copy_implementations, trivial_casts, trivial_numeric_casts,
-	missing_docs, missing_copy_implementations, trivial_casts, trivial_numeric_casts, unsafe_code,
+        unsafe_code, unused_import_braces, unused_qualifications)]
-	unused_import_braces, unused_qualifications
-)]
 
 // Crate Dependencies ---------------------------------------------------------
 extern crate cursive;
@@ -56,6 +54,7 @@ use std::hash::Hash;
 use std::rc::Rc;
 
 // External Dependencies ------------------------------------------------------
+use cursive::With;
 use cursive::align::HAlign;
 use cursive::direction::Direction;
 use cursive::event::{Callback, Event, EventResult, Key};
@@ -63,7 +62,6 @@ use cursive::theme::ColorStyle;
 use cursive::theme::PaletteColor::{Highlight, HighlightInactive, Primary};
 use cursive::vec::Vec2;
 use cursive::view::{ScrollBase, View};
-use cursive::With;
 use cursive::{Cursive, Printer};
 
 /// A trait for displaying and sorting items inside a

src/bin/tui/types.rs

@@ -14,8 +14,8 @@
 
 //! Types specific to the UI module
 
-use cursive::view::View;
 use cursive::Cursive;
+use cursive::view::View;
 use servers::ServerStats;
 
 /// Main message struct to communicate between the UI and

src/bin/tui/ui.rs

@@ -18,6 +18,7 @@
 use std::sync::{mpsc, Arc};
 use time;
 
+use cursive::Cursive;
 use cursive::direction::Orientation;
 use cursive::theme::BaseColor::{Black, Blue, Cyan, White};
 use cursive::theme::Color::Dark;
@@ -26,7 +27,6 @@ use cursive::theme::{BaseColor, BorderStyle, Color, Theme};
 use cursive::traits::Identifiable;
 use cursive::utils::markup::StyledString;
 use cursive::views::{LinearLayout, Panel, StackView, TextView, ViewBox};
-use cursive::Cursive;
 
 use servers::Server;
 

src/bin/tui/version.rs

@@ -14,11 +14,11 @@
 
 //! Version and build info
 
+use cursive::Cursive;
 use cursive::direction::Orientation;
 use cursive::traits::Identifiable;
 use cursive::view::View;
 use cursive::views::{BoxView, LinearLayout, TextView};
-use cursive::Cursive;
 
 use tui::constants::VIEW_VERSION;
 use tui::types::TUIStatusListener;

store/src/leaf_set.rs

@@ -21,10 +21,10 @@ use std::path::Path;
 
 use croaring::Bitmap;
 
+use core::core::BlockHeader;
 use core::core::hash::Hashed;
 use core::core::pmmr;
 use core::core::prune_list::PruneList;
-use core::core::BlockHeader;
 
 use util::LOGGER;
 

store/src/pmmr.rs

@@ -20,10 +20,10 @@ use std::path::Path;
 
 use croaring::Bitmap;
 
+use core::core::BlockHeader;
 use core::core::hash::{Hash, Hashed};
 use core::core::pmmr::{self, family, Backend};
 use core::core::prune_list::PruneList;
-use core::core::BlockHeader;
 use core::ser::{self, PMMRable};
 use leaf_set::LeafSet;
 use rm_log::RemoveLog;

store/src/types.rs

@@ -89,7 +89,9 @@ impl AppendOnlyFile {
 	pub fn rewind(&mut self, file_pos: u64) {
 		if self.buffer.is_empty() {
 			// rewinding from clean state, no buffer, not already rewound anything
-			self.buffer_start_bak = self.buffer_start;
+			if self.buffer_start_bak == 0 {
+				self.buffer_start_bak = self.buffer_start;
+			}
 			self.buffer_start = file_pos as usize;
 		} else {
 			// rewinding (within) the buffer

wallet/src/display.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use core::core::{self, amount_to_hr_string};
-use libwallet::types::{OutputData, WalletInfo};
 use libwallet::Error;
+use libwallet::types::{OutputData, WalletInfo};
 use prettytable;
 use std::io::prelude::Write;
 use term;

wallet/src/file_wallet.rs

@@ -22,8 +22,8 @@ use std::path::{Path, MAIN_SEPARATOR};
 
 use serde_json;
 use tokio_core::reactor;
-use tokio_retry::strategy::FibonacciBackoff;
 use tokio_retry::Retry;
+use tokio_retry::strategy::FibonacciBackoff;
 
 use failure::ResultExt;
 

wallet/src/libtx/slate.rs

@@ -23,8 +23,8 @@ use keychain::{BlindSum, BlindingFactor, Keychain};
 use libtx::error::{Error, ErrorKind};
 use libtx::{aggsig, build, tx_fee};
 
-use util::secp::key::{PublicKey, SecretKey};
 use util::secp::Signature;
+use util::secp::key::{PublicKey, SecretKey};
 use util::{secp, LOGGER};
 
 /// Public data for each participant in the slate
@@ -371,7 +371,7 @@ impl Slate {
 
 			// sum the input/output commitments on the final tx
 			let overage = final_tx.fee() as i64;
-			let tx_excess = final_tx.sum_commitments(overage, None)?;
+			let tx_excess = final_tx.sum_commitments(overage)?;
 
 			// subtract the kernel_excess (built from kernel_offset)
 			let offset_excess = keychain

wallet/src/libwallet/api.rs

@@ -22,10 +22,10 @@ use std::marker::PhantomData;
 use core::ser;
 use keychain::Keychain;
 use libtx::slate::Slate;
+use libwallet::Error;
 use libwallet::internal::{tx, updater};
 use libwallet::types::{BlockFees, CbData, OutputData, TxWrapper, WalletBackend, WalletClient,
                        WalletInfo};
-use libwallet::Error;
 use util::{self, LOGGER};
 
 /// Wrapper around internal API functions, containing a reference to

wallet/src/libwallet/error.rs

@@ -48,11 +48,8 @@ pub enum ErrorKind {
 	},
 
 	/// Fee Exceeds amount
-	#[fail(
+	#[fail(display = "Fee exceeds amount: sender amount {}, recipient fee {}", sender_amount,
-		display = "Fee exceeds amount: sender amount {}, recipient fee {}",
+	       recipient_fee)]
-		sender_amount,
-		recipient_fee
-	)]
 	FeeExceedsAmount {
 		/// sender amount
 		sender_amount: u64,

wallet/src/libwallet/internal/selection.rs

@@ -15,7 +15,7 @@
 //! Selection of inputs for building transactions
 
 use keychain::{Identifier, Keychain};
-use libtx::{build, slate::Slate, tx_fee};
+use libtx::{build, tx_fee, slate::Slate};
 use libwallet::error::{Error, ErrorKind};
 use libwallet::internal::{keys, sigcontext};
 use libwallet::types::*;

wallet/src/libwallet/internal/updater.rs

@@ -16,8 +16,8 @@
 //! the wallet storage and update them.
 
 use failure::ResultExt;
-use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use std::collections::hash_map::Entry;
 
 use core::consensus::reward;
 use core::core::{Output, TxKernel};

wallet/tests/common/mod.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 //! Common functions to facilitate wallet, walletlib and transaction testing
-use std::collections::hash_map::Entry;
 use std::collections::HashMap;
+use std::collections::hash_map::Entry;
 
 extern crate grin_api as api;
 extern crate grin_chain as chain;
@@ -145,7 +145,7 @@ pub fn add_block_with_reward(chain: &Chain, txs: Vec<&Transaction>, reward: (Out
 		reward,
 	).unwrap();
 	b.header.timestamp = prev.timestamp + time::Duration::seconds(60);
-	chain.set_txhashset_roots(&mut b, false).unwrap();
+	chain.set_block_roots(&mut b, false).unwrap();
 	pow::pow_size(
 		&mut b.header,
 		difficulty,

wallet/tests/transaction.rs

@@ -29,8 +29,8 @@ mod common;
 use std::fs;
 use std::sync::Arc;
 
-use chain::types::NoopAdapter;
 use chain::Chain;
+use chain::types::NoopAdapter;
 use core::global::ChainTypes;
 use core::{global, pow};
 use util::LOGGER;