height (header version) specific output PMMR root rules (#3147)

* height (header version) specific output PMMR root rules

* cleanup

* cleanup based on PR feedback

* address feedback

api/src/handlers/transactions_api.rs

@@ -47,7 +47,10 @@ pub struct TxHashSetHandler {
 impl TxHashSetHandler {
 	// gets roots
 	fn get_roots(&self) -> Result<TxHashSet, Error> {
-		Ok(TxHashSet::from_head(w(&self.chain)?))
+		let res = TxHashSet::from_head(w(&self.chain)?).context(ErrorKind::Internal(
+			"failed to read roots from txhashset".to_owned(),
+		))?;
+		Ok(res)
 	}
 
 	// gets last n outputs inserted in to the tree

api/src/types.rs

@@ -116,13 +116,16 @@ pub struct TxHashSet {
 }
 
 impl TxHashSet {
-	pub fn from_head(head: Arc<chain::Chain>) -> TxHashSet {
-		let roots = head.get_txhashset_roots();
-		TxHashSet {
-			output_root_hash: roots.output_root().to_hex(),
-			range_proof_root_hash: roots.rproof_root.to_hex(),
-			kernel_root_hash: roots.kernel_root.to_hex(),
-		}
+	/// A TxHashSet in the context of the api is simply the collection of PMMR roots.
+	/// We can obtain these in a lightweight way by reading them from the head of the chain.
+	/// We will have validated the roots on this header against the roots of the txhashset.
+	pub fn from_head(chain: Arc<chain::Chain>) -> Result<TxHashSet, chain::Error> {
+		let header = chain.head_header()?;
+		Ok(TxHashSet {
+			output_root_hash: header.output_root.to_hex(),
+			range_proof_root_hash: header.range_proof_root.to_hex(),
+			kernel_root_hash: header.kernel_root.to_hex(),
+		})
 	}
 }
 

chain/src/chain.rs

@@ -30,8 +30,7 @@ use crate::store;
 use crate::txhashset;
 use crate::txhashset::{PMMRHandle, TxHashSet};
 use crate::types::{
-	BlockStatus, ChainAdapter, NoStatus, Options, OutputMMRPosition, Tip, TxHashSetRoots,
-	TxHashsetWriteStatus,
+	BlockStatus, ChainAdapter, NoStatus, Options, OutputMMRPosition, Tip, TxHashsetWriteStatus,
 };
 use crate::util::secp::pedersen::{Commitment, RangeProof};
 use crate::util::RwLock;
@@ -590,15 +589,9 @@ impl Chain {
 				Ok((prev_root, extension.roots()?, extension.sizes()))
 			})?;
 
-		// Set the prev_root on the header.
-		b.header.prev_root = prev_root;
-
-		// Set the output, rangeproof and kernel MMR roots.
-		b.header.output_root = roots.output_root();
-		b.header.range_proof_root = roots.rproof_root;
-		b.header.kernel_root = roots.kernel_root;
-
 		// Set the output and kernel MMR sizes.
+		// Note: We need to do this *before* calculating the roots as the output_root
+		// depends on the output_mmr_size
 		{
 			// Carefully destructure these correctly...
 			let (output_mmr_size, _, kernel_mmr_size) = sizes;
@@ -606,6 +599,14 @@ impl Chain {
 			b.header.kernel_mmr_size = kernel_mmr_size;
 		}
 
+		// Set the prev_root on the header.
+		b.header.prev_root = prev_root;
+
+		// Set the output, rangeproof and kernel MMR roots.
+		b.header.output_root = roots.output_root(&b.header);
+		b.header.range_proof_root = roots.rproof_root;
+		b.header.kernel_root = roots.kernel_root;
+
 		Ok(())
 	}
 
@@ -634,11 +635,6 @@ impl Chain {
 		txhashset.merkle_proof(commit)
 	}
 
-	/// Returns current txhashset roots.
-	pub fn get_txhashset_roots(&self) -> TxHashSetRoots {
-		self.txhashset.read().roots()
-	}
-
 	/// Provides a reading view into the current kernel state.
 	pub fn kernel_data_read(&self) -> Result<File, Error> {
 		let txhashset = self.txhashset.read();

chain/src/pipe.rs

@@ -312,12 +312,16 @@ fn validate_header(header: &BlockHeader, ctx: &mut BlockContext<'_>) -> Result<(
 	// First I/O cost, delayed as late as possible.
 	let prev = prev_header_store(header, &mut ctx.batch)?;
 
-	// make sure this header has a height exactly one higher than the previous
-	// header
+	// This header height must increase the height from the previous header by exactly 1.
 	if header.height != prev.height + 1 {
 		return Err(ErrorKind::InvalidBlockHeight.into());
 	}
 
+	// This header must have a valid header version for its height.
+	if !consensus::valid_header_version(header.height, header.version) {
+		return Err(ErrorKind::InvalidBlockVersion(header.version).into());
+	}
+
 	if header.timestamp <= prev.timestamp {
 		// prevent time warp attacks and some timestamp manipulations by forcing strict
 		// time progression

chain/src/types.rs

@@ -18,7 +18,7 @@ use chrono::prelude::{DateTime, Utc};
 use std::sync::Arc;
 
 use crate::core::core::hash::{Hash, Hashed, ZERO_HASH};
-use crate::core::core::{Block, BlockHeader};
+use crate::core::core::{Block, BlockHeader, HeaderVersion};
 use crate::core::pow::Difficulty;
 use crate::core::ser::{self, PMMRIndexHashable};
 use crate::error::{Error, ErrorKind};
@@ -193,24 +193,26 @@ pub struct TxHashSetRoots {
 }
 
 impl TxHashSetRoots {
-	/// Accessor for the underlying output PMMR root
-	pub fn output_root(&self) -> Hash {
-		self.output_roots.output_root()
+	/// Accessor for the output PMMR root (rules here are block height dependent).
+	/// We assume the header version is consistent with the block height, validated
+	/// as part of pipe::validate_header().
+	pub fn output_root(&self, header: &BlockHeader) -> Hash {
+		self.output_roots.root(header)
 	}
 
 	/// Validate roots against the provided block header.
 	pub fn validate(&self, header: &BlockHeader) -> Result<(), Error> {
 		debug!(
-			"validate roots: {} at {}, output_root: {}, output pmmr: {} (bitmap: {}, merged: {})",
+			"validate roots: {} at {}, {} vs. {} (original: {}, merged: {})",
 			header.hash(),
 			header.height,
 			header.output_root,
-			self.output_roots.output_root(),
-			self.output_roots.bitmap_root,
+			self.output_root(header),
+			self.output_roots.pmmr_root,
 			self.output_roots.merged_root(header),
 		);
 
-		if header.output_root != self.output_roots.pmmr_root {
+		if header.output_root != self.output_root(header) {
 			Err(ErrorKind::InvalidRoot.into())
 		} else if header.range_proof_root != self.rproof_root {
 			Err(ErrorKind::InvalidRoot.into())
@@ -232,15 +234,27 @@ pub struct OutputRoots {
 }
 
 impl OutputRoots {
+	/// The root of our output PMMR. The rules here are block height specific.
+	/// We use the merged root here for header version 3 and later.
+	/// We assume the header version is consistent with the block height, validated
+	/// as part of pipe::validate_header().
+	pub fn root(&self, header: &BlockHeader) -> Hash {
+		if header.version < HeaderVersion(3) {
+			self.output_root()
+		} else {
+			self.merged_root(header)
+		}
+	}
+
 	/// The root of the underlying output PMMR.
-	pub fn output_root(&self) -> Hash {
+	fn output_root(&self) -> Hash {
 		self.pmmr_root
 	}
 
 	/// Hash the root of the output PMMR and the root of the bitmap accumulator
 	/// together with the size of the output PMMR (for consistency with existing PMMR impl).
 	/// H(pmmr_size | pmmr_root | bitmap_root)
-	pub fn merged_root(&self, header: &BlockHeader) -> Hash {
+	fn merged_root(&self, header: &BlockHeader) -> Hash {
 		(self.pmmr_root, self.bitmap_root).hash_with_index(header.output_mmr_size)
 	}
 }

core/src/consensus.rs

@@ -133,12 +133,19 @@ pub const FLOONET_FIRST_HARD_FORK: u64 = 185_040;
 /// Floonet second hard fork height, set to happen around 2019-12-19
 pub const FLOONET_SECOND_HARD_FORK: u64 = 298_080;
 
+/// AutomatedTesting and UserTesting first hard fork height.
+pub const TESTING_FIRST_HARD_FORK: u64 = 3;
+
+/// AutomatedTesting and UserTesting second hard fork height.
+pub const TESTING_SECOND_HARD_FORK: u64 = 6;
+
 /// Compute possible block version at a given height, implements
 /// 6 months interval scheduled hard forks for the first 2 years.
 pub fn header_version(height: u64) -> HeaderVersion {
 	let chain_type = global::CHAIN_TYPE.read().clone();
 	let hf_interval = (1 + height / HARD_FORK_INTERVAL) as u16;
 	match chain_type {
+		global::ChainTypes::Mainnet => HeaderVersion(hf_interval),
 		global::ChainTypes::Floonet => {
 			if height < FLOONET_FIRST_HARD_FORK {
 				(HeaderVersion(1))
@@ -150,8 +157,17 @@ pub fn header_version(height: u64) -> HeaderVersion {
 				HeaderVersion(hf_interval)
 			}
 		}
-		// everything else just like mainnet
-		_ => HeaderVersion(hf_interval),
+		global::ChainTypes::AutomatedTesting | global::ChainTypes::UserTesting => {
+			if height < TESTING_FIRST_HARD_FORK {
+				(HeaderVersion(1))
+			} else if height < TESTING_SECOND_HARD_FORK {
+				(HeaderVersion(2))
+			} else if height < 3 * HARD_FORK_INTERVAL {
+				(HeaderVersion(3))
+			} else {
+				HeaderVersion(hf_interval)
+			}
+		}
 	}
 }
 

core/src/core/block.rs

@@ -176,7 +176,7 @@ impl Hashed for HeaderEntry {
 }
 
 /// Some type safety around header versioning.
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Serialize)]
 pub struct HeaderVersion(pub u16);
 
 impl From<HeaderVersion> for u16 {