diff --git a/chain/src/chain.rs b/chain/src/chain.rs index 04fcbb991..d244a9405 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -26,7 +26,6 @@ use lmdb; use core::core::hash::{Hash, Hashed, ZERO_HASH}; use core::core::merkle_proof::MerkleProof; -use core::core::pmmr; use core::core::verifier_cache::VerifierCache; use core::core::{ Block, BlockHeader, BlockSums, Output, OutputIdentifier, Transaction, TxKernelEntry, @@ -744,11 +743,17 @@ impl Chain { ); } + // + // TODO - Investigate finding the "common header" by comparing header_mmr and + // sync_mmr (bytes will be identical up to the common header). + // while let Ok(header) = current { // break out of the while loop when we find a header common // between the header chain and the current body chain - if let Ok(_) = self.is_on_current_chain(&header) { - break; + if header.height <= body_head.height { + if let Ok(_) = self.is_on_current_chain(&header) { + break; + } } oldest_height = header.height; @@ -839,8 +844,6 @@ impl Chain { { let tip = Tip::from_header(&header); batch.save_body_head(&tip)?; - batch.save_header_height(&header)?; - batch.build_by_height_index(&header, true)?; // Reset the body tail to the body head after a txhashset write batch.save_body_tail(&tip)?; @@ -908,8 +911,11 @@ impl Chain { } let mut count = 0; + + let tail = self.get_header_by_height(head.height - horizon)?; + let mut current = self.get_header_by_height(head.height - horizon - 1)?; + let batch = self.store.batch()?; - let mut current = batch.get_header_by_height(head.height - horizon - 1)?; loop { // Go to the store directly so we can handle NotFoundErr robustly. match self.store.get_block(¤t.hash()) { @@ -935,7 +941,6 @@ impl Chain { Err(e) => return Err(From::from(e)), } } - let tail = batch.get_header_by_height(head.height - horizon)?; batch.save_body_tail(&Tip::from_header(&tail))?; batch.commit()?; debug!( @@ -1075,9 +1080,14 @@ impl Chain { /// Gets the block header at the provided height pub fn get_header_by_height(&self, height: u64) -> Result { - self.store - .get_header_by_height(height) - .map_err(|e| ErrorKind::StoreErr(e, "chain get header by height".to_owned()).into()) + let mut txhashset = self.txhashset.write(); + let mut batch = self.store.batch()?; + let header = txhashset::header_extending(&mut txhashset, &mut batch, |extension| { + let header = extension.get_header_by_height(height)?; + Ok(header) + })?; + + Ok(header) } /// Gets the block header in which a given output appears in the txhashset @@ -1118,9 +1128,12 @@ impl Chain { /// Checks the header_by_height index to verify the header is where we say /// it is pub fn is_on_current_chain(&self, header: &BlockHeader) -> Result<(), Error> { - self.store - .is_on_current_chain(header) - .map_err(|e| ErrorKind::StoreErr(e, "chain is_on_current_chain".to_owned()).into()) + let chain_header = self.get_header_by_height(header.height)?; + if chain_header.hash() == header.hash() { + Ok(()) + } else { + Err(ErrorKind::Other(format!("not on current chain")).into()) + } } /// Get the tip of the current "sync" header chain. @@ -1179,10 +1192,9 @@ fn setup_head( // If we have no header MMR then rebuild as necessary. // Supports old nodes with no header MMR. txhashset::header_extending(txhashset, &mut batch, |extension| { - let pos = pmmr::insertion_to_pmmr_index(head.height + 1); - let needs_rebuild = match extension.get_header_hash(pos) { - None => true, - Some(hash) => hash != head.last_block_h, + let needs_rebuild = match extension.get_header_by_height(head.height) { + Ok(header) => header.hash() != head.last_block_h, + Err(_) => true, }; if needs_rebuild { @@ -1237,7 +1249,6 @@ fn setup_head( // header and try again let prev_header = batch.get_block_header(&head.prev_block_h)?; let _ = batch.delete_block(&header.hash()); - let _ = batch.setup_height(&prev_header, &head)?; head = Tip::from_header(&prev_header); batch.save_head(&head)?; } @@ -1251,7 +1262,6 @@ fn setup_head( let tip = Tip::from_header(&genesis.header); batch.save_head(&tip)?; - batch.setup_height(&genesis.header, &tip)?; // Initialize our header MM with the genesis header. txhashset::header_extending(txhashset, &mut batch, |extension| { diff --git a/chain/src/pipe.rs b/chain/src/pipe.rs index 72dee13d9..b07b27a3f 100644 --- a/chain/src/pipe.rs +++ b/chain/src/pipe.rs @@ -521,11 +521,6 @@ fn update_head(b: &Block, ctx: &BlockContext) -> Result, Error> { // when extending the head), update it let head = ctx.batch.head()?; if has_more_work(&b.header, &head) { - // Update the block height index based on this new head. - ctx.batch - .setup_height(&b.header, &head) - .map_err(|e| ErrorKind::StoreErr(e, "pipe setup height".to_owned()))?; - let tip = Tip::from_header(&b.header); ctx.batch @@ -585,7 +580,7 @@ pub fn rewind_and_apply_header_fork( ) -> Result<(), Error> { let mut fork_hashes = vec![]; let mut current = ext.batch.get_previous_header(header)?; - while current.height > 0 && !ext.batch.is_on_current_chain(¤t).is_ok() { + while current.height > 0 && !ext.is_on_current_chain(¤t).is_ok() { fork_hashes.push(current.hash()); current = ext.batch.get_previous_header(¤t)?; } @@ -616,7 +611,7 @@ pub fn rewind_and_apply_fork(b: &Block, ext: &mut txhashset::Extension) -> Resul // keeping the hashes of blocks along the fork let mut fork_hashes = vec![]; let mut current = ext.batch.get_previous_header(&b.header)?; - while current.height > 0 && !ext.batch.is_on_current_chain(¤t).is_ok() { + while current.height > 0 && !ext.is_on_current_chain(¤t).is_ok() { fork_hashes.push(current.hash()); current = ext.batch.get_previous_header(¤t)?; } diff --git a/chain/src/store.rs b/chain/src/store.rs index af9d030dc..060b94e26 100644 --- a/chain/src/store.rs +++ b/chain/src/store.rs @@ -26,7 +26,7 @@ use core::core::hash::{Hash, Hashed}; use core::core::{Block, BlockHeader, BlockSums}; use core::pow::Difficulty; use grin_store as store; -use grin_store::{option_to_not_found, to_key, u64_to_key, Error}; +use grin_store::{option_to_not_found, to_key, Error}; use types::Tip; const STORE_SUBPATH: &'static str = "chain"; @@ -37,7 +37,6 @@ const HEAD_PREFIX: u8 = 'H' as u8; const TAIL_PREFIX: u8 = 'T' as u8; const HEADER_HEAD_PREFIX: u8 = 'I' as u8; const SYNC_HEAD_PREFIX: u8 = 's' as u8; -const HEADER_HEIGHT_PREFIX: u8 = '8' as u8; const COMMIT_POS_PREFIX: u8 = 'c' as u8; const BLOCK_INPUT_BITMAP_PREFIX: u8 = 'B' as u8; const BLOCK_SUMS_PREFIX: u8 = 'M' as u8; @@ -109,43 +108,6 @@ impl ChainStore { ) } - pub fn get_hash_by_height(&self, height: u64) -> Result { - option_to_not_found( - self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)), - &format!("Hash at height: {}", height), - ) - } - - // We are on the current chain if - - // * the header by height index matches the header, and - // * we are not ahead of the current head - pub fn is_on_current_chain(&self, header: &BlockHeader) -> Result<(), Error> { - let head = self.head()?; - - // check we are not out ahead of the current head - if header.height > head.height { - return Err(Error::NotFoundErr(String::from( - "header.height > head.height", - ))); - } - - let header_at_height = self.get_header_by_height(header.height)?; - if header.hash() == header_at_height.hash() { - Ok(()) - } else { - Err(Error::NotFoundErr(String::from( - "header.hash == header_at_height.hash", - ))) - } - } - - pub fn get_header_by_height(&self, height: u64) -> Result { - option_to_not_found( - self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)), - &format!("Header at height: {}", height), - ).and_then(|hash| self.get_block_header(&hash)) - } - pub fn get_output_pos(&self, commit: &Commitment) -> Result { option_to_not_found( self.db @@ -209,13 +171,6 @@ impl<'a> Batch<'a> { self.db.put_ser(&vec![HEADER_HEAD_PREFIX], t) } - pub fn get_hash_by_height(&self, height: u64) -> Result { - option_to_not_found( - self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)), - &format!("Hash at height: {}", height), - ) - } - pub fn save_sync_head(&self, t: &Tip) -> Result<(), Error> { self.db.put_ser(&vec![SYNC_HEAD_PREFIX], t) } @@ -283,15 +238,6 @@ impl<'a> Batch<'a> { Ok(()) } - pub fn save_header_height(&self, bh: &BlockHeader) -> Result<(), Error> { - self.db - .put_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, bh.height), &bh.hash()) - } - - pub fn delete_header_by_height(&self, height: u64) -> Result<(), Error> { - self.db.delete(&u64_to_key(HEADER_HEIGHT_PREFIX, height)) - } - pub fn save_output_pos(&self, commit: &Commitment, pos: u64) -> Result<(), Error> { self.db.put_ser( &to_key(COMMIT_POS_PREFIX, &mut commit.as_ref().to_vec())[..], @@ -352,70 +298,6 @@ impl<'a> Batch<'a> { self.db.delete(&to_key(BLOCK_SUMS_PREFIX, &mut bh.to_vec())) } - // We are on the current chain if - - // * the header by height index matches the header, and - // * we are not ahead of the current head - pub fn is_on_current_chain(&self, header: &BlockHeader) -> Result<(), Error> { - let head = self.head()?; - - // check we are not out ahead of the current head - if header.height > head.height { - return Err(Error::NotFoundErr(String::from( - "header.height > head.height", - ))); - } - - let header_at_height = self.get_header_by_height(header.height)?; - if header.hash() == header_at_height.hash() { - Ok(()) - } else { - Err(Error::NotFoundErr(String::from( - "header.hash == header_at_height.hash", - ))) - } - } - - pub fn get_header_by_height(&self, height: u64) -> Result { - option_to_not_found( - self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)), - &format!("Header at height: {}", height), - ).and_then(|hash| self.get_block_header(&hash)) - } - - /// Maintain consistency of the "header_by_height" index by traversing back - /// through the current chain and updating "header_by_height" until we reach - /// a block_header that is consistent with its height (everything prior to - /// this will be consistent). - /// We need to handle the case where we have no index entry for a given - /// height to account for the case where we just switched to a new fork and - /// the height jumped beyond current chain height. - pub fn setup_height(&self, header: &BlockHeader, old_tip: &Tip) -> Result<(), Error> { - // remove headers ahead if we backtracked - for n in header.height..old_tip.height { - self.delete_header_by_height(n + 1)?; - } - self.build_by_height_index(header, false) - } - - pub fn build_by_height_index(&self, header: &BlockHeader, force: bool) -> Result<(), Error> { - self.save_header_height(&header)?; - - if header.height > 0 { - let mut prev_header = self.get_previous_header(&header)?; - while prev_header.height > 0 { - if !force { - if let Ok(_) = self.is_on_current_chain(&prev_header) { - break; - } - } - self.save_header_height(&prev_header)?; - - prev_header = self.get_previous_header(&prev_header)?; - } - } - Ok(()) - } - fn build_block_input_bitmap(&self, block: &Block) -> Result { let bitmap = block .inputs() diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 422d444e2..1971b75c5 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -204,6 +204,22 @@ impl TxHashSet { kernel_pmmr.get_last_n_insertions(distance) } + /// Get the header at the specified height based on the current state of the txhashset. + /// Derives the MMR pos from the height (insertion index) and retrieves the header hash. + /// Looks the header up in the db by hash. + pub fn get_header_by_height(&mut self, height: u64) -> Result { + let pos = pmmr::insertion_to_pmmr_index(height + 1); + + let header_pmmr: PMMR = + PMMR::at(&mut self.header_pmmr_h.backend, self.header_pmmr_h.last_pos); + if let Some(hash) = header_pmmr.get_data(pos) { + let header = self.commit_index.get_block_header(&hash)?; + Ok(header) + } else { + Err(ErrorKind::Other(format!("get header by height")).into()) + } + } + /// returns outputs from the given insertion (leaf) index up to the /// specified limit. Also returns the last index actually populated pub fn outputs_by_insertion_index( @@ -267,7 +283,7 @@ impl TxHashSet { // horizon for compacting is based on current_height let horizon = current_height.saturating_sub(global::cut_through_horizon().into()); - let horizon_header = self.commit_index.get_header_by_height(horizon)?; + let horizon_header = self.get_header_by_height(horizon)?; let batch = self.commit_index.batch()?; @@ -608,10 +624,34 @@ impl<'a> HeaderExtension<'a> { } /// Get the header hash for the specified pos from the underlying MMR backend. - pub fn get_header_hash(&self, pos: u64) -> Option { + fn get_header_hash(&self, pos: u64) -> Option { self.pmmr.get_data(pos) } + /// Get the header at the specified height based on the current state of the header extension. + /// Derives the MMR pos from the height (insertion index) and retrieves the header hash. + /// Looks the header up in the db by hash. + pub fn get_header_by_height(&mut self, height: u64) -> Result { + let pos = pmmr::insertion_to_pmmr_index(height + 1); + if let Some(hash) = self.get_header_hash(pos) { + let header = self.batch.get_block_header(&hash)?; + Ok(header) + } else { + Err(ErrorKind::Other(format!("get header by height")).into()) + } + } + + /// Compares the provided header to the header in the header MMR at that height. + /// If these match we know the header is on the current chain. + pub fn is_on_current_chain(&mut self, header: &BlockHeader) -> Result<(), Error> { + let chain_header = self.get_header_by_height(header.height)?; + if chain_header.hash() == header.hash() { + Ok(()) + } else { + Err(ErrorKind::Other(format!("not on current chain")).into()) + } + } + /// Force the rollback of this extension, no matter the result. pub fn force_rollback(&mut self) { self.rollback = true; @@ -806,10 +846,13 @@ impl<'a> Extension<'a> { UTXOView::new(self.output_pmmr.readonly_pmmr(), self.batch) } - // TODO - move this into "utxo_view" /// Verify we are not attempting to spend any coinbase outputs /// that have not sufficiently matured. - pub fn verify_coinbase_maturity(&self, inputs: &Vec, height: u64) -> Result<(), Error> { + pub fn verify_coinbase_maturity( + &mut self, + inputs: &Vec, + height: u64, + ) -> Result<(), Error> { // Find the greatest output pos of any coinbase // outputs we are attempting to spend. let pos = inputs @@ -829,7 +872,7 @@ impl<'a> Extension<'a> { // Find the "cutoff" pos in the output MMR based on the // header from 1,000 blocks ago. let cutoff_height = height.checked_sub(global::coinbase_maturity()).unwrap_or(0); - let cutoff_header = self.batch.get_header_by_height(cutoff_height)?; + let cutoff_header = self.get_header_by_height(cutoff_height)?; let cutoff_pos = cutoff_header.output_mmr_size; // If any output pos exceed the cutoff_pos @@ -965,7 +1008,35 @@ impl<'a> Extension<'a> { Ok(()) } - /// TODO - move this into "utxo_view" + /// Get the header hash for the specified pos from the underlying MMR backend. + fn get_header_hash(&self, pos: u64) -> Option { + self.header_pmmr.get_data(pos) + } + + /// Get the header at the specified height based on the current state of the extension. + /// Derives the MMR pos from the height (insertion index) and retrieves the header hash. + /// Looks the header up in the db by hash. + pub fn get_header_by_height(&mut self, height: u64) -> Result { + let pos = pmmr::insertion_to_pmmr_index(height + 1); + if let Some(hash) = self.get_header_hash(pos) { + let header = self.batch.get_block_header(&hash)?; + Ok(header) + } else { + Err(ErrorKind::Other(format!("get header by height")).into()) + } + } + + /// Compares the provided header to the header in the header MMR at that height. + /// If these match we know the header is on the current chain. + pub fn is_on_current_chain(&mut self, header: &BlockHeader) -> Result<(), Error> { + let chain_header = self.get_header_by_height(header.height)?; + if chain_header.hash() == header.hash() { + Ok(()) + } else { + Err(ErrorKind::Other(format!("not on current chain")).into()) + } + } + /// Build a Merkle proof for the given output and the block /// this extension is currently referencing. /// Note: this relies on the MMR being stable even after pruning/compaction. @@ -1487,9 +1558,6 @@ pub fn input_pos_to_rewind( head_header: &BlockHeader, batch: &Batch, ) -> Result { - let mut current = head_header.hash(); - let mut height = head_header.height; - if head_header.height < block_header.height { debug!( "input_pos_to_rewind: {} < {}, nothing to rewind", @@ -1514,21 +1582,19 @@ pub fn input_pos_to_rewind( }; let mut block_input_bitmaps: Vec = vec![]; - let bh = block_header.hash(); - while current != bh { - // We cache recent block headers and block_input_bitmaps - // internally in our db layer (commit_index). - // I/O should be minimized or eliminated here for most - // rewind scenarios. - if let Ok(b_res) = batch.get_block_input_bitmap(¤t) { - bitmap_fast_or(Some(b_res), &mut block_input_bitmaps); - } - if height == 0 { + let mut current = head_header.clone(); + while current.hash() != block_header.hash() { + if current.height < 1 { break; } - height -= 1; - current = batch.get_hash_by_height(height)?; + + // I/O should be minimized or eliminated here for most + // rewind scenarios. + if let Ok(b_res) = batch.get_block_input_bitmap(¤t.hash()) { + bitmap_fast_or(Some(b_res), &mut block_input_bitmaps); + } + current = batch.get_previous_header(¤t)?; } let bitmap = bitmap_fast_or(None, &mut block_input_bitmaps).unwrap(); diff --git a/chain/tests/store_indices.rs b/chain/tests/store_indices.rs index 986f1cfbd..822ce9e96 100644 --- a/chain/tests/store_indices.rs +++ b/chain/tests/store_indices.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use chain::{Error, Tip}; use core::core::hash::Hashed; -use core::core::{Block, BlockHeader}; +use core::core::Block; use core::global::{self, ChainTypes}; use core::pow::{self, Difficulty}; use keychain::{ExtKeychain, ExtKeychainPath, Keychain}; @@ -41,7 +41,6 @@ fn setup_chain(genesis: &Block, chain_store: Arc) -> R batch.save_block(&genesis)?; let head = Tip::from_header(&genesis.header); batch.save_head(&head)?; - batch.setup_height(&genesis.header, &head)?; batch.save_block_header(&genesis.header)?; batch.commit()?; Ok(()) @@ -75,18 +74,12 @@ fn test_various_store_indices() { let batch = chain_store.batch().unwrap(); batch.save_block_header(&block.header).unwrap(); batch.save_block(&block).unwrap(); - batch - .setup_height(&block.header, &Tip::from_header(&block.header)) - .unwrap(); batch.commit().unwrap(); } let block_header = chain_store.get_block_header(&block_hash).unwrap(); assert_eq!(block_header.hash(), block_hash); - let block_header = chain_store.get_header_by_height(1).unwrap(); - assert_eq!(block_header.hash(), block_hash); - // Test we can retrive the block from the db and that we can safely delete the // block from the db even though the block_sums are missing. { @@ -109,43 +102,3 @@ fn test_various_store_indices() { assert!(chain_store.get_block(&block_hash).is_ok()); } } - -#[test] -fn test_store_header_height() { - match env_logger::try_init() { - Ok(_) => println!("Initializing env logger"), - Err(e) => println!("env logger already initialized: {:?}", e), - }; - let chain_dir = ".grin_idx_2"; - clean_output_dir(chain_dir); - - let db_env = Arc::new(store::new_env(chain_dir.to_string())); - let chain_store = Arc::new(chain::store::ChainStore::new(db_env).unwrap()); - - global::set_mining_mode(ChainTypes::AutomatedTesting); - let genesis = pow::mine_genesis_block().unwrap(); - - setup_chain(&genesis, chain_store.clone()).unwrap(); - - let mut block_header = BlockHeader::default(); - block_header.height = 1; - - { - let batch = chain_store.batch().unwrap(); - batch.save_block_header(&block_header).unwrap(); - batch.save_header_height(&block_header).unwrap(); - batch.commit().unwrap(); - } - - let stored_block_header = chain_store.get_header_by_height(1).unwrap(); - assert_eq!(block_header.hash(), stored_block_header.hash()); - - { - let batch = chain_store.batch().unwrap(); - batch.delete_header_by_height(1).unwrap(); - batch.commit().unwrap(); - } - - let result = chain_store.get_header_by_height(1); - assert_eq!(result.is_err(), true); -} diff --git a/servers/src/common/adapters.rs b/servers/src/common/adapters.rs index efed58237..ff3b4ff7e 100644 --- a/servers/src/common/adapters.rs +++ b/servers/src/common/adapters.rs @@ -261,33 +261,32 @@ impl p2p::ChainAdapter for NetToChainAdapter { } fn locate_headers(&self, locator: Vec) -> Vec { - debug!("locate_headers: {:?}", locator,); + debug!("locator: {:?}", locator); let header = match self.find_common_header(locator) { Some(header) => header, None => return vec![], }; - debug!("locate_headers: common header: {:?}", header.hash(),); + let max_height = self.chain().header_head().unwrap().height; // looks like we know one, getting as many following headers as allowed let hh = header.height; let mut headers = vec![]; for h in (hh + 1)..(hh + (p2p::MAX_BLOCK_HEADERS as u64)) { - let header = self.chain().get_header_by_height(h); - match header { - Ok(head) => headers.push(head), - Err(e) => match e.kind() { - chain::ErrorKind::StoreErr(store::Error::NotFoundErr(_), _) => break, - _ => { - error!("Could not build header locator: {:?}", e); - return vec![]; - } - }, + if h > max_height { + break; + } + + if let Ok(header) = self.chain().get_header_by_height(h) { + headers.push(header); + } else { + error!("Failed to locate headers successfully."); + break; } } - debug!("locate_headers: returning headers: {}", headers.len(),); + debug!("returning headers: {}", headers.len()); headers }