diff --git a/chain/src/chain.rs b/chain/src/chain.rs index b189df040..34835efd9 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -18,7 +18,7 @@ use crate::core::core::merkle_proof::MerkleProof; use crate::core::core::{ Block, BlockHeader, BlockSums, Committed, Inputs, KernelFeatures, Output, OutputIdentifier, - SegmentIdentifier, Transaction, TxKernel, + Transaction, TxKernel, }; use crate::core::global; use crate::core::pow; @@ -210,21 +210,6 @@ impl Chain { chain.log_heads()?; - // Temporarily exercising the initialization process. - // Note: This is *really* slow because we are starting from cold. - // - // This is not required as we will lazily initialize our segmenter as required - // once we start receiving PIBD segment requests. - // In reality we will do this based on PIBD segment requests. - // Initialization (once per 12 hour period) will not be this slow once lmdb and PMMRs - // are warmed up. - if let Ok(segmenter) = chain.segmenter() { - let _ = segmenter.kernel_segment(SegmentIdentifier { height: 9, idx: 0 }); - let _ = segmenter.bitmap_segment(SegmentIdentifier { height: 9, idx: 0 }); - let _ = segmenter.output_segment(SegmentIdentifier { height: 11, idx: 0 }); - let _ = segmenter.rangeproof_segment(SegmentIdentifier { height: 7, idx: 0 }); - } - Ok(chain) } @@ -1143,7 +1128,8 @@ impl Chain { return Ok(()); } - let horizon = global::cut_through_horizon() as u64; + let mut horizon = global::cut_through_horizon() as u64; + let head = batch.head()?; let tail = match batch.tail() { @@ -1151,7 +1137,16 @@ impl Chain { Err(_) => Tip::from_header(&self.genesis), }; - let cutoff = head.height.saturating_sub(horizon); + let mut cutoff = head.height.saturating_sub(horizon); + + // TODO: Check this, compaction selects a different horizon + // block from txhashset horizon/PIBD segmenter when using + // Automated testing chain + let archive_header = self.txhashset_archive_header()?; + if archive_header.height < cutoff { + cutoff = archive_header.height; + horizon = head.height - archive_header.height; + } debug!( "remove_historical_blocks: head height: {}, tail height: {}, horizon: {}, cutoff: {}", diff --git a/chain/src/txhashset/bitmap_accumulator.rs b/chain/src/txhashset/bitmap_accumulator.rs index 057e01667..efcf5a812 100644 --- a/chain/src/txhashset/bitmap_accumulator.rs +++ b/chain/src/txhashset/bitmap_accumulator.rs @@ -215,6 +215,16 @@ impl BitmapChunk { pub fn any(&self) -> bool { self.0.any() } + + /// Iterator over the integer set represented by this chunk, applying the given + /// offset to the values + pub fn set_iter(&self, idx_offset: usize) -> impl Iterator + '_ { + self.0 + .iter() + .enumerate() + .filter(|(_, val)| *val) + .map(move |(idx, _)| (idx as u32 + idx_offset as u32)) + } } impl PMMRable for BitmapChunk { diff --git a/chain/tests/test_data/chain_compacted/header/header_head/pmmr_data.bin b/chain/tests/test_data/chain_compacted/header/header_head/pmmr_data.bin new file mode 100644 index 000000000..fc4b2f52e Binary files /dev/null and b/chain/tests/test_data/chain_compacted/header/header_head/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_compacted/header/header_head/pmmr_hash.bin b/chain/tests/test_data/chain_compacted/header/header_head/pmmr_hash.bin new file mode 100644 index 000000000..741ecfb1e Binary files /dev/null and b/chain/tests/test_data/chain_compacted/header/header_head/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_compacted/lmdb/data.mdb b/chain/tests/test_data/chain_compacted/lmdb/data.mdb new file mode 100644 index 000000000..d1bed4039 Binary files /dev/null and b/chain/tests/test_data/chain_compacted/lmdb/data.mdb differ diff --git a/chain/tests/test_data/chain_compacted/lmdb/lock.mdb b/chain/tests/test_data/chain_compacted/lmdb/lock.mdb new file mode 100644 index 000000000..c32379f06 Binary files /dev/null and b/chain/tests/test_data/chain_compacted/lmdb/lock.mdb differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_data.bin b/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_data.bin new file mode 100644 index 000000000..4e24d88ac Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_hash.bin b/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_hash.bin new file mode 100644 index 000000000..6202e40e9 Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_size.bin b/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_size.bin new file mode 100644 index 000000000..bba2b6239 Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/kernel/pmmr_size.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_data.bin b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_data.bin new file mode 100644 index 000000000..eb7aaec6a Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_hash.bin b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_hash.bin new file mode 100644 index 000000000..cadd9a2d3 Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_leaf.bin b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_leaf.bin new file mode 100644 index 000000000..c8bfd9c9c Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_leaf.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_prun.bin b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_prun.bin new file mode 100644 index 000000000..4324337fb Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/output/pmmr_prun.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_data.bin b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_data.bin new file mode 100644 index 000000000..7a5f95c7f Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_hash.bin b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_hash.bin new file mode 100644 index 000000000..75f293358 Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_leaf.bin b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_leaf.bin new file mode 100644 index 000000000..c8bfd9c9c Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_leaf.bin differ diff --git a/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_prun.bin b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_prun.bin new file mode 100644 index 000000000..4324337fb Binary files /dev/null and b/chain/tests/test_data/chain_compacted/txhashset/rangeproof/pmmr_prun.bin differ diff --git a/chain/tests/test_data/chain_raw/header/header_head/pmmr_data.bin b/chain/tests/test_data/chain_raw/header/header_head/pmmr_data.bin new file mode 100644 index 000000000..fc4b2f52e Binary files /dev/null and b/chain/tests/test_data/chain_raw/header/header_head/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_raw/header/header_head/pmmr_hash.bin b/chain/tests/test_data/chain_raw/header/header_head/pmmr_hash.bin new file mode 100644 index 000000000..741ecfb1e Binary files /dev/null and b/chain/tests/test_data/chain_raw/header/header_head/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_raw/lmdb/data.mdb b/chain/tests/test_data/chain_raw/lmdb/data.mdb new file mode 100644 index 000000000..d907f73dc Binary files /dev/null and b/chain/tests/test_data/chain_raw/lmdb/data.mdb differ diff --git a/chain/tests/test_data/chain_raw/lmdb/lock.mdb b/chain/tests/test_data/chain_raw/lmdb/lock.mdb new file mode 100644 index 000000000..f973e6ac4 Binary files /dev/null and b/chain/tests/test_data/chain_raw/lmdb/lock.mdb differ diff --git a/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_data.bin b/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_data.bin new file mode 100644 index 000000000..4e24d88ac Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_hash.bin b/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_hash.bin new file mode 100644 index 000000000..6202e40e9 Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_size.bin b/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_size.bin new file mode 100644 index 000000000..bba2b6239 Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/kernel/pmmr_size.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/output/pmmr_data.bin b/chain/tests/test_data/chain_raw/txhashset/output/pmmr_data.bin new file mode 100644 index 000000000..eb7aaec6a Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/output/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/output/pmmr_hash.bin b/chain/tests/test_data/chain_raw/txhashset/output/pmmr_hash.bin new file mode 100644 index 000000000..cadd9a2d3 Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/output/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/output/pmmr_leaf.bin b/chain/tests/test_data/chain_raw/txhashset/output/pmmr_leaf.bin new file mode 100644 index 000000000..c8bfd9c9c Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/output/pmmr_leaf.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_data.bin b/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_data.bin new file mode 100644 index 000000000..7a5f95c7f Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_data.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_hash.bin b/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_hash.bin new file mode 100644 index 000000000..75f293358 Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_hash.bin differ diff --git a/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_leaf.bin b/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_leaf.bin new file mode 100644 index 000000000..c8bfd9c9c Binary files /dev/null and b/chain/tests/test_data/chain_raw/txhashset/rangeproof/pmmr_leaf.bin differ diff --git a/chain/tests/test_pibd_validation.rs b/chain/tests/test_pibd_validation.rs new file mode 100644 index 000000000..ebb160c17 --- /dev/null +++ b/chain/tests/test_pibd_validation.rs @@ -0,0 +1,235 @@ +// Copyright 2021 The Grin Developers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use grin_chain as chain; +use grin_core as core; +use grin_util as util; + +use std::sync::Arc; + +use crate::chain::txhashset::BitmapAccumulator; +use crate::chain::types::NoopAdapter; +use crate::core::core::pmmr; +use crate::core::core::{hash::Hashed, pmmr::segment::SegmentIdentifier}; +use crate::core::{genesis, global, pow}; + +use croaring::Bitmap; + +mod chain_test_helper; + +fn test_pibd_chain_validation_impl(is_test_chain: bool, src_root_dir: &str) { + global::set_local_chain_type(global::ChainTypes::Mainnet); + let mut genesis = genesis::genesis_main(); + // Height at which to read kernel segments (lower than thresholds defined in spec - for testing) + let mut target_segment_height = 11; + + if is_test_chain { + global::set_local_chain_type(global::ChainTypes::AutomatedTesting); + genesis = pow::mine_genesis_block().unwrap(); + target_segment_height = 3; + } + + { + println!("Reading Chain, genesis block: {}", genesis.hash()); + let dummy_adapter = Arc::new(NoopAdapter {}); + + // The original chain we're reading from + let src_chain = Arc::new( + chain::Chain::init( + src_root_dir.into(), + dummy_adapter.clone(), + genesis.clone(), + pow::verify_size, + false, + ) + .unwrap(), + ); + + // For test compaction purposes + /*src_chain.compact().unwrap(); + src_chain + .validate(true) + .expect("Source chain validation failed, stop");*/ + + let sh = src_chain.get_header_by_height(0).unwrap(); + println!("Source Genesis - {}", sh.hash()); + + let horizon_header = src_chain.txhashset_archive_header().unwrap(); + + println!("Horizon header: {:?}", horizon_header); + + // Copy the header from source to output + // Not necessary for this test, we're just validating the source + /*for h in 1..=horizon_height { + let h = src_chain.get_header_by_height(h).unwrap(); + dest_chain.process_block_header(&h, options).unwrap(); + }*/ + + // Init segmenter, (note this still has to be lazy init somewhere on a peer) + // This is going to use the same block as horizon_header + let segmenter = src_chain.segmenter().unwrap(); + + // BITMAP - Read + Validate, Also recreate bitmap accumulator for target tx hash set + // Predict number of leaves (chunks) in the bitmap MMR from the number of outputs + let bitmap_mmr_num_leaves = + (pmmr::n_leaves(horizon_header.output_mmr_size) as f64 / 1024f64).ceil() as u64; + println!("BITMAP PMMR NUM_LEAVES: {}", bitmap_mmr_num_leaves); + + // And total size of the bitmap PMMR + let bitmap_pmmr_size = pmmr::peaks(bitmap_mmr_num_leaves + 1) + .last() + .unwrap_or(&pmmr::insertion_to_pmmr_index(bitmap_mmr_num_leaves)) + .clone(); + println!("BITMAP PMMR SIZE: {}", bitmap_pmmr_size); + println!( + "Bitmap Segments required: {}", + SegmentIdentifier::count_segments_required(bitmap_pmmr_size, target_segment_height) + ); + // TODO: This can probably be derived from the PMMR we'll eventually be building + // (check if total size is equal to total size at horizon header) + let identifier_iter = + SegmentIdentifier::traversal_iter(bitmap_pmmr_size, target_segment_height); + + let mut bitmap_accumulator = BitmapAccumulator::new(); + // Raw bitmap for validation + let mut bitmap = Bitmap::create(); + let mut chunk_count = 0; + + for sid in identifier_iter { + println!("Getting bitmap segment with Segment Identifier {:?}", sid); + let (bitmap_segment, output_root_hash) = segmenter.bitmap_segment(sid).unwrap(); + println!( + "Bitmap segmenter reports output root hash is {:?}", + output_root_hash + ); + // Validate bitmap segment with provided output hash + if let Err(e) = bitmap_segment.validate_with( + bitmap_pmmr_size, // Last MMR pos at the height being validated, in this case of the bitmap root + None, + horizon_header.output_root, // Output root we're checking for + horizon_header.output_mmr_size, + output_root_hash, // Other root + true, + ) { + panic!("Unable to validate bitmap_root: {}", e); + } + + let (_sid, _hash_pos, _hashes, _leaf_pos, leaf_data, _proof) = bitmap_segment.parts(); + + // Add to raw bitmap to use in further validation + for chunk in leaf_data.iter() { + bitmap.add_many(&chunk.set_iter(chunk_count * 1024).collect::>()); + chunk_count += 1; + } + + // and append to bitmap accumulator + for chunk in leaf_data.into_iter() { + bitmap_accumulator.append_chunk(chunk).unwrap(); + } + } + + println!("Accumulator Root: {}", bitmap_accumulator.root()); + + // OUTPUTS - Read + Validate + let identifier_iter = SegmentIdentifier::traversal_iter( + horizon_header.output_mmr_size, + target_segment_height, + ); + + for sid in identifier_iter { + println!("Getting output segment with Segment Identifier {:?}", sid); + let (output_segment, bitmap_root_hash) = segmenter.output_segment(sid).unwrap(); + println!( + "Output segmenter reports bitmap hash is {:?}", + bitmap_root_hash + ); + // Validate Output + if let Err(e) = output_segment.validate_with( + horizon_header.output_mmr_size, // Last MMR pos at the height being validated + Some(&bitmap), + horizon_header.output_root, // Output root we're checking for + horizon_header.output_mmr_size, + bitmap_root_hash, // Other root + false, + ) { + panic!("Unable to validate output segment root: {}", e); + } + } + + // PROOFS - Read + Validate + let identifier_iter = SegmentIdentifier::traversal_iter( + horizon_header.output_mmr_size, + target_segment_height, + ); + + for sid in identifier_iter { + println!( + "Getting rangeproof segment with Segment Identifier {:?}", + sid + ); + let rangeproof_segment = segmenter.rangeproof_segment(sid).unwrap(); + // Validate Kernel segment (which does not require a bitmap) + if let Err(e) = rangeproof_segment.validate( + horizon_header.output_mmr_size, // Last MMR pos at the height being validated + Some(&bitmap), + horizon_header.range_proof_root, // Output root we're checking for + ) { + panic!("Unable to validate rangeproof segment root: {}", e); + } + } + + // KERNELS - Read + Validate + let identifier_iter = SegmentIdentifier::traversal_iter( + horizon_header.kernel_mmr_size, + target_segment_height, + ); + + for sid in identifier_iter { + println!("Getting kernel segment with Segment Identifier {:?}", sid); + let kernel_segment = segmenter.kernel_segment(sid).unwrap(); + // Validate Kernel segment (which does not require a bitmap) + if let Err(e) = kernel_segment.validate( + horizon_header.kernel_mmr_size, + None, + horizon_header.kernel_root, + ) { + panic!("Unable to validate kernel_segment root: {}", e); + } + } + } +} + +#[test] +fn test_pibd_chain_validation_sample() { + util::init_test_logger(); + // Note there is now a 'test' in grin_wallet_controller/build_chain + // that can be manually tweaked to create a + // small test chain with actual transaction data + + // Test on uncompacted and non-compacted chains + let src_root_dir = format!("./tests/test_data/chain_raw"); + test_pibd_chain_validation_impl(true, &src_root_dir); + let src_root_dir = format!("./tests/test_data/chain_compacted"); + test_pibd_chain_validation_impl(true, &src_root_dir); +} + +#[test] +#[ignore] +// As above, but run on a real instance of a chain pointed where you like +fn test_pibd_chain_validation_real() { + util::init_test_logger(); + // if testing against a real chain, insert location here + let src_root_dir = format!("/Users/yeastplume/Projects/grin_project/server/chain_data"); + test_pibd_chain_validation_impl(false, &src_root_dir); +} diff --git a/core/src/core/pmmr/segment.rs b/core/src/core/pmmr/segment.rs index 7d885e4b8..ecaa0bc15 100644 --- a/core/src/core/pmmr/segment.rs +++ b/core/src/core/pmmr/segment.rs @@ -69,6 +69,28 @@ impl Writeable for SegmentIdentifier { } } +impl SegmentIdentifier { + /// Test helper to get an iterator of SegmentIdentifiers required to read a + /// pmmr of size `target_mmr_size` in segments of height `segment_height` + pub fn traversal_iter( + target_mmr_size: u64, + segment_height: u8, + ) -> impl Iterator { + (0..SegmentIdentifier::count_segments_required(target_mmr_size, segment_height)).map( + move |idx| SegmentIdentifier { + height: segment_height, + idx: idx as u64, + }, + ) + } + + /// Returns number of segments required that would needed in order to read a + /// pmmr of size `target_mmr_size` in segments of height `segment_height` + pub fn count_segments_required(target_mmr_size: u64, segment_height: u8) -> usize { + pmmr::n_leaves(target_mmr_size) as usize / (1 << segment_height as usize) + } +} + /// Segment of a PMMR: unpruned leaves and the necessary data to verify /// segment membership in the original MMR. #[derive(Clone, Debug, Eq, PartialEq)]