diff --git a/chain/src/store.rs b/chain/src/store.rs index 2781a0458..7019a980c 100644 --- a/chain/src/store.rs +++ b/chain/src/store.rs @@ -140,6 +140,13 @@ impl ChainStore { } } + pub fn get_hash_by_height(&self, height: u64) -> Result<Hash, Error> { + option_to_not_found( + self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)), + &format!("Hash at height: {}", height), + ) + } + pub fn get_header_by_height(&self, height: u64) -> Result<BlockHeader, Error> { option_to_not_found( self.db.get_ser(&u64_to_key(HEADER_HEIGHT_PREFIX, height)), diff --git a/chain/src/txhashset.rs b/chain/src/txhashset.rs index 348fc66e8..298d9f974 100644 --- a/chain/src/txhashset.rs +++ b/chain/src/txhashset.rs @@ -1214,8 +1214,8 @@ fn input_pos_to_rewind( head_header: &BlockHeader, batch: &Batch, ) -> Result<Bitmap, Error> { - let mut bitmap = Bitmap::create(); let mut current = head_header.hash(); + let mut height = head_header.height; if head_header.height < block_header.height { debug!( @@ -1224,27 +1224,42 @@ fn input_pos_to_rewind( head_header.height, block_header.height ); - return Ok(bitmap); + return Ok(Bitmap::create()); } - // - // TODO - rework this loop to use Bitmap::fast_or() on a vec of bitmaps. - // - loop { - if current == block_header.hash() { - break; + // Batching up the block input bitmaps, and running fast_or() on every batch of 256 bitmaps. + // so to avoid maintaining a huge vec of bitmaps. + let bitmap_fast_or = |b_res, block_input_bitmaps: &mut Vec<Bitmap>| -> Option<Bitmap> { + if let Some(b) = b_res { + block_input_bitmaps.push(b); + if block_input_bitmaps.len() < 256 { + return None; + } } + let bitmap = Bitmap::fast_or(&block_input_bitmaps.iter().collect::<Vec<&Bitmap>>()); + block_input_bitmaps.clear(); + block_input_bitmaps.push(bitmap.clone()); + Some(bitmap) + }; + let mut block_input_bitmaps: Vec<Bitmap> = vec![]; + let bh = block_header.hash(); + + while current != bh { // We cache recent block headers and block_input_bitmaps // internally in our db layer (commit_index). // I/O should be minimized or eliminated here for most // rewind scenarios. - let current_header = commit_index.get_block_header(¤t)?; - let input_bitmap_res = batch.get_block_input_bitmap(¤t); - if let Ok(b) = input_bitmap_res { - bitmap.or_inplace(&b); + if let Ok(b_res) = batch.get_block_input_bitmap(¤t) { + bitmap_fast_or(Some(b_res), &mut block_input_bitmaps); } - current = current_header.previous; + if height == 0 { + break; + } + height -= 1; + current = commit_index.get_hash_by_height(height)?; } + + let bitmap = bitmap_fast_or(None, &mut block_input_bitmaps).unwrap(); Ok(bitmap) } diff --git a/store/tests/test_bitmap.rs b/store/tests/test_bitmap.rs index 7da718ff9..9ca6cec3c 100644 --- a/store/tests/test_bitmap.rs +++ b/store/tests/test_bitmap.rs @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +extern crate chrono; extern crate croaring; extern crate rand; +use chrono::prelude::Utc; use croaring::Bitmap; use rand::Rng; @@ -97,3 +99,62 @@ fn test_a_big_bitmap() { let serialized_buffer = bitmap.serialize(); assert_eq!(serialized_buffer.len(), 230); } + +#[ignore] +#[test] +fn bench_fast_or() { + let nano_to_millis = 1.0 / 1_000_000.0; + + let bitmaps_number = 256; + let size_of_each_bitmap = 1_000; + + let init_bitmaps = || -> Vec<Bitmap> { + let mut rng = rand::thread_rng(); + let mut bitmaps = vec![]; + for _ in 0..bitmaps_number { + let mut bitmap = Bitmap::create(); + for _ in 0..size_of_each_bitmap { + let n = rng.gen_range(0, 1_000_000); + bitmap.add(n); + } + bitmaps.push(bitmap); + } + bitmaps + }; + + let mut bitmaps = init_bitmaps(); + let mut bitmap = Bitmap::create(); + let start = Utc::now().timestamp_nanos(); + for _ in 0..bitmaps_number { + bitmap.or_inplace(&bitmaps.pop().unwrap()); + } + let fin = Utc::now().timestamp_nanos(); + let dur_ms = (fin - start) as f64 * nano_to_millis; + println!( + " or_inplace(): {:9.3?}ms. bitmap cardinality: {}", + dur_ms, + bitmap.cardinality() + ); + + let bitmaps = init_bitmaps(); + let start = Utc::now().timestamp_nanos(); + let bitmap = Bitmap::fast_or(&bitmaps.iter().map(|x| x).collect::<Vec<&Bitmap>>()); + let fin = Utc::now().timestamp_nanos(); + let dur_ms = (fin - start) as f64 * nano_to_millis; + println!( + " fast_or(): {:9.3?}ms. bitmap cardinality: {}", + dur_ms, + bitmap.cardinality() + ); + + let bitmaps = init_bitmaps(); + let start = Utc::now().timestamp_nanos(); + let bitmap = Bitmap::fast_or_heap(&bitmaps.iter().map(|x| x).collect::<Vec<&Bitmap>>()); + let fin = Utc::now().timestamp_nanos(); + let dur_ms = (fin - start) as f64 * nano_to_millis; + println!( + "fast_or_heap(): {:9.3?}ms. bitmap cardinality: {}", + dur_ms, + bitmap.cardinality() + ); +}