Retire pruned cache (#3573)

* use range beneath subtree for efficient is_pruned check

core/src/core/pmmr/pmmr.rs

@@ -12,8 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::marker;
+use std::{marker, ops::Range, u64};
-use std::u64;
 
 use croaring::Bitmap;
 
@@ -541,10 +540,18 @@ pub fn peak_map_height(mut pos: u64) -> (u64, u64) {
 /// index. This function is the base on which all others, as well as the MMR,
 /// are built.
 pub fn bintree_postorder_height(num: u64) -> u64 {
-	if num == 0 {
+	let mut pos = num.saturating_sub(1);
+	if pos == 0 {
 		return 0;
 	}
-	peak_map_height(num - 1).1
+	let mut peak_size = ALL_ONES >> pos.leading_zeros();
+	while peak_size != 0 {
+		if pos >= peak_size {
+			pos -= peak_size;
+		}
+		peak_size >>= 1;
+	}
+	pos
 }
 
 /// Is this position a leaf in the MMR?
@@ -658,3 +665,10 @@ pub fn bintree_leftmost(num: u64) -> u64 {
 	let height = bintree_postorder_height(num);
 	num + 2 - (2 << height)
 }
+
+/// All pos in the subtree beneath the provided root, including root itself.
+pub fn bintree_range(num: u64) -> Range<u64> {
+	let height = bintree_postorder_height(num);
+	let leftmost = num + 2 - (2 << height);
+	leftmost..(num + 1)
+}

core/tests/pmmr.rs

@@ -90,6 +90,18 @@ fn first_100_mmr_heights() {
 	}
 }
 
+#[test]
+fn test_bintree_range() {
+	assert_eq!(pmmr::bintree_range(0), 0..1);
+	assert_eq!(pmmr::bintree_range(1), 1..2);
+	assert_eq!(pmmr::bintree_range(2), 2..3);
+	assert_eq!(pmmr::bintree_range(3), 1..4);
+	assert_eq!(pmmr::bintree_range(4), 4..5);
+	assert_eq!(pmmr::bintree_range(5), 5..6);
+	assert_eq!(pmmr::bintree_range(6), 4..7);
+	assert_eq!(pmmr::bintree_range(7), 1..8);
+}
+
 // The pos of the rightmost leaf for the provided MMR size (last leaf in subtree).
 #[test]
 fn test_bintree_rightmost() {

store/src/pmmr.rs

@@ -285,6 +285,9 @@ impl<T: PMMRable> PMMRBackend<T> {
 	}
 
 	fn is_compacted(&self, pos: u64) -> bool {
+		if self.leaf_set.includes(pos) {
+			return false;
+		}
 		self.is_pruned(pos) && !self.is_pruned_root(pos)
 	}
 

store/src/prune_list.rs

@@ -25,8 +25,9 @@ use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 
 use croaring::Bitmap;
+use grin_core::core::pmmr;
 
-use crate::core::core::pmmr::{bintree_postorder_height, family, path};
+use crate::core::core::pmmr::{bintree_postorder_height, family};
 use crate::{read_bitmap, save_via_temp_file};
 
 /// Maintains a list of previously pruned nodes in PMMR, compacting the list as
@@ -44,8 +45,6 @@ pub struct PruneList {
 	path: Option<PathBuf>,
 	/// Bitmap representing pruned root node positions.
 	bitmap: Bitmap,
-	/// Bitmap representing all pruned node positions (everything under the pruned roots).
-	pruned_cache: Bitmap,
 	shift_cache: Vec<u64>,
 	leaf_shift_cache: Vec<u64>,
 }
@@ -59,7 +58,6 @@ impl PruneList {
 		PruneList {
 			path,
 			bitmap,
-			pruned_cache: Bitmap::create(),
 			shift_cache: vec![],
 			leaf_shift_cache: vec![],
 		}
@@ -85,11 +83,10 @@ impl PruneList {
 		prune_list.init_caches();
 
 		if !prune_list.bitmap.is_empty() {
-			debug!("bitmap {} pos ({} bytes), pruned_cache {} pos ({} bytes), shift_cache {}, leaf_shift_cache {}",
+			debug!(
+				"bitmap {} pos ({} bytes), shift_cache {}, leaf_shift_cache {}",
 				prune_list.bitmap.cardinality(),
 				prune_list.bitmap.get_serialized_size_in_bytes(),
-				prune_list.pruned_cache.cardinality(),
-				prune_list.pruned_cache.get_serialized_size_in_bytes(),
 				prune_list.shift_cache.len(),
 				prune_list.leaf_shift_cache.len(),
 			);
@@ -102,7 +99,6 @@ impl PruneList {
 	pub fn init_caches(&mut self) {
 		self.build_shift_cache();
 		self.build_leaf_shift_cache();
-		self.build_pruned_cache();
 	}
 
 	/// Save the prune_list to disk.
@@ -232,16 +228,17 @@ impl PruneList {
 	pub fn add(&mut self, pos: u64) {
 		assert!(pos > 0, "prune list 1-indexed, 0 not valid pos");
 
+		if self.is_pruned(pos) {
+			return;
+		}
+
 		let mut current = pos;
 		loop {
 			let (parent, sibling) = family(current);
-
+			if self.is_pruned_root(sibling) {
-			if self.bitmap.contains(sibling as u32) || self.pruned_cache.contains(sibling as u32) {
-				self.pruned_cache.add(current as u32);
 				self.bitmap.remove(sibling as u32);
 				current = parent;
 			} else {
-				self.pruned_cache.add(current as u32);
 				self.bitmap.add(current as u32);
 				break;
 			}
@@ -263,26 +260,21 @@ impl PruneList {
 		self.bitmap.iter().map(|x| x as u64).collect()
 	}
 
-	/// Is the pos pruned?
+	/// A pos is pruned if it is a pruned root directly or if it is
-	/// Assumes the pruned_cache is fully built and up to date.
+	/// beneath the "next" pruned subtree.
+	/// We only need to consider the "next" subtree due to the append-only MMR structure.
 	pub fn is_pruned(&self, pos: u64) -> bool {
 		assert!(pos > 0, "prune list 1-indexed, 0 not valid pos");
-		self.pruned_cache.contains(pos as u32)
+		if self.is_pruned_root(pos) {
-	}
+			return true;
-
-	fn build_pruned_cache(&mut self) {
-		if self.bitmap.is_empty() {
-			return;
 		}
-		let maximum = self.bitmap.maximum().unwrap_or(0);
+		let rank = self.bitmap.rank(pos as u32);
-		self.pruned_cache = Bitmap::create_with_capacity(maximum);
+		if let Some(root) = self.bitmap.select(rank as u32) {
-		for pos in 1..(maximum + 1) {
+			let range = pmmr::bintree_range(root as u64);
-			let pruned = path(pos as u64, maximum as u64).any(|x| self.bitmap.contains(x as u32));
+			range.contains(&pos)
-			if pruned {
+		} else {
-				self.pruned_cache.add(pos as u32)
+			false
-			}
 		}
-		self.pruned_cache.run_optimize();
 	}
 
 	/// Is the specified position a root of a pruned subtree?