Retrieve outputs within a block height range (#3103)

* add function to retrieve a set of pmmr indices between a given block height range

* typo

* change APU to just return required indices

* change pmmr index retrieval, change new function to only return pmmr indices between blocks

api/src/handlers/transactions_api.rs

@@ -35,6 +35,7 @@ use std::sync::Weak;
 
 // UTXO traversal::
 // GET /v1/txhashset/outputs?start_index=1&max=100
+// GET /v1/txhashset/heightstopmmr?start_height=1&end_height=1000
 //
 // Build a merkle proof for a given pos
 // GET /v1/txhashset/merkleproof?n=1
@@ -68,14 +69,19 @@ impl TxHashSetHandler {
 	}
 
 	// allows traversal of utxo set
-	fn outputs(&self, start_index: u64, mut max: u64) -> Result<OutputListing, Error> {
+	fn outputs(
+		&self,
+		start_index: u64,
+		end_index: Option<u64>,
+		mut max: u64,
+	) -> Result<OutputListing, Error> {
 		//set a limit here
 		if max > 10_000 {
 			max = 10_000;
 		}
 		let chain = w(&self.chain)?;
 		let outputs = chain
-			.unspent_outputs_by_insertion_index(start_index, max)
+			.unspent_outputs_by_pmmr_index(start_index, max, end_index)
 			.context(ErrorKind::NotFound)?;
 		let out = OutputListing {
 			last_retrieved_index: outputs.0,
@@ -85,7 +91,25 @@ impl TxHashSetHandler {
 				.iter()
 				.map(|x| OutputPrintable::from_output(x, chain.clone(), None, true, true))
 				.collect::<Result<Vec<_>, _>>()
-				.context(ErrorKind::Internal("cain error".to_owned()))?,
+				.context(ErrorKind::Internal("chain error".to_owned()))?,
+		};
+		Ok(out)
+	}
+
+	// allows traversal of utxo set bounded within a block range
+	fn block_height_range_to_pmmr_indices(
+		&self,
+		start_block_height: u64,
+		end_block_height: Option<u64>,
+	) -> Result<OutputListing, Error> {
+		let chain = w(&self.chain)?;
+		let range = chain
+			.block_height_range_to_pmmr_indices(start_block_height, end_block_height)
+			.context(ErrorKind::NotFound)?;
+		let out = OutputListing {
+			last_retrieved_index: range.0,
+			highest_index: range.1,
+			outputs: vec![],
 		};
 		Ok(out)
 	}
@@ -121,15 +145,27 @@ impl Handler for TxHashSetHandler {
 		let params = QueryParams::from(req.uri().query());
 		let last_n = parse_param_no_err!(params, "n", 10);
 		let start_index = parse_param_no_err!(params, "start_index", 1);
+		let end_index = match parse_param_no_err!(params, "end_index", 0) {
+			0 => None,
+			i => Some(i),
+		};
 		let max = parse_param_no_err!(params, "max", 100);
 		let id = parse_param_no_err!(params, "id", "".to_owned());
+		let start_height = parse_param_no_err!(params, "start_height", 1);
+		let end_height = match parse_param_no_err!(params, "end_height", 0) {
+			0 => None,
+			h => Some(h),
+		};
 
 		match right_path_element!(req) {
 			"roots" => result_to_response(self.get_roots()),
 			"lastoutputs" => result_to_response(self.get_last_n_output(last_n)),
 			"lastrangeproofs" => result_to_response(self.get_last_n_rangeproof(last_n)),
 			"lastkernels" => result_to_response(self.get_last_n_kernel(last_n)),
-			"outputs" => result_to_response(self.outputs(start_index, max)),
+			"outputs" => result_to_response(self.outputs(start_index, end_index, max)),
+			"heightstopmmr" => result_to_response(
+				self.block_height_range_to_pmmr_indices(start_height, end_height),
+			),
 			"merkleproof" => result_to_response(self.get_merkle_proof_for_output(&id)),
 			_ => response(StatusCode::BAD_REQUEST, ""),
 		}

api/src/rest.rs

@@ -247,7 +247,7 @@ impl ApiServer {
 			// TODO re-enable stop after investigation
 			//let tx = mem::replace(&mut self.shutdown_sender, None).unwrap();
 			//tx.send(()).expect("Failed to stop API server");
-			info!("API server has been stoped");
+			info!("API server has been stopped");
 			true
 		} else {
 			error!("Can't stop API server, it's not running or doesn't spport stop operation");

chain/src/chain.rs

@@ -1146,15 +1146,20 @@ impl Chain {
 	}
 
 	/// outputs by insertion index
-	pub fn unspent_outputs_by_insertion_index(
+	pub fn unspent_outputs_by_pmmr_index(
 		&self,
 		start_index: u64,
-		max: u64,
+		max_count: u64,
+		max_pmmr_index: Option<u64>,
 	) -> Result<(u64, u64, Vec<Output>), Error> {
 		let txhashset = self.txhashset.read();
-		let max_index = txhashset.highest_output_insertion_index();
+		let last_index = match max_pmmr_index {
-		let outputs = txhashset.outputs_by_insertion_index(start_index, max);
+			Some(i) => i,
-		let rangeproofs = txhashset.rangeproofs_by_insertion_index(start_index, max);
+			None => txhashset.highest_output_insertion_index(),
+		};
+		let outputs = txhashset.outputs_by_pmmr_index(start_index, max_count, max_pmmr_index);
+		let rangeproofs =
+			txhashset.rangeproofs_by_pmmr_index(start_index, max_count, max_pmmr_index);
 		if outputs.0 != rangeproofs.0 || outputs.1.len() != rangeproofs.1.len() {
 			return Err(ErrorKind::TxHashSetErr(String::from(
 				"Output and rangeproof sets don't match",
@@ -1169,7 +1174,27 @@ impl Chain {
 				proof: y,
 			});
 		}
-		Ok((outputs.0, max_index, output_vec))
+		Ok((outputs.0, last_index, output_vec))
+	}
+
+	/// Return unspent outputs as above, but bounded between a particular range of blocks
+	pub fn block_height_range_to_pmmr_indices(
+		&self,
+		start_block_height: u64,
+		end_block_height: Option<u64>,
+	) -> Result<(u64, u64), Error> {
+		let end_block_height = match end_block_height {
+			Some(h) => h,
+			None => self.head_header()?.height,
+		};
+		// Return headers at the given heights
+		let prev_to_start_header =
+			self.get_header_by_height(start_block_height.saturating_sub(1))?;
+		let end_header = self.get_header_by_height(end_block_height)?;
+		Ok((
+			prev_to_start_header.output_mmr_size + 1,
+			end_header.output_mmr_size,
+		))
 	}
 
 	/// Orphans pool size

chain/src/txhashset/txhashset.rs

@@ -267,15 +267,17 @@ impl TxHashSet {
 		Ok(self.commit_index.get_all_output_pos()?)
 	}
 
-	/// returns outputs from the given insertion (leaf) index up to the
+	/// returns outputs from the given pmmr index up to the
 	/// specified limit. Also returns the last index actually populated
-	pub fn outputs_by_insertion_index(
+	/// max index is the last PMMR index to consider, not leaf index
+	pub fn outputs_by_pmmr_index(
 		&self,
 		start_index: u64,
 		max_count: u64,
+		max_index: Option<u64>,
 	) -> (u64, Vec<OutputIdentifier>) {
 		ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos)
-			.elements_from_insertion_index(start_index, max_count)
+			.elements_from_pmmr_index(start_index, max_count, max_index)
 	}
 
 	/// highest output insertion index available
@@ -284,13 +286,14 @@ impl TxHashSet {
 	}
 
 	/// As above, for rangeproofs
-	pub fn rangeproofs_by_insertion_index(
+	pub fn rangeproofs_by_pmmr_index(
 		&self,
 		start_index: u64,
 		max_count: u64,
+		max_index: Option<u64>,
 	) -> (u64, Vec<RangeProof>) {
 		ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos)
-			.elements_from_insertion_index(start_index, max_count)
+			.elements_from_pmmr_index(start_index, max_count, max_index)
 	}
 
 	/// Find a kernel with a given excess. Work backwards from `max_index` to `min_index`

core/src/core/pmmr/readonly_pmmr.rs

@@ -17,7 +17,7 @@
 use std::marker;
 
 use crate::core::hash::{Hash, ZERO_HASH};
-use crate::core::pmmr::pmmr::{bintree_rightmost, insertion_to_pmmr_index, peaks};
+use crate::core::pmmr::pmmr::{bintree_rightmost, peaks};
 use crate::core::pmmr::{is_leaf, Backend};
 use crate::ser::{PMMRIndexHashable, PMMRable};
 
@@ -133,27 +133,28 @@ where
 
 	/// Helper function which returns un-pruned nodes from the insertion index
 	/// forward
-	/// returns last insertion index returned along with data
+	/// returns last pmmr index returned along with data
-	pub fn elements_from_insertion_index(
+	pub fn elements_from_pmmr_index(
 		&self,
-		mut index: u64,
+		mut pmmr_index: u64,
 		max_count: u64,
+		max_pmmr_pos: Option<u64>,
 	) -> (u64, Vec<T::E>) {
 		let mut return_vec = vec![];
-		if index == 0 {
+		let last_pos = match max_pmmr_pos {
-			index = 1;
+			Some(p) => p,
+			None => self.last_pos,
+		};
+		if pmmr_index == 0 {
+			pmmr_index = 1;
 		}
-		let mut return_index = index;
+		while return_vec.len() < max_count as usize && pmmr_index <= last_pos {
-		let mut pmmr_index = insertion_to_pmmr_index(index);
-		while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos {
 			if let Some(t) = self.get_data(pmmr_index) {
 				return_vec.push(t);
-				return_index = index;
 			}
-			index += 1;
+			pmmr_index += 1;
-			pmmr_index = insertion_to_pmmr_index(index);
 		}
-		(return_index, return_vec)
+		(pmmr_index.saturating_sub(1), return_vec)
 	}
 
 	/// Helper function to get the last N nodes inserted, i.e. the last

core/tests/pmmr.rs

@@ -514,46 +514,48 @@ fn check_insertion_to_pmmr_index() {
 }
 
 #[test]
-fn check_elements_from_insertion_index() {
+fn check_elements_from_pmmr_index() {
 	let mut ba = VecBackend::new();
 	let mut pmmr = PMMR::new(&mut ba);
-	for x in 1..1000 {
+	// 20 elements should give max index 38
+	for x in 1..21 {
 		pmmr.push(&TestElem([0, 0, 0, x])).unwrap();
 	}
+
 	// Normal case
-	let res = pmmr.readonly_pmmr().elements_from_insertion_index(1, 100);
+	let res = pmmr.readonly_pmmr().elements_from_pmmr_index(1, 1000, None);
-	assert_eq!(res.0, 100);
+	assert_eq!(res.0, 38);
-	assert_eq!(res.1.len(), 100);
+	assert_eq!(res.1.len(), 20);
 	assert_eq!(res.1[0].0[3], 1);
-	assert_eq!(res.1[99].0[3], 100);
+	assert_eq!(res.1[19].0[3], 20);
 
 	// middle of pack
-	let res = pmmr.readonly_pmmr().elements_from_insertion_index(351, 70);
-	assert_eq!(res.0, 420);
-	assert_eq!(res.1.len(), 70);
-	assert_eq!(res.1[0].0[3], 351);
-	assert_eq!(res.1[69].0[3], 420);
-
-	// past the end
 	let res = pmmr
 		.readonly_pmmr()
-		.elements_from_insertion_index(650, 1000);
+		.elements_from_pmmr_index(8, 1000, Some(34));
-	assert_eq!(res.0, 999);
+	assert_eq!(res.0, 34);
-	assert_eq!(res.1.len(), 350);
+	assert_eq!(res.1.len(), 14);
-	assert_eq!(res.1[0].0[3], 650);
+	assert_eq!(res.1[0].0[3], 5);
-	assert_eq!(res.1[349].0[3], 999);
+	assert_eq!(res.1[13].0[3], 18);
+
+	// bounded
+	let res = pmmr
+		.readonly_pmmr()
+		.elements_from_pmmr_index(8, 7, Some(34));
+	assert_eq!(res.0, 19);
+	assert_eq!(res.1.len(), 7);
+	assert_eq!(res.1[0].0[3], 5);
+	assert_eq!(res.1[6].0[3], 11);
 
 	// pruning a few nodes should get consistent results
-	pmmr.prune(pmmr::insertion_to_pmmr_index(650)).unwrap();
+	pmmr.prune(pmmr::insertion_to_pmmr_index(5)).unwrap();
-	pmmr.prune(pmmr::insertion_to_pmmr_index(651)).unwrap();
+	pmmr.prune(pmmr::insertion_to_pmmr_index(20)).unwrap();
-	pmmr.prune(pmmr::insertion_to_pmmr_index(800)).unwrap();
+
-	pmmr.prune(pmmr::insertion_to_pmmr_index(900)).unwrap();
-	pmmr.prune(pmmr::insertion_to_pmmr_index(998)).unwrap();
 	let res = pmmr
 		.readonly_pmmr()
-		.elements_from_insertion_index(650, 1000);
+		.elements_from_pmmr_index(8, 7, Some(34));
-	assert_eq!(res.0, 999);
+	assert_eq!(res.0, 20);
-	assert_eq!(res.1.len(), 345);
+	assert_eq!(res.1.len(), 7);
-	assert_eq!(res.1[0].0[3], 652);
+	assert_eq!(res.1[0].0[3], 6);
-	assert_eq!(res.1[344].0[3], 999);
+	assert_eq!(res.1[6].0[3], 12);
 }