From 38e649791957dd34433d57f0248010378f7deeca Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Mon, 4 Nov 2019 15:04:21 +0000 Subject: [PATCH] Retrieve outputs within a block height range (#3103) * add function to retrieve a set of pmmr indices between a given block height range * typo * change APU to just return required indices * change pmmr index retrieval, change new function to only return pmmr indices between blocks --- api/src/handlers/transactions_api.rs | 44 +++++++++++++++++++-- api/src/rest.rs | 2 +- chain/src/chain.rs | 37 +++++++++++++++--- chain/src/txhashset/txhashset.rs | 13 ++++--- core/src/core/pmmr/readonly_pmmr.rs | 27 ++++++------- core/tests/pmmr.rs | 58 ++++++++++++++-------------- 6 files changed, 124 insertions(+), 57 deletions(-) diff --git a/api/src/handlers/transactions_api.rs b/api/src/handlers/transactions_api.rs index 49398faba..2bf5f9a77 100644 --- a/api/src/handlers/transactions_api.rs +++ b/api/src/handlers/transactions_api.rs @@ -35,6 +35,7 @@ use std::sync::Weak; // UTXO traversal:: // GET /v1/txhashset/outputs?start_index=1&max=100 +// GET /v1/txhashset/heightstopmmr?start_height=1&end_height=1000 // // Build a merkle proof for a given pos // GET /v1/txhashset/merkleproof?n=1 @@ -68,14 +69,19 @@ impl TxHashSetHandler { } // allows traversal of utxo set - fn outputs(&self, start_index: u64, mut max: u64) -> Result { + fn outputs( + &self, + start_index: u64, + end_index: Option, + mut max: u64, + ) -> Result { //set a limit here if max > 10_000 { max = 10_000; } let chain = w(&self.chain)?; let outputs = chain - .unspent_outputs_by_insertion_index(start_index, max) + .unspent_outputs_by_pmmr_index(start_index, max, end_index) .context(ErrorKind::NotFound)?; let out = OutputListing { last_retrieved_index: outputs.0, @@ -85,7 +91,25 @@ impl TxHashSetHandler { .iter() .map(|x| OutputPrintable::from_output(x, chain.clone(), None, true, true)) .collect::, _>>() - .context(ErrorKind::Internal("cain error".to_owned()))?, + .context(ErrorKind::Internal("chain error".to_owned()))?, + }; + Ok(out) + } + + // allows traversal of utxo set bounded within a block range + fn block_height_range_to_pmmr_indices( + &self, + start_block_height: u64, + end_block_height: Option, + ) -> Result { + let chain = w(&self.chain)?; + let range = chain + .block_height_range_to_pmmr_indices(start_block_height, end_block_height) + .context(ErrorKind::NotFound)?; + let out = OutputListing { + last_retrieved_index: range.0, + highest_index: range.1, + outputs: vec![], }; Ok(out) } @@ -121,15 +145,27 @@ impl Handler for TxHashSetHandler { let params = QueryParams::from(req.uri().query()); let last_n = parse_param_no_err!(params, "n", 10); let start_index = parse_param_no_err!(params, "start_index", 1); + let end_index = match parse_param_no_err!(params, "end_index", 0) { + 0 => None, + i => Some(i), + }; let max = parse_param_no_err!(params, "max", 100); let id = parse_param_no_err!(params, "id", "".to_owned()); + let start_height = parse_param_no_err!(params, "start_height", 1); + let end_height = match parse_param_no_err!(params, "end_height", 0) { + 0 => None, + h => Some(h), + }; match right_path_element!(req) { "roots" => result_to_response(self.get_roots()), "lastoutputs" => result_to_response(self.get_last_n_output(last_n)), "lastrangeproofs" => result_to_response(self.get_last_n_rangeproof(last_n)), "lastkernels" => result_to_response(self.get_last_n_kernel(last_n)), - "outputs" => result_to_response(self.outputs(start_index, max)), + "outputs" => result_to_response(self.outputs(start_index, end_index, max)), + "heightstopmmr" => result_to_response( + self.block_height_range_to_pmmr_indices(start_height, end_height), + ), "merkleproof" => result_to_response(self.get_merkle_proof_for_output(&id)), _ => response(StatusCode::BAD_REQUEST, ""), } diff --git a/api/src/rest.rs b/api/src/rest.rs index 9e4e6e307..6789d148f 100644 --- a/api/src/rest.rs +++ b/api/src/rest.rs @@ -247,7 +247,7 @@ impl ApiServer { // TODO re-enable stop after investigation //let tx = mem::replace(&mut self.shutdown_sender, None).unwrap(); //tx.send(()).expect("Failed to stop API server"); - info!("API server has been stoped"); + info!("API server has been stopped"); true } else { error!("Can't stop API server, it's not running or doesn't spport stop operation"); diff --git a/chain/src/chain.rs b/chain/src/chain.rs index a484f24c0..5d09d564d 100644 --- a/chain/src/chain.rs +++ b/chain/src/chain.rs @@ -1146,15 +1146,20 @@ impl Chain { } /// outputs by insertion index - pub fn unspent_outputs_by_insertion_index( + pub fn unspent_outputs_by_pmmr_index( &self, start_index: u64, - max: u64, + max_count: u64, + max_pmmr_index: Option, ) -> Result<(u64, u64, Vec), Error> { let txhashset = self.txhashset.read(); - let max_index = txhashset.highest_output_insertion_index(); - let outputs = txhashset.outputs_by_insertion_index(start_index, max); - let rangeproofs = txhashset.rangeproofs_by_insertion_index(start_index, max); + let last_index = match max_pmmr_index { + Some(i) => i, + None => txhashset.highest_output_insertion_index(), + }; + let outputs = txhashset.outputs_by_pmmr_index(start_index, max_count, max_pmmr_index); + let rangeproofs = + txhashset.rangeproofs_by_pmmr_index(start_index, max_count, max_pmmr_index); if outputs.0 != rangeproofs.0 || outputs.1.len() != rangeproofs.1.len() { return Err(ErrorKind::TxHashSetErr(String::from( "Output and rangeproof sets don't match", @@ -1169,7 +1174,27 @@ impl Chain { proof: y, }); } - Ok((outputs.0, max_index, output_vec)) + Ok((outputs.0, last_index, output_vec)) + } + + /// Return unspent outputs as above, but bounded between a particular range of blocks + pub fn block_height_range_to_pmmr_indices( + &self, + start_block_height: u64, + end_block_height: Option, + ) -> Result<(u64, u64), Error> { + let end_block_height = match end_block_height { + Some(h) => h, + None => self.head_header()?.height, + }; + // Return headers at the given heights + let prev_to_start_header = + self.get_header_by_height(start_block_height.saturating_sub(1))?; + let end_header = self.get_header_by_height(end_block_height)?; + Ok(( + prev_to_start_header.output_mmr_size + 1, + end_header.output_mmr_size, + )) } /// Orphans pool size diff --git a/chain/src/txhashset/txhashset.rs b/chain/src/txhashset/txhashset.rs index 0cdc906a9..68d233d52 100644 --- a/chain/src/txhashset/txhashset.rs +++ b/chain/src/txhashset/txhashset.rs @@ -267,15 +267,17 @@ impl TxHashSet { Ok(self.commit_index.get_all_output_pos()?) } - /// returns outputs from the given insertion (leaf) index up to the + /// returns outputs from the given pmmr index up to the /// specified limit. Also returns the last index actually populated - pub fn outputs_by_insertion_index( + /// max index is the last PMMR index to consider, not leaf index + pub fn outputs_by_pmmr_index( &self, start_index: u64, max_count: u64, + max_index: Option, ) -> (u64, Vec) { ReadonlyPMMR::at(&self.output_pmmr_h.backend, self.output_pmmr_h.last_pos) - .elements_from_insertion_index(start_index, max_count) + .elements_from_pmmr_index(start_index, max_count, max_index) } /// highest output insertion index available @@ -284,13 +286,14 @@ impl TxHashSet { } /// As above, for rangeproofs - pub fn rangeproofs_by_insertion_index( + pub fn rangeproofs_by_pmmr_index( &self, start_index: u64, max_count: u64, + max_index: Option, ) -> (u64, Vec) { ReadonlyPMMR::at(&self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos) - .elements_from_insertion_index(start_index, max_count) + .elements_from_pmmr_index(start_index, max_count, max_index) } /// Find a kernel with a given excess. Work backwards from `max_index` to `min_index` diff --git a/core/src/core/pmmr/readonly_pmmr.rs b/core/src/core/pmmr/readonly_pmmr.rs index 402826ed1..e4eaee864 100644 --- a/core/src/core/pmmr/readonly_pmmr.rs +++ b/core/src/core/pmmr/readonly_pmmr.rs @@ -17,7 +17,7 @@ use std::marker; use crate::core::hash::{Hash, ZERO_HASH}; -use crate::core::pmmr::pmmr::{bintree_rightmost, insertion_to_pmmr_index, peaks}; +use crate::core::pmmr::pmmr::{bintree_rightmost, peaks}; use crate::core::pmmr::{is_leaf, Backend}; use crate::ser::{PMMRIndexHashable, PMMRable}; @@ -133,27 +133,28 @@ where /// Helper function which returns un-pruned nodes from the insertion index /// forward - /// returns last insertion index returned along with data - pub fn elements_from_insertion_index( + /// returns last pmmr index returned along with data + pub fn elements_from_pmmr_index( &self, - mut index: u64, + mut pmmr_index: u64, max_count: u64, + max_pmmr_pos: Option, ) -> (u64, Vec) { let mut return_vec = vec![]; - if index == 0 { - index = 1; + let last_pos = match max_pmmr_pos { + Some(p) => p, + None => self.last_pos, + }; + if pmmr_index == 0 { + pmmr_index = 1; } - let mut return_index = index; - let mut pmmr_index = insertion_to_pmmr_index(index); - while return_vec.len() < max_count as usize && pmmr_index <= self.last_pos { + while return_vec.len() < max_count as usize && pmmr_index <= last_pos { if let Some(t) = self.get_data(pmmr_index) { return_vec.push(t); - return_index = index; } - index += 1; - pmmr_index = insertion_to_pmmr_index(index); + pmmr_index += 1; } - (return_index, return_vec) + (pmmr_index.saturating_sub(1), return_vec) } /// Helper function to get the last N nodes inserted, i.e. the last diff --git a/core/tests/pmmr.rs b/core/tests/pmmr.rs index 6357dcdf4..74d17559f 100644 --- a/core/tests/pmmr.rs +++ b/core/tests/pmmr.rs @@ -514,46 +514,48 @@ fn check_insertion_to_pmmr_index() { } #[test] -fn check_elements_from_insertion_index() { +fn check_elements_from_pmmr_index() { let mut ba = VecBackend::new(); let mut pmmr = PMMR::new(&mut ba); - for x in 1..1000 { + // 20 elements should give max index 38 + for x in 1..21 { pmmr.push(&TestElem([0, 0, 0, x])).unwrap(); } + // Normal case - let res = pmmr.readonly_pmmr().elements_from_insertion_index(1, 100); - assert_eq!(res.0, 100); - assert_eq!(res.1.len(), 100); + let res = pmmr.readonly_pmmr().elements_from_pmmr_index(1, 1000, None); + assert_eq!(res.0, 38); + assert_eq!(res.1.len(), 20); assert_eq!(res.1[0].0[3], 1); - assert_eq!(res.1[99].0[3], 100); + assert_eq!(res.1[19].0[3], 20); // middle of pack - let res = pmmr.readonly_pmmr().elements_from_insertion_index(351, 70); - assert_eq!(res.0, 420); - assert_eq!(res.1.len(), 70); - assert_eq!(res.1[0].0[3], 351); - assert_eq!(res.1[69].0[3], 420); - - // past the end let res = pmmr .readonly_pmmr() - .elements_from_insertion_index(650, 1000); - assert_eq!(res.0, 999); - assert_eq!(res.1.len(), 350); - assert_eq!(res.1[0].0[3], 650); - assert_eq!(res.1[349].0[3], 999); + .elements_from_pmmr_index(8, 1000, Some(34)); + assert_eq!(res.0, 34); + assert_eq!(res.1.len(), 14); + assert_eq!(res.1[0].0[3], 5); + assert_eq!(res.1[13].0[3], 18); + + // bounded + let res = pmmr + .readonly_pmmr() + .elements_from_pmmr_index(8, 7, Some(34)); + assert_eq!(res.0, 19); + assert_eq!(res.1.len(), 7); + assert_eq!(res.1[0].0[3], 5); + assert_eq!(res.1[6].0[3], 11); // pruning a few nodes should get consistent results - pmmr.prune(pmmr::insertion_to_pmmr_index(650)).unwrap(); - pmmr.prune(pmmr::insertion_to_pmmr_index(651)).unwrap(); - pmmr.prune(pmmr::insertion_to_pmmr_index(800)).unwrap(); - pmmr.prune(pmmr::insertion_to_pmmr_index(900)).unwrap(); - pmmr.prune(pmmr::insertion_to_pmmr_index(998)).unwrap(); + pmmr.prune(pmmr::insertion_to_pmmr_index(5)).unwrap(); + pmmr.prune(pmmr::insertion_to_pmmr_index(20)).unwrap(); + let res = pmmr .readonly_pmmr() - .elements_from_insertion_index(650, 1000); - assert_eq!(res.0, 999); - assert_eq!(res.1.len(), 345); - assert_eq!(res.1[0].0[3], 652); - assert_eq!(res.1[344].0[3], 999); + .elements_from_pmmr_index(8, 7, Some(34)); + assert_eq!(res.0, 20); + assert_eq!(res.1.len(), 7); + assert_eq!(res.1[0].0[3], 6); + assert_eq!(res.1[6].0[3], 12); }