Sum tree and improved chain API Endpoints (#214)

* adding more useful handlers
* added method to return last n leaf nodes inserted into the sum tree
* endpoints in place for getting last n sumtree nodes

api/src/endpoints.rs

@@ -20,38 +20,12 @@ use chain;
 use core::core::Transaction;
 use core::ser;
 use pool;
-use handlers::UtxoHandler;
+use handlers::{UtxoHandler, ChainHandler, SumTreeHandler};
 use rest::*;
 use types::*;
 use util;
 use util::LOGGER;
 
-/// ApiEndpoint implementation for the blockchain. Exposes the current chain
-/// state as a simple JSON object.
-#[derive(Clone)]
-pub struct ChainApi {
-	/// data store access
-	chain: Arc<chain::Chain>,
-}
-
-impl ApiEndpoint for ChainApi {
-	type ID = String;
-	type T = Tip;
-	type OP_IN = ();
-	type OP_OUT = ();
-
-	fn operations(&self) -> Vec<Operation> {
-		vec![Operation::Get]
-	}
-
-	fn get(&self, _: String) -> ApiResult<Tip> {
-		match self.chain.head() {
-			Ok(tip) => Ok(Tip::from_tip(tip)),
-			Err(e) => Err(Error::Internal(format!("{:?}", e))),
-		}
-	}
-}
-
 /// ApiEndpoint implementation for the transaction pool, to check its status
 /// and size as well as push new transactions.
 #[derive(Clone)]
@@ -132,14 +106,17 @@ pub fn start_rest_apis<T>(
 
 	thread::spawn(move || {
 		let mut apis = ApiServer::new("/v1".to_string());
-		apis.register_endpoint("/chain".to_string(), ChainApi {chain: chain.clone()});
 		apis.register_endpoint("/pool".to_string(), PoolApi {tx_pool: tx_pool});
 
 		// register a nested router at "/v2" for flexibility
 		// so we can experiment with raw iron handlers
 		let utxo_handler = UtxoHandler {chain: chain.clone()};
+		let chain_tip_handler = ChainHandler {chain: chain.clone()};
+		let sumtree_handler = SumTreeHandler {chain: chain.clone()};
 		let router = router!(
+			chain_tip: get "/chain" => chain_tip_handler,
 			chain_utxos: get "/chain/utxos" => utxo_handler,
+			sumtree_roots: get "/sumtrees/*" => sumtree_handler,
 		);
 		apis.register_handler("/v2", router);
 

api/src/handlers.rs

@@ -85,3 +85,108 @@ impl Handler for UtxoHandler {
 		}
 	}
 }
+
+// Sum tree handler
+
+pub struct SumTreeHandler {
+	pub chain: Arc<chain::Chain>,
+}
+
+impl SumTreeHandler {
+	//gets roots
+	fn get_roots(&self) -> SumTrees {
+		SumTrees::from_head(self.chain.clone())
+	}
+
+	// gets last n utxos inserted in to the tree
+	fn get_last_n_utxo(&self, distance:u64) -> Vec<SumTreeNode> {
+		SumTreeNode::get_last_n_utxo(self.chain.clone(), distance)
+	}
+
+	// gets last n utxos inserted in to the tree
+	fn get_last_n_rangeproof(&self, distance:u64) -> Vec<SumTreeNode> {
+		SumTreeNode::get_last_n_rangeproof(self.chain.clone(), distance)
+	}
+
+	// gets last n utxos inserted in to the tree
+	fn get_last_n_kernel(&self, distance:u64) -> Vec<SumTreeNode> {
+		SumTreeNode::get_last_n_kernel(self.chain.clone(), distance)
+	}
+
+}
+
+//
+// Retrieve the roots:
+// GET /v2/sumtrees/roots
+//
+// Last inserted nodes::
+// GET /v2/sumtrees/lastutxos (gets last 10)
+// GET /v2/sumtrees/lastutxos?n=5
+// GET /v2/sumtrees/lastrangeproofs
+// GET /v2/sumtrees/lastkernels
+//
+
+impl Handler for SumTreeHandler {
+	fn handle(&self, req: &mut Request) -> IronResult<Response> {
+		let url = req.url.clone();
+		let mut path_elems = url.path();
+		if *path_elems.last().unwrap() == "" {
+			path_elems.pop();
+		}
+		//TODO: probably need to set a reasonable max limit here
+		let mut last_n=10;
+		if let Ok(params) = req.get_ref::<UrlEncodedQuery>() {
+			if let Some(nums) = params.get("n") {
+				for num in nums {
+					if let Ok(n) = str::parse(num) {
+						last_n=n;
+					}
+				}
+			}
+		}
+		match *path_elems.last().unwrap(){
+			"roots" => match serde_json::to_string_pretty(&self.get_roots()) {
+				Ok(json) => Ok(Response::with((status::Ok, json))),
+				Err(_) => Ok(Response::with((status::BadRequest, ""))),
+			},
+			"lastutxos" => match serde_json::to_string_pretty(&self.get_last_n_utxo(last_n)) {
+				Ok(json) => Ok(Response::with((status::Ok, json))),
+				Err(_) => Ok(Response::with((status::BadRequest, ""))),
+			},
+			"lastrangeproofs" => match serde_json::to_string_pretty(&self.get_last_n_rangeproof(last_n)) {
+				Ok(json) => Ok(Response::with((status::Ok, json))),
+				Err(_) => Ok(Response::with((status::BadRequest, ""))),
+			},
+			"lastkernels" => match serde_json::to_string_pretty(&self.get_last_n_kernel(last_n)) {
+				Ok(json) => Ok(Response::with((status::Ok, json))),
+				Err(_) => Ok(Response::with((status::BadRequest, ""))),
+			},_ => Ok(Response::with((status::BadRequest, "")))
+		}
+	}
+}
+
+// Chain Handler
+
+pub struct ChainHandler {
+	pub chain: Arc<chain::Chain>,
+}
+
+impl ChainHandler {
+	fn get_tip(&self) -> Tip {
+		Tip::from_tip(self.chain.head().unwrap())
+	}
+}
+
+//
+// Get the head details
+// GET /v2/chain
+//
+
+impl Handler for ChainHandler {
+	fn handle(&self, _req: &mut Request) -> IronResult<Response> {
+		match serde_json::to_string_pretty(&self.get_tip()) {
+			Ok(json) => Ok(Response::with((status::Ok, json))),
+			Err(_) => Ok(Response::with((status::BadRequest, ""))),
+		}
+	}
+}

api/src/types.rs

@@ -12,25 +12,117 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
 use core::{core, global};
+use core::core::hash::Hashed;
 use chain;
 use secp::pedersen;
+use rest::*;
+use util;
 
+/// The state of the current fork tip
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Tip {
 	/// Height of the tip (max height of the fork)
 	pub height: u64,
 	// Last block pushed to the fork
-	// pub last_block_h: Hash,
+	pub last_block_pushed: String,
 	// Block previous to last
-	// pub prev_block_h: Hash,
+	pub prev_block_to_last: String,
 	// Total difficulty accumulated on that fork
-	// pub total_difficulty: Difficulty,
+	pub total_difficulty: u64,
 }
 
 impl Tip {
 	pub fn from_tip(tip: chain::Tip) -> Tip {
-		Tip { height: tip.height }
+		Tip {
+			height: tip.height,
+			last_block_pushed: util::to_hex(tip.last_block_h.to_vec()),
+			prev_block_to_last: util::to_hex(tip.prev_block_h.to_vec()),
+			total_difficulty: tip.total_difficulty.into_num(),
+		}
+	}
+}
+
+/// Sumtrees 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SumTrees {
+	/// UTXO Root Hash
+	pub utxo_root_hash: String,
+	// UTXO Root Sum
+	pub utxo_root_sum: String,
+	// Rangeproof root hash
+	pub range_proof_root_hash: String,
+	// Kernel set root hash
+	pub kernel_root_hash: String,
+}
+
+impl SumTrees {
+	pub fn from_head(head: Arc<chain::Chain>) -> SumTrees {
+		let roots=head.get_sumtree_roots();
+		SumTrees {
+			utxo_root_hash: util::to_hex(roots.0.hash.to_vec()),
+			utxo_root_sum: util::to_hex(roots.0.sum.commit.0.to_vec()),
+			range_proof_root_hash: util::to_hex(roots.1.hash.to_vec()),
+			kernel_root_hash: util::to_hex(roots.2.hash.to_vec()),
+		}
+	}
+}
+
+/// Wrapper around a list of sumtree nodes, so it can be
+/// presented properly via json
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SumTreeNode {
+	// The hash
+	pub hash: String,
+	// Output (if included)
+	pub output: Option<OutputPrintable>,
+}
+
+impl SumTreeNode {
+
+	pub fn get_last_n_utxo(chain: Arc<chain::Chain>, distance:u64) -> Vec<SumTreeNode> {
+		let mut return_vec = Vec::new();
+		let last_n = chain.get_last_n_utxo(distance);
+		for elem_output in last_n {
+			let header = chain
+			.get_block_header_by_output_commit(&elem_output.1.commit)
+			.map_err(|_| Error::NotFound);
+			// Need to call further method to check if output is spent
+			let mut output = OutputPrintable::from_output(&elem_output.1, &header.unwrap());
+			if let Ok(_) = chain.get_unspent(&elem_output.1.commit) {
+				output.spent = false;
+			}
+			return_vec.push(SumTreeNode {
+				hash: util::to_hex(elem_output.0.to_vec()),
+				output: Some(output),
+			});
+		}
+		return_vec
+	}
+
+	pub fn get_last_n_rangeproof(head: Arc<chain::Chain>, distance:u64) -> Vec<SumTreeNode> {
+		let mut return_vec = Vec::new();
+		let last_n = head.get_last_n_rangeproof(distance);
+		for elem in last_n {
+			return_vec.push(SumTreeNode {
+				hash: util::to_hex(elem.hash.to_vec()),
+				output: None,
+			});
+		}
+		return_vec
+	}
+
+	pub fn get_last_n_kernel(head: Arc<chain::Chain>, distance:u64) -> Vec<SumTreeNode> {
+		let mut return_vec = Vec::new();
+		let last_n = head.get_last_n_kernel(distance);
+		for elem in last_n {
+			return_vec.push(SumTreeNode {
+				hash: util::to_hex(elem.hash.to_vec()),
+				output: None,
+			});
+		}
+		return_vec
 	}
 }
 
@@ -73,6 +165,42 @@ impl Output {
 	}
 }
 
+//As above, except formatted a bit better for human viewing
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct OutputPrintable {
+	/// The type of output Coinbase|Transaction
+	pub output_type: OutputType,
+	/// The homomorphic commitment representing the output's amount (as hex string)
+	pub commit: String,
+	/// The height of the block creating this output
+	pub height: u64,
+	/// The lock height (earliest block this output can be spent)
+	pub lock_height: u64,
+	/// Whether the output has been spent
+	pub spent: bool,
+	/// Rangeproof hash  (as hex string)
+	pub proof_hash: String,
+}
+
+impl OutputPrintable {
+	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader) -> OutputPrintable {
+		let (output_type, lock_height) = match output.features {
+			x if x.contains(core::transaction::COINBASE_OUTPUT) => {
+				(OutputType::Coinbase, block_header.height + global::coinbase_maturity())
+			}
+			_ => (OutputType::Transaction, 0),
+		};
+		OutputPrintable {
+			output_type: output_type,
+			commit: util::to_hex(output.commit.0.to_vec()),
+			height: block_header.height,
+			lock_height: lock_height,
+			spent: true,
+			proof_hash: util::to_hex(output.proof.hash().to_vec()),
+		}
+	}
+}
+
 #[derive(Serialize, Deserialize)]
 pub struct PoolInfo {
 	/// Size of the pool

chain/src/chain.rs

@@ -18,9 +18,12 @@
 use std::collections::VecDeque;
 use std::sync::{Arc, Mutex, RwLock};
 
-use secp::pedersen::Commitment;
+use secp::pedersen::{Commitment, RangeProof};
 
-use core::core::{Block, BlockHeader, Output};
+use core::core::{SumCommit};
+use core::core::pmmr::{NoSum, HashSum};
+
+use core::core::{Block, BlockHeader, Output, TxKernel};
 use core::core::target::Difficulty;
 use core::core::hash::Hash;
 use grin_store::Error::NotFoundErr;
@@ -250,6 +253,40 @@ impl Chain {
 		Ok(())
 	}
 
+	/// returs sumtree roots
+	pub fn get_sumtree_roots(&self) -> (HashSum<SumCommit>,
+		HashSum<NoSum<RangeProof>>,
+		HashSum<NoSum<TxKernel>>) {
+		let mut sumtrees = self.sumtrees.write().unwrap();
+		sumtrees.roots()
+	}
+
+	/// returns the last n nodes inserted into the utxo sum tree
+	/// returns sum tree hash plus output itself (as the sum is contained
+	/// in the output anyhow)
+	pub fn get_last_n_utxo(&self, distance: u64) -> Vec<(Hash, Output)>{
+		let mut sumtrees = self.sumtrees.write().unwrap();
+		let mut return_vec = Vec::new();
+		let sum_nodes=sumtrees.last_n_utxo(distance);
+		for sum_commit in sum_nodes {
+			let output = self.store.get_output_by_commit(&sum_commit.sum.commit);
+			return_vec.push((sum_commit.hash, output.unwrap()));
+		}
+		return_vec
+	}
+
+	/// as above, for rangeproofs
+	pub fn get_last_n_rangeproof(&self, distance: u64) -> Vec<HashSum<NoSum<RangeProof>>>{
+		let mut sumtrees = self.sumtrees.write().unwrap();
+		sumtrees.last_n_rangeproof(distance)
+	}
+
+	/// as above, for kernels
+	pub fn get_last_n_kernel(&self, distance: u64) -> Vec<HashSum<NoSum<TxKernel>>>{
+		let mut sumtrees = self.sumtrees.write().unwrap();
+		sumtrees.last_n_kernel(distance)
+	}
+
 	/// Total difficulty at the head of the chain
 	pub fn total_difficulty(&self) -> Difficulty {
 		self.head.lock().unwrap().clone().total_difficulty

chain/src/sumtree.rs

@@ -23,7 +23,7 @@ use std::sync::Arc;
 use secp;
 use secp::pedersen::{RangeProof, Commitment};
 
-use core::core::{Block, TxKernel, Output, SumCommit};
+use core::core::{Block, Output, SumCommit, TxKernel};
 use core::core::pmmr::{Summable, NoSum, PMMR, HashSum, Backend};
 use grin_store;
 use grin_store::sumtree::PMMRBackend;
@@ -89,7 +89,7 @@ impl SumTrees {
 		})
 	}
 
-	/// Wether a given commitment exists in the Output MMR and it's unspent
+	/// Whether a given commitment exists in the Output MMR and it's unspent
 	pub fn is_unspent(&self, commit: &Commitment) -> Result<bool, Error> {
 		let rpos = self.commit_index.get_output_pos(commit);
 		match rpos {
@@ -99,27 +99,33 @@ impl SumTrees {
 		}
 	}
 
+	/// returns the last N nodes inserted into the tree (i.e. the 'bottom'
+	/// nodes at level 0
+	pub fn last_n_utxo(&mut self, distance: u64) -> Vec<HashSum<SumCommit>> {
+		let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
+		output_pmmr.get_last_n_insertions(distance)
+	}
+
+	/// as above, for range proofs
+	pub fn last_n_rangeproof(&mut self, distance: u64) -> Vec<HashSum<NoSum<RangeProof>>> {
+		let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+		rproof_pmmr.get_last_n_insertions(distance)
+	}
+
+	/// as above, for kernels
+	pub fn last_n_kernel(&mut self, distance: u64) -> Vec<HashSum<NoSum<TxKernel>>> {
+		let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+		kernel_pmmr.get_last_n_insertions(distance)
+	}
+
 	/// Get sum tree roots
 	pub fn roots(
 		&mut self,
 	) -> (HashSum<SumCommit>, HashSum<NoSum<RangeProof>>, HashSum<NoSum<TxKernel>>) {
-			let output_pmmr = PMMR::at(
-				&mut self.output_pmmr_h.backend,
-				self.output_pmmr_h.last_pos,
-			);
-			let rproof_pmmr = PMMR::at(
-				&mut self.rproof_pmmr_h.backend,
-				self.rproof_pmmr_h.last_pos,
-			);
-			let kernel_pmmr = PMMR::at(
-				&mut self.kernel_pmmr_h.backend,
-				self.kernel_pmmr_h.last_pos,
-			);
-		(
-			output_pmmr.root(),
-			rproof_pmmr.root(),
-			kernel_pmmr.root(),
-		)
+		let output_pmmr = PMMR::at(&mut self.output_pmmr_h.backend, self.output_pmmr_h.last_pos);
+		let rproof_pmmr = PMMR::at(&mut self.rproof_pmmr_h.backend, self.rproof_pmmr_h.last_pos);
+		let kernel_pmmr = PMMR::at(&mut self.kernel_pmmr_h.backend, self.kernel_pmmr_h.last_pos);
+		(output_pmmr.root(), rproof_pmmr.root(), kernel_pmmr.root())
 	}
 }
 
@@ -249,19 +255,21 @@ impl<'a> Extension<'a> {
 			}
 			// push new outputs commitments in their MMR and save them in the index
 			let pos = self.output_pmmr
-				.push(SumCommit {
+				.push(
+					SumCommit {
 						commit: out.commitment(),
 						secp: secp.clone(),
 					},
-				Some(out.switch_commit_hash()))
+					Some(out.switch_commit_hash()),
+				)
 				.map_err(&Error::SumTreeErr)?;
 
 			self.new_output_commits.insert(out.commitment(), pos);
 
 			// push range proofs in their MMR
-			self.rproof_pmmr.push(NoSum(out.proof), None::<RangeProof>).map_err(
-				&Error::SumTreeErr,
-			)?;
+			self.rproof_pmmr
+				.push(NoSum(out.proof), None::<RangeProof>)
+				.map_err(&Error::SumTreeErr)?;
 		}
 
 		for kernel in &b.kernels {
@@ -269,9 +277,9 @@ impl<'a> Extension<'a> {
 				return Err(Error::DuplicateKernel(kernel.excess.clone()));
 			}
 			// push kernels in their MMR
-			let pos = self.kernel_pmmr.push(NoSum(kernel.clone()),None::<RangeProof>).map_err(
-				&Error::SumTreeErr,
-			)?;
+			let pos = self.kernel_pmmr
+				.push(NoSum(kernel.clone()), None::<RangeProof>)
+				.map_err(&Error::SumTreeErr)?;
 			self.new_kernel_excesses.insert(kernel.excess, pos);
 		}
 		Ok(())
@@ -324,7 +332,8 @@ impl<'a> Extension<'a> {
 		self.rollback = true;
 	}
 
-	/// Dumps the state of the 3 sum trees to stdout for debugging. Short version
+	/// Dumps the state of the 3 sum trees to stdout for debugging. Short
+	/// version
 	/// only prints the UTXO tree.
 	pub fn dump(&self, short: bool) {
 		debug!(LOGGER, "-- outputs --");

core/src/core/pmmr.rs

@@ -350,6 +350,37 @@ where
 		self.backend.get(position)
 	}
 
+	/// Helper function to get the last N nodes inserted, i.e. the last
+	/// n nodes along the bottom of the tree
+	pub fn get_last_n_insertions(&self, n: u64) -> Vec<HashSum<T>> {
+		let mut return_vec=Vec::new();
+		let mut last_leaf = self.last_pos;
+		let size=self.unpruned_size();
+		//Special case that causes issues in bintree functions,
+		//just return
+		if size==1 {
+			return_vec.push(self.backend.get(last_leaf).unwrap());
+			return return_vec;
+		}
+		//if size is even, we're already at the bottom, otherwise
+		//we need to traverse down to it (reverse post-order direction)
+		if size % 2 == 1 {
+			last_leaf=bintree_rightmost(self.last_pos);
+		}
+		for _ in 0..n as u64 {
+			if last_leaf==0 {
+				break;
+			}
+			if bintree_postorder_height(last_leaf) > 0 {
+				last_leaf = bintree_rightmost(last_leaf);
+			}
+			return_vec.push(self.backend.get(last_leaf).unwrap());
+			
+			last_leaf=bintree_jump_left_sibling(last_leaf);
+		}
+		return_vec
+	}
+
 	/// Total size of the tree, including intermediary nodes an ignoring any
 	/// pruning.
 	pub fn unpruned_size(&self) -> u64 {
@@ -693,6 +724,15 @@ fn bintree_move_down_left(num: u64) -> Option<u64> {
 	Some(num - (1 << height))
 }
 
+/// Gets the position of the rightmost node (i.e. leaf) relative to the current
+fn bintree_rightmost(num: u64) -> u64 {
+	let height = bintree_postorder_height(num);
+	if height == 0 {
+		return 0;
+	}
+	num - height
+}
+
 /// Calculates the position of the right sibling of a node a subtree in the
 /// postorder traversal of a full binary tree.
 fn bintree_jump_right_sibling(num: u64) -> u64 {
@@ -907,6 +947,56 @@ mod test {
 		assert_eq!(pmmr.unpruned_size(), 16);
 	}
 
+	#[test]
+	fn pmmr_get_last_n_insertions() {
+
+		let elems = [
+			TestElem([0, 0, 0, 1]),
+			TestElem([0, 0, 0, 2]),
+			TestElem([0, 0, 0, 3]),
+			TestElem([0, 0, 0, 4]),
+			TestElem([0, 0, 0, 5]),
+			TestElem([0, 0, 0, 6]),
+			TestElem([0, 0, 0, 7]),
+			TestElem([0, 0, 0, 8]),
+			TestElem([0, 0, 0, 9]),
+		];
+		let mut ba = VecBackend::new();
+		let mut pmmr = PMMR::new(&mut ba);
+
+		//test when empty
+		let res=pmmr.get_last_n_insertions(19);
+		assert!(res.len()==0);
+
+		pmmr.push(elems[0], None::<TestElem>).unwrap();
+		let res=pmmr.get_last_n_insertions(19);
+		assert!(res.len()==1 && res[0].sum==1);
+
+		pmmr.push(elems[1], None::<TestElem>).unwrap();
+
+		let res = pmmr.get_last_n_insertions(12);
+		assert!(res[0].sum==2 && res[1].sum==1);
+
+		pmmr.push(elems[2], None::<TestElem>).unwrap();
+
+		let res = pmmr.get_last_n_insertions(2);
+		assert!(res[0].sum==3 && res[1].sum==2);
+
+		pmmr.push(elems[3], None::<TestElem>).unwrap();
+
+		let res = pmmr.get_last_n_insertions(19);
+		assert!(res[0].sum==4 && res[1].sum==3 && res[2].sum==2 && res[3].sum==1 && res.len()==4);
+
+		pmmr.push(elems[5], None::<TestElem>).unwrap();
+		pmmr.push(elems[6], None::<TestElem>).unwrap();
+		pmmr.push(elems[7], None::<TestElem>).unwrap();
+		pmmr.push(elems[8], None::<TestElem>).unwrap();
+
+		let res = pmmr.get_last_n_insertions(7);
+		assert!(res[0].sum==9 && res[1].sum==8 && res[2].sum==7 && res[3].sum==6 && res.len()==7);
+
+	}
+
 	#[test]
 	#[allow(unused_variables)]
 	fn pmmr_prune() {
@@ -1030,4 +1120,5 @@ mod test {
 		assert_eq!(pl.get_shift(9), Some(8));
 		assert_eq!(pl.get_shift(17), Some(11));
 	}
+
 }