refresh multiple wallet outputs in single api call (#205)

* mount v2 router for flexibility - wallet checker now refreshes multiple outputs via single api call
* fix the api router

api/Cargo.toml

@@ -15,6 +15,8 @@ hyper = "~0.10.6"
 slog = { version = "^2.0.12", features = ["max_level_trace", "release_max_level_trace"] }
 iron = "~0.5.1"
 router = "~0.5.1"
+mount = "~0.3.0"
+urlencoded = "~0.5.0"
 serde = "~1.0.8"
 serde_derive = "~1.0.8"
 serde_json = "~1.0.2"

api/src/endpoints.rs

@@ -20,9 +20,9 @@ use chain;
 use core::core::Transaction;
 use core::ser;
 use pool;
+use handlers::UtxoHandler;
 use rest::*;
 use types::*;
-use secp::pedersen::Commitment;
 use util;
 use util::LOGGER;
 
@@ -52,40 +52,6 @@ impl ApiEndpoint for ChainApi {
 	}
 }
 
-/// ApiEndpoint implementation for outputs that have been included in the chain.
-#[derive(Clone)]
-pub struct OutputApi {
-	/// data store access
-	chain: Arc<chain::Chain>,
-}
-
-impl ApiEndpoint for OutputApi {
-	type ID = String;
-	type T = Output;
-	type OP_IN = ();
-	type OP_OUT = ();
-
-	fn operations(&self) -> Vec<Operation> {
-		vec![Operation::Get]
-	}
-
-	fn get(&self, id: String) -> ApiResult<Output> {
-		debug!(LOGGER, "GET output {}", id);
-		let c = util::from_hex(id.clone()).map_err(|_| {
-			Error::Argument(format!("Not a valid commitment: {}", id))
-		})?;
-		let commit = Commitment::from_vec(c);
-
-		let out = self.chain.get_unspent(&commit).map_err(|_| Error::NotFound)?;
-
-		let header = self.chain
-			.get_block_header_by_output_commit(&commit)
-			.map_err(|_| Error::NotFound)?;
-
-		Ok(Output::from_output(&out, &header))
-	}
-}
-
 /// ApiEndpoint implementation for the transaction pool, to check its status
 /// and size as well as push new transactions.
 #[derive(Clone)]
@@ -166,12 +132,16 @@ pub fn start_rest_apis<T>(
 
 	thread::spawn(move || {
 		let mut apis = ApiServer::new("/v1".to_string());
-		apis.register_endpoint("/chain".to_string(), ChainApi { chain: chain.clone() });
+		apis.register_endpoint("/chain".to_string(), ChainApi {chain: chain.clone()});
-		apis.register_endpoint(
+		apis.register_endpoint("/pool".to_string(), PoolApi {tx_pool: tx_pool});
-			"/chain/utxo".to_string(),
+
-			OutputApi { chain: chain.clone() },
+		// register a nested router at "/v2" for flexibility
+		// so we can experiment with raw iron handlers
+		let utxo_handler = UtxoHandler {chain: chain.clone()};
+		let router = router!(
+			chain_utxos: get "/chain/utxos" => utxo_handler,
 		);
-		apis.register_endpoint("/pool".to_string(), PoolApi { tx_pool: tx_pool });
+		apis.register_handler("/v2", router);
 
 		apis.start(&addr[..]).unwrap_or_else(|e| {
 			error!(LOGGER, "Failed to start API HTTP server: {}.", e);

api/src/handlers.rs

@@ -0,0 +1,87 @@
+// Copyright 2016 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+use std::sync::Arc;
+
+use iron::prelude::*;
+use iron::Handler;
+use iron::status;
+use urlencoded::UrlEncodedQuery;
+use serde_json;
+
+use chain;
+use rest::*;
+use types::*;
+use secp::pedersen::Commitment;
+use util;
+use util::LOGGER;
+
+
+pub struct UtxoHandler {
+	pub chain: Arc<chain::Chain>,
+}
+
+impl UtxoHandler {
+	fn get_utxo(&self, id: &str) -> Result<Output, Error> {
+		debug!(LOGGER, "getting utxo: {}", id);
+		let c = util::from_hex(String::from(id))
+			.map_err(|_| {
+				Error::Argument(format!("Not a valid commitment: {}", id))
+			})?;
+		let commit = Commitment::from_vec(c);
+
+		let out = self.chain.get_unspent(&commit)
+			.map_err(|_| Error::NotFound)?;
+
+		let header = self.chain
+			.get_block_header_by_output_commit(&commit)
+			.map_err(|_| Error::NotFound)?;
+
+		Ok(Output::from_output(&out, &header))
+	}
+}
+
+//
+// Supports retrieval of multiple outputs in a single request -
+// GET /v2/chain/utxos?id=xxx,yyy,zzz
+// GET /v2/chain/utxos?id=xxx&id=yyy&id=zzz
+//
+impl Handler for UtxoHandler {
+	fn handle(&self, req: &mut Request) -> IronResult<Response> {
+		let mut commitments: Vec<&str> = vec![];
+		if let Ok(params) = req.get_ref::<UrlEncodedQuery>() {
+			if let Some(ids) = params.get("id") {
+				for id in ids {
+					for id in id.split(",") {
+						commitments.push(id.clone());
+					}
+				}
+			}
+		}
+
+		let mut utxos: Vec<Output> = vec![];
+
+		for commit in commitments {
+			if let Ok(out) = self.get_utxo(commit) {
+				utxos.push(out);
+			}
+		}
+
+		match serde_json::to_string(&utxos) {
+			Ok(json) => Ok(Response::with((status::Ok, json))),
+			Err(_) => Ok(Response::with((status::BadRequest, ""))),
+		}
+	}
+}

api/src/lib.rs

@@ -23,7 +23,10 @@ extern crate hyper;
 #[macro_use]
 extern crate slog;
 extern crate iron;
+extern crate urlencoded;
+#[macro_use]
 extern crate router;
+extern crate mount;
 extern crate serde;
 #[macro_use]
 extern crate serde_derive;
@@ -31,6 +34,7 @@ extern crate serde_json;
 
 pub mod client;
 mod endpoints;
+mod handlers;
 mod rest;
 mod types;
 

api/src/rest.rs

@@ -26,11 +26,13 @@ use std::string::ToString;
 use std::str::FromStr;
 use std::mem;
 
-use iron::{Iron, Request, Response, IronResult, IronError, status, headers, Listening};
+use iron::prelude::*;
+use iron::{status, headers, Listening};
 use iron::method::Method;
 use iron::modifiers::Header;
 use iron::middleware::Handler;
 use router::Router;
+use mount::Mount;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use serde_json;
@@ -244,6 +246,7 @@ fn extract_param<ID>(req: &mut Request, param: &'static str) -> IronResult<ID>
 pub struct ApiServer {
 	root: String,
 	router: Router,
+	mount: Mount,
 	server_listener: Option<Listening>,
 }
 
@@ -254,6 +257,7 @@ impl ApiServer {
 		ApiServer {
 			root: root,
 			router: Router::new(),
+			mount: Mount::new(),
 			server_listener: None,
 		}
 	}
@@ -262,7 +266,9 @@ impl ApiServer {
 	pub fn start<A: ToSocketAddrs>(&mut self, addr: A) -> Result<(), String> {
 		// replace this value to satisfy borrow checker
 		let r = mem::replace(&mut self.router, Router::new());
-		let result = Iron::new(r).http(addr);
+		let mut m = mem::replace(&mut self.mount, Mount::new());
+		m.mount("/", r);
+		let result = Iron::new(m).http(addr);
 		let return_value = result.as_ref().map(|_| ()).map_err(|e| e.to_string());
 		self.server_listener = Some(result.unwrap());
 		return_value
@@ -274,13 +280,17 @@ impl ApiServer {
 		r.unwrap().close().unwrap();
 	}
 
+	/// Registers an iron handler (via mount)
+	pub fn register_handler<H: Handler>(&mut self, route: &str, handler: H) -> &mut Mount {
+		self.mount.mount(route, handler)
+	}
+
 	/// Register a new API endpoint, providing a relative URL for the new
 	/// endpoint.
 	pub fn register_endpoint<E>(&mut self, subpath: String, endpoint: E)
 		where E: ApiEndpoint,
-		      <<E as ApiEndpoint>::ID as FromStr>::Err: Debug + Send + error::Error
+			<<E as ApiEndpoint>::ID as FromStr>::Err: Debug + Send + error::Error
 	{
-
 		assert_eq!(subpath.chars().nth(0).unwrap(), '/');
 
 		// declare a route for each method actually implemented by the endpoint

api/src/types.rs

@@ -34,13 +34,13 @@ impl Tip {
 	}
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub enum OutputType {
 	Coinbase,
 	Transaction,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Output {
 	/// The type of output Coinbase|Transaction
 	pub output_type: OutputType,

core/src/core/mod.rs

@@ -191,7 +191,6 @@ mod test {
 	use ser;
 	use keychain;
 	use keychain::{Keychain, BlindingFactor};
-	use blake2::blake2b::blake2b;
 
 	#[test]
 	#[should_panic(expected = "InvalidSecretKey")]

src/bin/grin.rs

@@ -379,22 +379,7 @@ fn wallet_command(wallet_args: &ArgMatches) {
 				);
 				wallet::receive_json_tx(&wallet_config, &keychain, contents.as_str()).unwrap();
 			} else {
-				info!(
+				wallet::server::start_rest_apis(wallet_config, keychain);
-					LOGGER,
-					"Starting the Grin wallet receiving daemon at {}...",
-					wallet_config.api_http_addr
-				);
-				let mut apis = api::ApiServer::new("/v1".to_string());
-				apis.register_endpoint(
-					"/receive".to_string(),
-					wallet::WalletReceiver {
-						keychain: keychain,
-						config: wallet_config.clone(),
-					},
-				);
-				apis.start(wallet_config.api_http_addr).unwrap_or_else(|e| {
-					error!(LOGGER, "Failed to start Grin wallet receiver: {}.", e);
-				});
 			}
 		}
 		("send", Some(send_args)) => {

wallet/Cargo.toml

@@ -16,7 +16,8 @@ blake2-rfc = "~0.2.17"
 serde = "~1.0.8"
 serde_derive = "~1.0.8"
 serde_json = "~1.0.2"
-
+iron = "~0.5.1"
+router = "~0.5.1"
 grin_api = { path = "../api" }
 grin_core = { path = "../core" }
 grin_keychain = { path = "../keychain" }

wallet/src/checker.rs

@@ -15,44 +15,95 @@
 //! Utilities to check the status of all the outputs we have stored in
 //! the wallet storage and update them.
 
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+
 use api;
 use types::*;
-use keychain::Keychain;
+use keychain::{Identifier, Keychain};
+use secp::pedersen;
 use util;
 
-fn refresh_output(out: &mut OutputData, api_out: Option<api::Output>) {
+// Transitions a local wallet output from Unconfirmed -> Unspent.
-	if let Some(api_out) = api_out {
+// Also updates the height and lock_height based on latest from the api.
-		out.height = api_out.height;
+fn refresh_output(out: &mut OutputData, api_out: &api::Output) {
-		out.lock_height = api_out.lock_height;
+	out.height = api_out.height;
+	out.lock_height = api_out.lock_height;
 
-		if out.status != OutputStatus::Locked {
+	if out.status == OutputStatus::Unconfirmed {
-			out.status = OutputStatus::Unspent;
+		out.status = OutputStatus::Unspent;
-		}
+	}
-	} else if vec![OutputStatus::Unspent, OutputStatus::Locked].contains(&out.status) {
+}
+
+// Transitions a local wallet output (based on it not being in the node utxo set) -
+// Unspent -> Spent
+// Locked -> Spent
+fn mark_spent_output(out: &mut OutputData) {
+	if vec![OutputStatus::Unspent, OutputStatus::Locked].contains(&out.status) {
 		out.status = OutputStatus::Spent;
 	}
 }
 
-/// Goes through the list of outputs that haven't been spent yet and check
+/// Builds a single api query to retrieve the latest output data from the node.
-/// with a node whether their status has changed.
+/// So we can refresh the local wallet outputs.
 pub fn refresh_outputs(
 	config: &WalletConfig,
 	keychain: &Keychain,
 ) -> Result<(), Error> {
 	WalletData::with_wallet(&config.data_file_dir, |wallet_data| {
-		// check each output that's not spent
+		let mut wallet_outputs: HashMap<pedersen::Commitment, Identifier> = HashMap::new();
-		for mut out in wallet_data.outputs.values_mut().filter(|out| {
+		let mut commits: Vec<pedersen::Commitment> = vec![];
-			out.status != OutputStatus::Spent
+
-		})
+		// build a local map of wallet outputs by commits
-		{
+		// and a list of outputs we wantot query the node for
-			// TODO check the pool for unconfirmed
+		for out in wallet_data.outputs
-			match get_output_from_node(config, keychain, out.value, out.n_child) {
+			.values_mut()
-				Ok(api_out) => refresh_output(&mut out, api_out),
+			.filter(|out| out.root_key_id == keychain.root_key_id())
-				Err(_) => {
+			.filter(|out| out.status != OutputStatus::Spent) {
-					// TODO find error with connection and return
+				let key_id = keychain.derive_key_id(out.n_child).unwrap();
-					// error!(LOGGER, "Error contacting server node at {}. Is it running?",
+				let commit = keychain.commit(out.value, &key_id).unwrap();
-					// config.check_node_api_http_addr);
+				commits.push(commit);
+				wallet_outputs.insert(commit, out.key_id.clone());
+			}
+
+		// build the necessary query params -
+		// ?id=xxx&id=yyy&id=zzz
+		let query_params: Vec<String> = commits
+			.iter()
+			.map(|commit| {
+				let id = util::to_hex(commit.as_ref().to_vec());
+				format!("id={}", id)
+			})
+			.collect();
+		let query_string = query_params.join("&");
+
+		let url = format!(
+			"{}/v2/chain/utxos?{}",
+			config.check_node_api_http_addr,
+			query_string,
+		);
+
+		// build a map of api outputs by commit so we can look them up efficiently
+		let mut api_outputs: HashMap<pedersen::Commitment, api::Output> = HashMap::new();
+		match api::client::get::<Vec<api::Output>>(url.as_str()) {
+			Ok(outputs) => {
+				for out in outputs {
+					api_outputs.insert(out.commit, out);
 				}
+			},
+			Err(_) => {},
+		};
+
+		// now for each commit we want to refresh the output for
+		// find the corresponding api output (if it exists)
+		// and refresh it in-place in the wallet
+		for commit in commits {
+			let id = wallet_outputs.get(&commit).unwrap();
+			if let Entry::Occupied(mut output) = wallet_data.outputs.entry(id.to_hex()) {
+				match api_outputs.get(&commit) {
+					Some(api_output) => refresh_output(&mut output.get_mut(), api_output),
+					None => mark_spent_output(&mut output.get_mut()),
+				};
 			}
 		}
 	})
@@ -62,27 +113,3 @@ pub fn get_tip_from_node(config: &WalletConfig) -> Result<api::Tip, Error> {
 	let url = format!("{}/v1/chain/1", config.check_node_api_http_addr);
 	api::client::get::<api::Tip>(url.as_str()).map_err(|e| Error::Node(e))
 }
-
-// queries a reachable node for a given output, checking whether it's been
-// confirmed
-fn get_output_from_node(
-	config: &WalletConfig,
-	keychain: &Keychain,
-	amount: u64,
-	derivation: u32,
-) -> Result<Option<api::Output>, Error> {
-	// do we want to store these commitments in wallet.dat?
-	let key_id = keychain.derive_key_id(derivation)?;
-	let commit = keychain.commit(amount, &key_id)?;
-
-	let url = format!(
-		"{}/v1/chain/utxo/{}",
-		config.check_node_api_http_addr,
-		util::to_hex(commit.as_ref().to_vec())
-	);
-	match api::client::get::<api::Output>(url.as_str()) {
-		Ok(out) => Ok(Some(out)),
-		Err(api::Error::NotFound) => Ok(None),
-		Err(e) => Err(Error::Node(e)),
-	}
-}

wallet/src/lib.rs

@@ -24,6 +24,9 @@ extern crate serde;
 extern crate serde_derive;
 extern crate serde_json;
 
+extern crate iron;
+extern crate router;
+
 extern crate grin_api as api;
 extern crate grin_core as core;
 extern crate grin_keychain as keychain;
@@ -35,6 +38,7 @@ mod info;
 mod receiver;
 mod sender;
 mod types;
+pub mod server;
 
 pub use info::show_info;
 pub use receiver::{WalletReceiver, receive_json_tx};

wallet/src/server.rs

@@ -0,0 +1,42 @@
+// Copyright 2016 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+use api::ApiServer;
+use keychain::Keychain;
+use receiver::WalletReceiver;
+use types::WalletConfig;
+use util::LOGGER;
+
+pub fn start_rest_apis(wallet_config: WalletConfig, keychain: Keychain) {
+	info!(
+		LOGGER,
+		"Starting the Grin wallet receiving daemon at {}...",
+		wallet_config.api_http_addr
+	);
+
+	let mut apis = ApiServer::new("/v1".to_string());
+
+	apis.register_endpoint(
+		"/receive".to_string(),
+		WalletReceiver {
+			keychain: keychain,
+			config: wallet_config.clone(),
+		},
+	);
+
+	apis.start(wallet_config.api_http_addr).unwrap_or_else(|e| {
+		error!(LOGGER, "Failed to start Grin wallet receiver: {}.", e);
+	});
+}