Wallet Restore feature (#338)

* beginning to add wallet restore... api endpoints and basic restore

* basic restore working, still missing features

* rustfmt

* large speed up to output search, should be more or less working

* properly mark coinbase status

api/src/handlers.rs

@@ -25,6 +25,7 @@ use serde_json;
 
 use chain;
 use core::core::Transaction;
+use core::core::hash::Hashed;
 use core::ser;
 use pool;
 use p2p;
@@ -51,35 +52,39 @@ impl Handler for IndexHandler {
 }
 
 // Supports retrieval of multiple outputs in a single request -
-// GET /v1/chain/utxos?id=xxx,yyy,zzz
-// GET /v1/chain/utxos?id=xxx&id=yyy&id=zzz
+// GET /v1/chain/utxos/byids?id=xxx,yyy,zzz
+// GET /v1/chain/utxos/byids?id=xxx&id=yyy&id=zzz
+// GET /v1/chain/utxos/byheight?height=n
 struct UtxoHandler {
 	chain: Arc<chain::Chain>,
 }
 
 impl UtxoHandler {
-	fn get_utxo(&self, id: &str) -> Result<Output, Error> {
+	fn get_utxo(&self, id: &str, include_rp: bool, include_switch: bool) -> Result<Output, Error> {
 		debug!(LOGGER, "getting utxo: {}", id);
 		let c = util::from_hex(String::from(id)).map_err(|_| {
 			Error::Argument(format!("Not a valid commitment: {}", id))
 		})?;
 		let commit = Commitment::from_vec(c);
 
-		let out = self.chain
-			.get_unspent(&commit)
-			.map_err(|_| Error::NotFound)?;
+		let out = self.chain.get_unspent(&commit).map_err(|_| Error::NotFound)?;
 
 		let header = self.chain
 			.get_block_header_by_output_commit(&commit)
 			.map_err(|_| Error::NotFound)?;
 
-		Ok(Output::from_output(&out, &header, false))
+		Ok(Output::from_output(
+			&out,
+			&header,
+			include_rp,
+			include_switch,
+		))
 	}
-}
 
-impl Handler for UtxoHandler {
-	fn handle(&self, req: &mut Request) -> IronResult<Response> {
+	fn utxos_by_ids(&self, req: &mut Request) -> Vec<Output> {
 		let mut commitments: Vec<&str> = vec![];
+		let mut rp = false;
+		let mut switch = false;
 		if let Ok(params) = req.get_ref::<UrlEncodedQuery>() {
 			if let Some(ids) = params.get("id") {
 				for id in ids {
@@ -88,15 +93,75 @@ impl Handler for UtxoHandler {
 					}
 				}
 			}
+			if let Some(_) = params.get("include_rp") {
+				rp = true;
+			}
+			if let Some(_) = params.get("include_switch") {
+				switch = true;
+			}
 		}
-
 		let mut utxos: Vec<Output> = vec![];
 		for commit in commitments {
-			if let Ok(out) = self.get_utxo(commit) {
+			if let Ok(out) = self.get_utxo(commit, rp, switch) {
 				utxos.push(out);
 			}
 		}
-		json_response(&utxos)
+		utxos
+	}
+
+	fn utxos_at_height(&self, block_height: u64) -> BlockOutputs {
+		let header = self.chain
+			.clone()
+			.get_header_by_height(block_height)
+			.unwrap();
+		let block = self.chain.clone().get_block(&header.hash()).unwrap();
+		let outputs = block
+			.outputs
+			.iter()
+			.map(|k| OutputSwitch::from_output(k, &header))
+			.collect();
+		BlockOutputs {
+			header: BlockHeaderInfo::from_header(&header),
+			outputs: outputs,
+		}
+	}
+
+	// returns utxos for a specified range of blocks
+	fn utxo_block_batch(&self, req: &mut Request) -> Vec<BlockOutputs> {
+		let mut start_height = 1;
+		let mut end_height = 1;
+		if let Ok(params) = req.get_ref::<UrlEncodedQuery>() {
+			if let Some(heights) = params.get("start_height") {
+				for height in heights {
+					start_height = height.parse().unwrap();
+				}
+			}
+			if let Some(heights) = params.get("end_height") {
+				for height in heights {
+					end_height = height.parse().unwrap();
+				}
+			}
+		}
+		let mut return_vec = vec![];
+		for i in start_height..end_height + 1 {
+			return_vec.push(self.utxos_at_height(i));
+		}
+		return_vec
+	}
+}
+
+impl Handler for UtxoHandler {
+	fn handle(&self, req: &mut Request) -> IronResult<Response> {
+		let url = req.url.clone();
+		let mut path_elems = url.path();
+		if *path_elems.last().unwrap() == "" {
+			path_elems.pop();
+		}
+		match *path_elems.last().unwrap() {
+			"byids" => json_response(&self.utxos_by_ids(req)),
+			"atheight" => json_response(&self.utxo_block_batch(req)),
+			_ => Ok(Response::with((status::BadRequest, ""))),
+		}
 	}
 }
 
@@ -242,15 +307,18 @@ where
 	T: pool::BlockChain + Send + Sync + 'static,
 {
 	fn handle(&self, req: &mut Request) -> IronResult<Response> {
-		let wrapper: TxWrapper = serde_json::from_reader(req.body.by_ref())
-			.map_err(|e| IronError::new(e, status::BadRequest))?;
+		let wrapper: TxWrapper = serde_json::from_reader(req.body.by_ref()).map_err(|e| {
+			IronError::new(e, status::BadRequest)
+		})?;
 
 		let tx_bin = util::from_hex(wrapper.tx_hex).map_err(|_| {
 			Error::Argument(format!("Invalid hex in transaction wrapper."))
 		})?;
 
 		let tx: Transaction = ser::deserialize(&mut &tx_bin[..]).map_err(|_| {
-			Error::Argument("Could not deserialize transaction, invalid format.".to_string())
+			Error::Argument(
+				"Could not deserialize transaction, invalid format.".to_string(),
+			)
 		})?;
 
 		let source = pool::TxSource {
@@ -349,7 +417,7 @@ pub fn start_rest_apis<T>(
 		let router = router!(
 			index: get "/" => index_handler,
 			chain_tip: get "/chain" => chain_tip_handler,
-			chain_utxos: get "/chain/utxos" => utxo_handler,
+			chain_utxos: get "/chain/utxos/*" => utxo_handler,
 			sumtree_roots: get "/sumtrees/*" => sumtree_handler,
 			pool_info: get "/pool" => pool_info_handler,
 			pool_push: post "/pool/push" => pool_push_handler,

api/src/types.rs

@@ -88,7 +88,7 @@ impl SumTreeNode {
 				.get_block_header_by_output_commit(&elem_output.1.commit)
 				.map_err(|_| Error::NotFound);
 			// Need to call further method to check if output is spent
-			let mut output = OutputPrintable::from_output(&elem_output.1, &header.unwrap());
+			let mut output = OutputPrintable::from_output(&elem_output.1, &header.unwrap(),true);
 			if let Ok(_) = chain.get_unspent(&elem_output.1.commit) {
 				output.spent = false;
 			}
@@ -137,6 +137,8 @@ pub struct Output {
 	pub output_type: OutputType,
 	/// The homomorphic commitment representing the output's amount
 	pub commit: pedersen::Commitment,
+	/// switch commit hash
+	pub switch_commit_hash: Option<core::SwitchCommitHash>,
 	/// A proof that the commitment is in the right range
 	pub proof: Option<pedersen::RangeProof>,
 	/// The height of the block creating this output
@@ -146,7 +148,8 @@ pub struct Output {
 }
 
 impl Output {
-	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader, include_proof:bool) -> Output {
+	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader, 
+		include_proof:bool, include_switch: bool) -> Output {
 		let (output_type, lock_height) = match output.features {
 			x if x.contains(core::transaction::COINBASE_OUTPUT) => (
 				OutputType::Coinbase,
@@ -158,6 +161,10 @@ impl Output {
 		Output {
 			output_type: output_type,
 			commit: output.commit,
+			switch_commit_hash: match include_switch {
+				true => Some(output.switch_commit_hash),
+				false => None,
+			},
 			proof: match include_proof {
 				true => Some(output.proof),
 				false => None,
@@ -176,6 +183,8 @@ pub struct OutputPrintable {
 	/// The homomorphic commitment representing the output's amount (as hex
 	/// string)
 	pub commit: String,
+	/// switch commit hash
+	pub switch_commit_hash: String,
 	/// The height of the block creating this output
 	pub height: u64,
 	/// The lock height (earliest block this output can be spent)
@@ -183,11 +192,11 @@ pub struct OutputPrintable {
 	/// Whether the output has been spent
 	pub spent: bool,
 	/// Rangeproof hash  (as hex string)
-	pub proof_hash: String,
+	pub proof_hash: Option<String>,
 }
 
 impl OutputPrintable {
-	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader) -> OutputPrintable {
+	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader, include_proof_hash:bool) -> OutputPrintable {
 		let (output_type, lock_height) = match output.features {
 			x if x.contains(core::transaction::COINBASE_OUTPUT) => (
 				OutputType::Coinbase,
@@ -198,14 +207,68 @@ impl OutputPrintable {
 		OutputPrintable {
 			output_type: output_type,
 			commit: util::to_hex(output.commit.0.to_vec()),
+			switch_commit_hash: util::to_hex(output.switch_commit_hash.hash.to_vec()),
 			height: block_header.height,
 			lock_height: lock_height,
 			spent: true,
-			proof_hash: util::to_hex(output.proof.hash().to_vec()),
+			proof_hash: match include_proof_hash {
+				true => Some(util::to_hex(output.proof.hash().to_vec())),
+				false => None,
+			}
 		}
 	}
 }
 
+// As above, except just the info needed for wallet reconstruction
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct OutputSwitch {
+	/// the commit 
+	pub commit: String,
+	/// switch commit hash
+	pub switch_commit_hash: [u8; core::SWITCH_COMMIT_HASH_SIZE],
+	/// The height of the block creating this output
+	pub height: u64,
+}
+
+impl OutputSwitch {
+	pub fn from_output(output: &core::Output, block_header: &core::BlockHeader) -> OutputSwitch {
+		OutputSwitch {
+			commit: util::to_hex(output.commit.0.to_vec()),
+			switch_commit_hash: output.switch_commit_hash.hash,
+			height: block_header.height,
+		}
+	}
+}
+// Just the information required for wallet reconstruction
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct BlockHeaderInfo {
+	/// Hash
+	pub hash: String,
+	/// Previous block hash
+	pub previous: String,
+	/// Height
+	pub height: u64
+}
+
+impl BlockHeaderInfo {
+	pub fn from_header(block_header: &core::BlockHeader) -> BlockHeaderInfo{
+		BlockHeaderInfo {
+			hash: util::to_hex(block_header.hash().to_vec()),
+			previous: util::to_hex(block_header.previous.to_vec()),
+			height: block_header.height,
+		}
+	}
+}
+// For wallet reconstruction, include the header info along with the
+// transactions in the block
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct BlockOutputs {
+	/// The block header
+	pub header: BlockHeaderInfo,
+	/// A printable version of the outputs
+	pub outputs: Vec<OutputSwitch>,
+}
+
 #[derive(Serialize, Deserialize)]
 pub struct PoolInfo {
 	/// Size of the pool

core/src/core/transaction.rs

@@ -401,7 +401,8 @@ bitflags! {
 /// Definition of the switch commitment hash
 #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub struct SwitchCommitHash {
-	hash: [u8; SWITCH_COMMIT_HASH_SIZE],
+	/// simple hash
+	pub hash: [u8; SWITCH_COMMIT_HASH_SIZE],
 }
 
 /// Implementation of Writeable for a switch commitment hash

keychain/src/keychain.rs

@@ -172,6 +172,14 @@ impl Keychain {
 		Ok(commit)
 	}
 
+	pub fn switch_commit_from_index(&self, index:u32) -> Result<Commitment, Error> {
+		//just do this directly, because cache seems really slow for wallet reconstruct
+		let skey = self.extkey.derive(&self.secp, index)?;
+		let skey = skey.key;
+		let commit = self.secp.switch_commit(skey)?;
+		Ok(commit)
+	}
+
 	pub fn range_proof(
 		&self,
 		amount: u64,

src/bin/grin.rs

@@ -188,7 +188,14 @@ fn main() {
 			.long("api_server_address")
 			.help("Api address of running node on which to check inputs and post transactions")
 			.takes_value(true))
-
+		.arg(Arg::with_name("key_derivations")
+				.help("The number of keys possiblities to search for each output. \
+				Ideally, set this to a number greater than the number of outputs \
+				you believe should belong to this seed/password. (Default 500)")
+				.short("k")
+				.long("key_derivations")
+				.default_value("1000")
+				.takes_value(true))
 		.subcommand(SubCommand::with_name("listen")
 			.about("Run the wallet in listening mode. If an input file is \
 				provided, will process it, otherwise runs in server mode waiting \
@@ -250,7 +257,10 @@ fn main() {
 			.about("basic wallet contents summary"))
 
 		.subcommand(SubCommand::with_name("init")
-			.about("Initialize a new wallet seed file.")))
+			.about("Initialize a new wallet seed file."))
+
+		.subcommand(SubCommand::with_name("restore")
+			.about("Attempt to restore wallet contents from the chain using seed and password.")))
 			
 	.get_matches();
 
@@ -376,6 +386,11 @@ fn wallet_command(wallet_args: &ArgMatches) {
 		wallet_config.check_node_api_http_addr = sa.to_string().clone();
 	}
 
+	let mut key_derivations:u32=1000;
+	if let Some(kd) = wallet_args.value_of("key_derivations") {
+		key_derivations=kd.parse().unwrap();
+	}
+
 	let mut show_spent=false;
 	if wallet_args.is_present("show_spent") {
 		show_spent=true;
@@ -469,6 +484,9 @@ fn wallet_command(wallet_args: &ArgMatches) {
 		("outputs", Some(_)) => {
 			wallet::show_outputs(&wallet_config, &keychain, show_spent);
 		}
+		("restore", Some(_)) => {
+			let _=wallet::restore(&wallet_config, &keychain, key_derivations);
+		}
 		("receive", Some(_)) => {
 			panic!("Command 'receive' is depreciated, use 'listen' instead");
 		}

wallet/src/checker.rs

@@ -84,7 +84,7 @@ pub fn refresh_outputs(config: &WalletConfig, keychain: &Keychain) -> Result<(),
 	let query_string = query_params.join("&");
 
 	let url = format!(
-		"{}/v1/chain/utxos?{}",
+		"{}/v1/chain/utxos/byids?{}",
 		config.check_node_api_http_addr,
 		query_string,
 	);

wallet/src/lib.rs

@@ -48,6 +48,7 @@ mod info;
 mod receiver;
 mod sender;
 mod types;
+mod restore;
 pub mod client;
 pub mod server;
 
@@ -56,3 +57,4 @@ pub use info::show_info;
 pub use receiver::{receive_json_tx, receive_json_tx_str, WalletReceiver};
 pub use sender::{issue_burn_tx, issue_send_tx};
 pub use types::{BlockFees, CbData, Error, WalletConfig, WalletReceiveRequest, WalletSeed};
+pub use restore::restore;

wallet/src/restore.rs

@@ -0,0 +1,202 @@
+// Copyright 2017 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use keychain::{Keychain, Identifier};
+use util::{LOGGER, from_hex};
+use util::secp::pedersen;
+use api;
+use core::core::{Output,SwitchCommitHash};
+use core::core::transaction::{COINBASE_OUTPUT, DEFAULT_OUTPUT, SWITCH_COMMIT_HASH_SIZE};
+use types::{WalletConfig, WalletData, OutputData, OutputStatus, Error};
+use byteorder::{BigEndian, ByteOrder};
+
+pub fn get_chain_height(config: &WalletConfig)->
+	Result<u64, Error>{
+	let url = format!(
+		"{}/v1/chain",
+		config.check_node_api_http_addr
+	);
+
+	match api::client::get::<api::Tip>(url.as_str()) {
+		Ok(tip) => {
+			Ok(tip.height)
+		},
+		Err(e) => {
+			// if we got anything other than 200 back from server, bye
+			error!(LOGGER, "Restore failed... unable to contact node: {}", e);
+			Err(Error::Node(e))
+		}
+	}
+}
+
+fn output_with_range_proof(config:&WalletConfig, commit_id: &str) ->
+	Result<api::Output, Error>{
+
+	let url = format!(
+		"{}/v1/chain/utxos/byids?id={}&include_rp&include_switch",
+		config.check_node_api_http_addr,
+		commit_id,
+	);
+
+	match api::client::get::<Vec<api::Output>>(url.as_str()) {
+		Ok(outputs) => {
+			Ok(outputs[0].clone())
+		},
+		Err(e) => {
+			// if we got anything other than 200 back from server, don't attempt to refresh the wallet
+			// data after
+			Err(Error::Node(e))
+		}
+	}
+}
+
+fn retrieve_amount_and_coinbase_status(config:&WalletConfig, keychain: &Keychain,
+	key_id: Identifier, commit_id: &str) -> (u64, bool) {
+	let output = output_with_range_proof(config, commit_id).unwrap();
+	let core_output = Output {
+		features : match output.output_type {
+			api::OutputType::Coinbase => COINBASE_OUTPUT,
+			api::OutputType::Transaction => DEFAULT_OUTPUT,
+		},
+		proof: output.proof.unwrap(),
+		switch_commit_hash: output.switch_commit_hash.unwrap(),
+		commit: output.commit,
+	};
+	let amount=core_output.recover_value(keychain, &key_id).unwrap();
+	let is_coinbase = match output.output_type {
+		api::OutputType::Coinbase => true,
+		api::OutputType::Transaction => false,
+	};
+	(amount, is_coinbase)
+}
+
+pub fn utxos_batch_block(config: &WalletConfig, start_height: u64, end_height:u64)->
+	Result<Vec<api::BlockOutputs>, Error>{
+	// build the necessary query param -
+	// ?height=x
+	let query_param= format!("start_height={}&end_height={}", start_height, end_height);
+
+	let url = format!(
+		"{}/v1/chain/utxos/atheight?{}",
+		config.check_node_api_http_addr,
+		query_param,
+	);
+
+	match api::client::get::<Vec<api::BlockOutputs>>(url.as_str()) {
+		Ok(outputs) => Ok(outputs),
+		Err(e) => {
+			// if we got anything other than 200 back from server, bye
+			error!(LOGGER, "Restore failed... unable to contact node: {}", e);
+			Err(Error::Node(e))
+		}
+	}
+}
+
+fn find_utxos_with_key(config:&WalletConfig, keychain: &Keychain, 
+	switch_commit_cache : &Vec<[u8;SWITCH_COMMIT_HASH_SIZE]>,
+	block_outputs:api::BlockOutputs, key_iterations: &mut usize, padding: &mut usize) 
+	-> Vec<(pedersen::Commitment, Identifier, u32, u64, u64, bool) > {
+	//let key_id = keychain.clone().root_key_id();
+	let mut wallet_outputs: Vec<(pedersen::Commitment, Identifier, u32, u64, u64, bool)> = Vec::new();
+
+	info!(LOGGER, "Scanning block {} over {} key derivation possibilities.", block_outputs.header.height, *key_iterations);
+	for output in block_outputs.outputs {
+		for i in 0..*key_iterations {
+			if switch_commit_cache[i as usize]==output.switch_commit_hash {
+				info!(LOGGER, "Output found: {:?}, key_index: {:?}", output.switch_commit_hash,i);
+				//add it to result set here
+				let commit_id = from_hex(output.commit.clone()).unwrap();
+				let key_id = keychain.derive_key_id(i as u32).unwrap();
+				let (amount, is_coinbase) = retrieve_amount_and_coinbase_status(config,
+					keychain, key_id.clone(), &output.commit);
+				info!(LOGGER, "Amount: {}", amount);
+				let commit = keychain.commit_with_key_index(BigEndian::read_u64(&commit_id), i as u32).unwrap();
+				wallet_outputs.push((commit, key_id.clone(), i as u32, amount, output.height, is_coinbase));
+				//probably don't have to look for indexes greater than this now
+				*key_iterations=i+*padding;
+				if *key_iterations > switch_commit_cache.len() {
+					*key_iterations = switch_commit_cache.len();
+				}
+				info!(LOGGER, "Setting max key index to: {}", *key_iterations);
+				break;
+			}
+		}
+	}
+	wallet_outputs
+}
+
+pub fn restore(config: &WalletConfig, keychain: &Keychain, key_derivations:u32) ->
+	Result<(), Error>{
+	// Don't proceed if wallet.dat has anything in it
+	let is_empty = WalletData::read_wallet(&config.data_file_dir, |wallet_data| {
+		wallet_data.outputs.len() ==  0
+	})?;
+	if !is_empty {
+		error!(LOGGER, "Not restoring. Please back up and remove existing wallet.dat first.");
+		return Ok(())
+	}
+
+// Get height of chain from node (we'll check again when done)
+	let chain_height = get_chain_height(config)?;
+	info!(LOGGER, "Starting restore: Chain height is {}.", chain_height);
+
+	let mut switch_commit_cache : Vec<[u8;SWITCH_COMMIT_HASH_SIZE]> = vec![];
+	info!(LOGGER, "Building key derivation cache to index {}.", key_derivations);
+	for i in 0..key_derivations {
+		let switch_commit = keychain.switch_commit_from_index(i as u32).unwrap();
+		let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit);
+		switch_commit_cache.push(switch_commit_hash.hash);
+	}
+
+	let batch_size=100;
+	//this will start here, then lower as outputs are found, moving backwards on the chain
+	let mut key_iterations=key_derivations as usize;
+	//set to a percentage of the key_derivation value
+	let mut padding = (key_iterations as f64 *0.25) as usize;
+	let mut h = chain_height;
+	while {
+		let end_batch=h;
+		if h >= batch_size {
+			h-=batch_size;
+		} else {
+			h=0;
+		}
+		let mut blocks = utxos_batch_block(config, h+1, end_batch)?;
+		blocks.reverse();
+		for block in blocks {
+			let result_vec=find_utxos_with_key(config, keychain, &switch_commit_cache,
+				block, &mut key_iterations, &mut padding);
+			if result_vec.len() > 0 {
+				for output in result_vec.clone() {
+					let _ = WalletData::with_wallet(&config.data_file_dir, |wallet_data| {
+						let root_key_id = keychain.root_key_id();
+						//Just plonk it in for now, and refresh actual values via wallet info command later 
+						wallet_data.add_output(OutputData {
+							root_key_id: root_key_id.clone(),
+							key_id: output.1.clone(),
+							n_child: output.2,
+							value: output.3,
+							status: OutputStatus::Unconfirmed,
+							height: output.4,
+							lock_height: 0,
+							is_coinbase: output.5,
+						});
+					});
+				}
+			}
+		}
+		h > 0
+	}{} 
+	Ok(())
+}