Wallet Restore fixes and updates (#1399)

* add wallet restore tests

* rustfmt

* begin to add tests for wallet restore

* rustfmt

* complete wallet restore automated tests, correct wallet restore heights and ensure TX log is repopulated on restore

wallet/src/client.rs

@@ -169,7 +169,7 @@ impl WalletClient for HTTPWalletClient {
 		(
 			u64,
 			u64,
-			Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>,
+			Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>,
 		),
 		libwallet::Error,
 	> {
@@ -178,7 +178,8 @@ impl WalletClient for HTTPWalletClient {
 
 		let url = format!("{}/v1/txhashset/outputs?{}", addr, query_param,);
 
-		let mut api_outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool)> = Vec::new();
+		let mut api_outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)> =
+			Vec::new();
 
 		match api::client::get::<api::OutputListing>(url.as_str()) {
 			Ok(o) => {
@@ -187,7 +188,12 @@ impl WalletClient for HTTPWalletClient {
 						api::OutputType::Coinbase => true,
 						api::OutputType::Transaction => false,
 					};
-					api_outputs.push((out.commit, out.range_proof().unwrap(), is_coinbase));
+					api_outputs.push((
+						out.commit,
+						out.range_proof().unwrap(),
+						is_coinbase,
+						out.block_height.unwrap(),
+					));
 				}
 
 				Ok((o.highest_index, o.last_retrieved_index, api_outputs))

wallet/src/libwallet/internal/restore.rs

@@ -43,7 +43,7 @@ struct OutputResult {
 
 fn identify_utxo_outputs<T, C, K>(
 	wallet: &mut T,
-	outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>,
+	outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>,
 ) -> Result<Vec<OutputResult>, Error>
 where
 	T: WalletBackend<C, K>,
@@ -57,10 +57,9 @@ where
 		"Scanning {} outputs in the current Grin utxo set",
 		outputs.len(),
 	);
-	let current_chain_height = wallet.client().get_chain_height()?;
 
 	for output in outputs.iter() {
-		let (commit, proof, is_coinbase) = output;
+		let (commit, proof, is_coinbase, height) = output;
 		// attempt to unwind message from the RP and get a value
 		// will fail if it's not ours
 		let info = proof::rewind(wallet.keychain(), *commit, None, *proof)?;
@@ -74,11 +73,10 @@ where
 			"Output found: {:?}, amount: {:?}", commit, info.value
 		);
 
-		let height = current_chain_height;
 		let lock_height = if *is_coinbase {
-			height + global::coinbase_maturity()
+			*height + global::coinbase_maturity()
 		} else {
-			height
+			*height
 		};
 
 		wallet_outputs.push(OutputResult {
@@ -86,7 +84,7 @@ where
 			key_id: None,
 			n_child: None,
 			value: info.value,
-			height: height,
+			height: *height,
 			lock_height: lock_height,
 			is_coinbase: *is_coinbase,
 			blinding: info.blinding,
@@ -206,6 +204,16 @@ where
 	let mut max_child_index = 0;
 	for output in result_vec {
 		if output.key_id.is_some() && output.n_child.is_some() {
+			let mut tx_log_entry = None;
+			if !output.is_coinbase {
+				let log_id = batch.next_tx_log_id(root_key_id.clone())?;
+				// also keep tx log updated so everything still tallies
+				let mut t = TxLogEntry::new(TxLogEntryType::TxReceived, log_id);
+				t.amount_credited = output.value;
+				t.num_outputs = 1;
+				tx_log_entry = Some(log_id);
+				let _ = batch.save_tx_log_entry(t);
+			}
 			let _ = batch.save(OutputData {
 				root_key_id: root_key_id.clone(),
 				key_id: output.key_id.unwrap(),
@@ -215,7 +223,7 @@ where
 				height: output.height,
 				lock_height: output.lock_height,
 				is_coinbase: output.is_coinbase,
-				tx_log_entry: None,
+				tx_log_entry: tx_log_entry,
 			});
 
 			max_child_index = if max_child_index >= output.n_child.unwrap() {

wallet/src/libwallet/types.rs

@@ -171,7 +171,7 @@ pub trait WalletClient: Sync + Send + Clone {
 	/// set in PMMR index order.
 	/// Returns
 	/// (last available output index, last insertion index retrieved,
-	/// outputs(commit, proof, is_coinbase))
+	/// outputs(commit, proof, is_coinbase, height))
 	fn get_outputs_by_pmmr_index(
 		&self,
 		start_height: u64,
@@ -180,7 +180,7 @@ pub trait WalletClient: Sync + Send + Clone {
 		(
 			u64,
 			u64,
-			Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>,
+			Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>,
 		),
 		Error,
 	>;
@@ -399,7 +399,7 @@ pub struct CbData {
 
 /// a contained wallet info struct, so automated tests can parse wallet info
 /// can add more fields here over time as needed
-#[derive(Serialize, Deserialize, Debug, Clone)]
+#[derive(Serialize, Eq, PartialEq, Deserialize, Debug, Clone)]
 pub struct WalletInfo {
 	/// height from which info was taken
 	pub last_confirmed_height: u64,

wallet/tests/common/mod.rs

@@ -63,6 +63,26 @@ fn get_output_local(chain: &chain::Chain, commit: &pedersen::Commitment) -> Opti
 	None
 }
 
+/// get output listing traversing pmmr from local
+fn get_outputs_by_pmmr_index_local(
+	chain: Arc<chain::Chain>,
+	start_index: u64,
+	max: u64,
+) -> api::OutputListing {
+	let outputs = chain
+		.unspent_outputs_by_insertion_index(start_index, max)
+		.unwrap();
+	api::OutputListing {
+		last_retrieved_index: outputs.0,
+		highest_index: outputs.1,
+		outputs: outputs
+			.2
+			.iter()
+			.map(|x| api::OutputPrintable::from_output(x, chain.clone(), None, true))
+			.collect(),
+	}
+}
+
 /// Adds a block with a given reward to the chain and mines it
 pub fn add_block_with_reward(chain: &Chain, txs: Vec<&Transaction>, reward: CbData) {
 	let prev = chain.head_header().unwrap();

wallet/tests/common/testclient.rs

@@ -145,6 +145,7 @@ where
 			let resp = match m.method.as_ref() {
 				"get_chain_height" => self.get_chain_height(m)?,
 				"get_outputs_from_node" => self.get_outputs_from_node(m)?,
+				"get_outputs_by_pmmr_index" => self.get_outputs_by_pmmr_index(m)?,
 				"send_tx_slate" => self.send_tx_slate(m)?,
 				"post_tx" => self.post_tx(m)?,
 				_ => panic!("Unknown Wallet Proxy Message"),
@@ -256,6 +257,23 @@ where
 			body: serde_json::to_string(&outputs).unwrap(),
 		})
 	}
+
+	/// get api outputs
+	fn get_outputs_by_pmmr_index(
+		&mut self,
+		m: WalletProxyMessage,
+	) -> Result<WalletProxyMessage, libwallet::Error> {
+		let split = m.body.split(",").collect::<Vec<&str>>();
+		let start_index = split[0].parse::<u64>().unwrap();
+		let max = split[1].parse::<u64>().unwrap();
+		let ol = common::get_outputs_by_pmmr_index_local(self.chain.clone(), start_index, max);
+		Ok(WalletProxyMessage {
+			sender_id: "node".to_owned(),
+			dest: m.sender_id,
+			method: m.method,
+			body: serde_json::to_string(&ol).unwrap(),
+		})
+	}
 }
 
 #[derive(Clone)]
@@ -412,16 +430,50 @@ impl WalletClient for LocalWalletClient {
 
 	fn get_outputs_by_pmmr_index(
 		&self,
-		_start_height: u64,
+		start_height: u64,
-		_max_outputs: u64,
+		max_outputs: u64,
 	) -> Result<
 		(
 			u64,
 			u64,
-			Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>,
+			Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>,
 		),
 		libwallet::Error,
 	> {
-		unimplemented!();
+		// start index, max
+		let query_str = format!("{},{}", start_height, max_outputs);
+		let m = WalletProxyMessage {
+			sender_id: self.id.clone(),
+			dest: self.node_url().to_owned(),
+			method: "get_outputs_by_pmmr_index".to_owned(),
+			body: query_str,
+		};
+		{
+			let p = self.proxy_tx.lock().unwrap();
+			p.send(m).context(libwallet::ErrorKind::ClientCallback(
+				"Get outputs from node by PMMR index send",
+			))?;
+		}
+
+		let r = self.rx.lock().unwrap();
+		let m = r.recv().unwrap();
+		let o: api::OutputListing = serde_json::from_str(&m.body).unwrap();
+
+		let mut api_outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)> =
+			Vec::new();
+
+		for out in o.outputs {
+			let is_coinbase = match out.output_type {
+				api::OutputType::Coinbase => true,
+				api::OutputType::Transaction => false,
+			};
+			api_outputs.push((
+				out.commit,
+				out.range_proof().unwrap(),
+				is_coinbase,
+				out.block_height.unwrap(),
+			));
+		}
+		Ok((o.highest_index, o.last_retrieved_index, api_outputs))
 	}
 }

wallet/tests/restore.rs

@@ -0,0 +1,308 @@
+// Copyright 2018 The Grin Developers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! tests for wallet restore
+extern crate grin_chain as chain;
+extern crate grin_core as core;
+extern crate grin_keychain as keychain;
+extern crate grin_store as store;
+extern crate grin_util as util;
+extern crate grin_wallet as wallet;
+extern crate rand;
+#[macro_use]
+extern crate slog;
+extern crate chrono;
+extern crate serde;
+extern crate uuid;
+
+mod common;
+use common::testclient::{LocalWalletClient, WalletProxy};
+
+use std::fs;
+use std::thread;
+use std::time::Duration;
+
+use core::global;
+use core::global::ChainTypes;
+use keychain::ExtKeychain;
+use util::LOGGER;
+use wallet::libtx::slate::Slate;
+use wallet::libwallet;
+
+fn clean_output_dir(test_dir: &str) {
+	let _ = fs::remove_dir_all(test_dir);
+}
+
+fn setup(test_dir: &str) {
+	util::init_test_logger();
+	clean_output_dir(test_dir);
+	global::set_mining_mode(ChainTypes::AutomatedTesting);
+}
+
+fn restore_wallet(
+	base_dir: &str,
+	wallet_dir: &str,
+	backend_type: common::BackendType,
+) -> Result<(), libwallet::Error> {
+	let source_seed = format!("{}/{}/wallet.seed", base_dir, wallet_dir);
+	let dest_dir = format!("{}/{}_restore", base_dir, wallet_dir);
+	fs::create_dir_all(dest_dir.clone())?;
+	let dest_seed = format!("{}/wallet.seed", dest_dir);
+	fs::copy(source_seed, dest_seed)?;
+
+	let mut wallet_proxy: WalletProxy<LocalWalletClient, ExtKeychain> = WalletProxy::new(base_dir);
+	let client = LocalWalletClient::new(wallet_dir, wallet_proxy.tx.clone());
+
+	let wallet = common::create_wallet(&dest_dir, client.clone(), backend_type.clone());
+
+	wallet_proxy.add_wallet(wallet_dir, client.get_send_instance(), wallet.clone());
+
+	// Set the wallet proxy listener running
+	thread::spawn(move || {
+		if let Err(e) = wallet_proxy.run() {
+			error!(LOGGER, "Wallet Proxy error: {}", e);
+		}
+	});
+
+	// perform the restore and update wallet info
+	wallet::controller::owner_single_use(wallet.clone(), |api| {
+		let _ = api.restore()?;
+		let _ = api.retrieve_summary_info(true)?;
+		Ok(())
+	})?;
+
+	Ok(())
+}
+
+fn compare_wallet_restore(
+	base_dir: &str,
+	wallet_dir: &str,
+	backend_type: common::BackendType,
+) -> Result<(), libwallet::Error> {
+	let restore_name = format!("{}_restore", wallet_dir);
+	let source_dir = format!("{}/{}", base_dir, wallet_dir);
+	let dest_dir = format!("{}/{}", base_dir, restore_name);
+
+	let mut wallet_proxy: WalletProxy<LocalWalletClient, ExtKeychain> = WalletProxy::new(base_dir);
+
+	let client = LocalWalletClient::new(wallet_dir, wallet_proxy.tx.clone());
+	let wallet_source = common::create_wallet(&source_dir, client.clone(), backend_type.clone());
+	wallet_proxy.add_wallet(
+		&wallet_dir,
+		client.get_send_instance(),
+		wallet_source.clone(),
+	);
+
+	let client = LocalWalletClient::new(&restore_name, wallet_proxy.tx.clone());
+	let wallet_dest = common::create_wallet(&dest_dir, client.clone(), backend_type.clone());
+	wallet_proxy.add_wallet(
+		&restore_name,
+		client.get_send_instance(),
+		wallet_dest.clone(),
+	);
+
+	// Set the wallet proxy listener running
+	thread::spawn(move || {
+		if let Err(e) = wallet_proxy.run() {
+			error!(LOGGER, "Wallet Proxy error: {}", e);
+		}
+	});
+
+	let mut src_info: Option<libwallet::types::WalletInfo> = None;
+	let mut dest_info: Option<libwallet::types::WalletInfo> = None;
+
+	let mut src_txs: Option<Vec<libwallet::types::TxLogEntry>> = None;
+	let mut dest_txs: Option<Vec<libwallet::types::TxLogEntry>> = None;
+
+	// Overall wallet info should be the same
+	wallet::controller::owner_single_use(wallet_source.clone(), |api| {
+		src_info = Some(api.retrieve_summary_info(true)?.1);
+		src_txs = Some(api.retrieve_txs(true, None)?.1);
+		Ok(())
+	})?;
+
+	wallet::controller::owner_single_use(wallet_dest.clone(), |api| {
+		dest_info = Some(api.retrieve_summary_info(true)?.1);
+		dest_txs = Some(api.retrieve_txs(true, None)?.1);
+		Ok(())
+	})?;
+
+	// Info should all be the same
+	assert_eq!(src_info, dest_info);
+
+	// Net differences in TX logs should be the same
+	let src_sum: i64 = src_txs
+		.unwrap()
+		.iter()
+		.map(|t| t.amount_credited as i64 - t.amount_debited as i64)
+		.sum();
+
+	let dest_sum: i64 = dest_txs
+		.unwrap()
+		.iter()
+		.map(|t| t.amount_credited as i64 - t.amount_debited as i64)
+		.sum();
+
+	assert_eq!(src_sum, dest_sum);
+
+	Ok(())
+}
+
+/// Build up 2 wallets, perform a few transactions on them
+/// Then attempt to restore them in separate directories and check contents are the same
+fn setup_restore(
+	test_dir: &str,
+	backend_type: common::BackendType,
+) -> Result<(), libwallet::Error> {
+	setup(test_dir);
+	// Create a new proxy to simulate server and wallet responses
+	let mut wallet_proxy: WalletProxy<LocalWalletClient, ExtKeychain> = WalletProxy::new(test_dir);
+	let chain = wallet_proxy.chain.clone();
+
+	// Create a new wallet test client, and set its queues to communicate with the
+	// proxy
+	let client = LocalWalletClient::new("wallet1", wallet_proxy.tx.clone());
+	let wallet1 = common::create_wallet(
+		&format!("{}/wallet1", test_dir),
+		client.clone(),
+		backend_type.clone(),
+	);
+	wallet_proxy.add_wallet("wallet1", client.get_send_instance(), wallet1.clone());
+
+	// define recipient wallet, add to proxy
+	let client = LocalWalletClient::new("wallet2", wallet_proxy.tx.clone());
+	let wallet2 = common::create_wallet(
+		&format!("{}/wallet2", test_dir),
+		client.clone(),
+		backend_type.clone(),
+	);
+	wallet_proxy.add_wallet("wallet2", client.get_send_instance(), wallet2.clone());
+
+	// Another wallet
+	let client = LocalWalletClient::new("wallet3", wallet_proxy.tx.clone());
+	let wallet3 = common::create_wallet(
+		&format!("{}/wallet3", test_dir),
+		client.clone(),
+		backend_type.clone(),
+	);
+	wallet_proxy.add_wallet("wallet3", client.get_send_instance(), wallet3.clone());
+
+	// Set the wallet proxy listener running
+	thread::spawn(move || {
+		if let Err(e) = wallet_proxy.run() {
+			error!(LOGGER, "Wallet Proxy error: {}", e);
+		}
+	});
+
+	// mine a few blocks
+	let _ = common::award_blocks_to_wallet(&chain, wallet1.clone(), 10);
+
+	// assert wallet contents
+	// and a single use api for a send command
+	let amount = 60_000_000_000;
+	let mut slate = Slate::blank(1);
+	wallet::controller::owner_single_use(wallet1.clone(), |sender_api| {
+		// note this will increment the block count as part of the transaction "Posting"
+		slate = sender_api.issue_send_tx(
+			amount,    // amount
+			2,         // minimum confirmations
+			"wallet2", // dest
+			500,       // max outputs
+			1,         // num change outputs
+			true,      // select all outputs
+		)?;
+		sender_api.post_tx(&slate, false)?;
+		Ok(())
+	})?;
+
+	// mine a few more blocks
+	let _ = common::award_blocks_to_wallet(&chain, wallet1.clone(), 3);
+
+	// Send some to wallet 3
+	wallet::controller::owner_single_use(wallet1.clone(), |sender_api| {
+		// note this will increment the block count as part of the transaction "Posting"
+		slate = sender_api.issue_send_tx(
+			amount * 2, // amount
+			2,          // minimum confirmations
+			"wallet3",  // dest
+			500,        // max outputs
+			1,          // num change outputs
+			true,       // select all outputs
+		)?;
+		sender_api.post_tx(&slate, false)?;
+		Ok(())
+	})?;
+
+	// mine a few more blocks
+	let _ = common::award_blocks_to_wallet(&chain, wallet3.clone(), 10);
+
+	// Wallet3 to wallet 2
+	wallet::controller::owner_single_use(wallet3.clone(), |sender_api| {
+		// note this will increment the block count as part of the transaction "Posting"
+		slate = sender_api.issue_send_tx(
+			amount * 3, // amount
+			2,          // minimum confirmations
+			"wallet2",  // dest
+			500,        // max outputs
+			1,          // num change outputs
+			true,       // select all outputs
+		)?;
+		sender_api.post_tx(&slate, false)?;
+		Ok(())
+	})?;
+
+	// mine a few more blocks
+	let _ = common::award_blocks_to_wallet(&chain, wallet1.clone(), 5);
+
+	// update everyone
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		let _ = api.retrieve_summary_info(true)?;
+		Ok(())
+	})?;
+	wallet::controller::owner_single_use(wallet2.clone(), |api| {
+		let _ = api.retrieve_summary_info(true)?;
+		Ok(())
+	})?;
+	wallet::controller::owner_single_use(wallet3.clone(), |api| {
+		let _ = api.retrieve_summary_info(true)?;
+		Ok(())
+	})?;
+
+	Ok(())
+}
+
+fn perform_restore(
+	test_dir: &str,
+	backend_type: common::BackendType,
+) -> Result<(), libwallet::Error> {
+	restore_wallet(test_dir, "wallet1", backend_type.clone())?;
+	compare_wallet_restore(test_dir, "wallet1", backend_type.clone())?;
+	restore_wallet(test_dir, "wallet2", backend_type.clone())?;
+	compare_wallet_restore(test_dir, "wallet2", backend_type.clone())?;
+	restore_wallet(test_dir, "wallet3", backend_type.clone())?;
+	compare_wallet_restore(test_dir, "wallet3", backend_type)?;
+	Ok(())
+}
+
+#[test]
+fn db_wallet_restore() {
+	let test_dir = "test_output/wallet_restore_db";
+	if let Err(e) = setup_restore(test_dir, common::BackendType::LMDBBackend) {
+		println!("Set up restore: Libwallet Error: {}", e);
+	}
+	if let Err(e) = perform_restore(test_dir, common::BackendType::LMDBBackend) {
+		println!("Perform restore: Libwallet Error: {}", e);
+	}
+	// let logging finish
+	thread::sleep(Duration::from_millis(200));
+}