From ccc9c0db90608126b21753b23777c16fb3d72b3c Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Tue, 21 Aug 2018 18:05:28 +0100 Subject: [PATCH] Wallet Restore fixes and updates (#1399) * add wallet restore tests * rustfmt * begin to add tests for wallet restore * rustfmt * complete wallet restore automated tests, correct wallet restore heights and ensure TX log is repopulated on restore --- wallet/src/client.rs | 12 +- wallet/src/libwallet/internal/restore.rs | 24 +- wallet/src/libwallet/types.rs | 6 +- wallet/tests/common/mod.rs | 20 ++ wallet/tests/common/testclient.rs | 60 ++++- wallet/tests/restore.rs | 308 +++++++++++++++++++++++ 6 files changed, 412 insertions(+), 18 deletions(-) create mode 100644 wallet/tests/restore.rs diff --git a/wallet/src/client.rs b/wallet/src/client.rs index bf2fe8a98..ac3655654 100644 --- a/wallet/src/client.rs +++ b/wallet/src/client.rs @@ -169,7 +169,7 @@ impl WalletClient for HTTPWalletClient { ( u64, u64, - Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>, + Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>, ), libwallet::Error, > { @@ -178,7 +178,8 @@ impl WalletClient for HTTPWalletClient { let url = format!("{}/v1/txhashset/outputs?{}", addr, query_param,); - let mut api_outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool)> = Vec::new(); + let mut api_outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)> = + Vec::new(); match api::client::get::(url.as_str()) { Ok(o) => { @@ -187,7 +188,12 @@ impl WalletClient for HTTPWalletClient { api::OutputType::Coinbase => true, api::OutputType::Transaction => false, }; - api_outputs.push((out.commit, out.range_proof().unwrap(), is_coinbase)); + api_outputs.push(( + out.commit, + out.range_proof().unwrap(), + is_coinbase, + out.block_height.unwrap(), + )); } Ok((o.highest_index, o.last_retrieved_index, api_outputs)) diff --git a/wallet/src/libwallet/internal/restore.rs b/wallet/src/libwallet/internal/restore.rs index 192dd85e8..658efd3ed 100644 --- a/wallet/src/libwallet/internal/restore.rs +++ b/wallet/src/libwallet/internal/restore.rs @@ -43,7 +43,7 @@ struct OutputResult { fn identify_utxo_outputs( wallet: &mut T, - outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>, + outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>, ) -> Result, Error> where T: WalletBackend, @@ -57,10 +57,9 @@ where "Scanning {} outputs in the current Grin utxo set", outputs.len(), ); - let current_chain_height = wallet.client().get_chain_height()?; for output in outputs.iter() { - let (commit, proof, is_coinbase) = output; + let (commit, proof, is_coinbase, height) = output; // attempt to unwind message from the RP and get a value // will fail if it's not ours let info = proof::rewind(wallet.keychain(), *commit, None, *proof)?; @@ -74,11 +73,10 @@ where "Output found: {:?}, amount: {:?}", commit, info.value ); - let height = current_chain_height; let lock_height = if *is_coinbase { - height + global::coinbase_maturity() + *height + global::coinbase_maturity() } else { - height + *height }; wallet_outputs.push(OutputResult { @@ -86,7 +84,7 @@ where key_id: None, n_child: None, value: info.value, - height: height, + height: *height, lock_height: lock_height, is_coinbase: *is_coinbase, blinding: info.blinding, @@ -206,6 +204,16 @@ where let mut max_child_index = 0; for output in result_vec { if output.key_id.is_some() && output.n_child.is_some() { + let mut tx_log_entry = None; + if !output.is_coinbase { + let log_id = batch.next_tx_log_id(root_key_id.clone())?; + // also keep tx log updated so everything still tallies + let mut t = TxLogEntry::new(TxLogEntryType::TxReceived, log_id); + t.amount_credited = output.value; + t.num_outputs = 1; + tx_log_entry = Some(log_id); + let _ = batch.save_tx_log_entry(t); + } let _ = batch.save(OutputData { root_key_id: root_key_id.clone(), key_id: output.key_id.unwrap(), @@ -215,7 +223,7 @@ where height: output.height, lock_height: output.lock_height, is_coinbase: output.is_coinbase, - tx_log_entry: None, + tx_log_entry: tx_log_entry, }); max_child_index = if max_child_index >= output.n_child.unwrap() { diff --git a/wallet/src/libwallet/types.rs b/wallet/src/libwallet/types.rs index 018922d55..1f3a7f7d7 100644 --- a/wallet/src/libwallet/types.rs +++ b/wallet/src/libwallet/types.rs @@ -171,7 +171,7 @@ pub trait WalletClient: Sync + Send + Clone { /// set in PMMR index order. /// Returns /// (last available output index, last insertion index retrieved, - /// outputs(commit, proof, is_coinbase)) + /// outputs(commit, proof, is_coinbase, height)) fn get_outputs_by_pmmr_index( &self, start_height: u64, @@ -180,7 +180,7 @@ pub trait WalletClient: Sync + Send + Clone { ( u64, u64, - Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>, + Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>, ), Error, >; @@ -399,7 +399,7 @@ pub struct CbData { /// a contained wallet info struct, so automated tests can parse wallet info /// can add more fields here over time as needed -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Eq, PartialEq, Deserialize, Debug, Clone)] pub struct WalletInfo { /// height from which info was taken pub last_confirmed_height: u64, diff --git a/wallet/tests/common/mod.rs b/wallet/tests/common/mod.rs index 2b173d165..07285cf64 100644 --- a/wallet/tests/common/mod.rs +++ b/wallet/tests/common/mod.rs @@ -63,6 +63,26 @@ fn get_output_local(chain: &chain::Chain, commit: &pedersen::Commitment) -> Opti None } +/// get output listing traversing pmmr from local +fn get_outputs_by_pmmr_index_local( + chain: Arc, + start_index: u64, + max: u64, +) -> api::OutputListing { + let outputs = chain + .unspent_outputs_by_insertion_index(start_index, max) + .unwrap(); + api::OutputListing { + last_retrieved_index: outputs.0, + highest_index: outputs.1, + outputs: outputs + .2 + .iter() + .map(|x| api::OutputPrintable::from_output(x, chain.clone(), None, true)) + .collect(), + } +} + /// Adds a block with a given reward to the chain and mines it pub fn add_block_with_reward(chain: &Chain, txs: Vec<&Transaction>, reward: CbData) { let prev = chain.head_header().unwrap(); diff --git a/wallet/tests/common/testclient.rs b/wallet/tests/common/testclient.rs index 013f59670..560f9d4f4 100644 --- a/wallet/tests/common/testclient.rs +++ b/wallet/tests/common/testclient.rs @@ -145,6 +145,7 @@ where let resp = match m.method.as_ref() { "get_chain_height" => self.get_chain_height(m)?, "get_outputs_from_node" => self.get_outputs_from_node(m)?, + "get_outputs_by_pmmr_index" => self.get_outputs_by_pmmr_index(m)?, "send_tx_slate" => self.send_tx_slate(m)?, "post_tx" => self.post_tx(m)?, _ => panic!("Unknown Wallet Proxy Message"), @@ -256,6 +257,23 @@ where body: serde_json::to_string(&outputs).unwrap(), }) } + + /// get api outputs + fn get_outputs_by_pmmr_index( + &mut self, + m: WalletProxyMessage, + ) -> Result { + let split = m.body.split(",").collect::>(); + let start_index = split[0].parse::().unwrap(); + let max = split[1].parse::().unwrap(); + let ol = common::get_outputs_by_pmmr_index_local(self.chain.clone(), start_index, max); + Ok(WalletProxyMessage { + sender_id: "node".to_owned(), + dest: m.sender_id, + method: m.method, + body: serde_json::to_string(&ol).unwrap(), + }) + } } #[derive(Clone)] @@ -412,16 +430,50 @@ impl WalletClient for LocalWalletClient { fn get_outputs_by_pmmr_index( &self, - _start_height: u64, - _max_outputs: u64, + start_height: u64, + max_outputs: u64, ) -> Result< ( u64, u64, - Vec<(pedersen::Commitment, pedersen::RangeProof, bool)>, + Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)>, ), libwallet::Error, > { - unimplemented!(); + // start index, max + let query_str = format!("{},{}", start_height, max_outputs); + let m = WalletProxyMessage { + sender_id: self.id.clone(), + dest: self.node_url().to_owned(), + method: "get_outputs_by_pmmr_index".to_owned(), + body: query_str, + }; + { + let p = self.proxy_tx.lock().unwrap(); + p.send(m).context(libwallet::ErrorKind::ClientCallback( + "Get outputs from node by PMMR index send", + ))?; + } + + let r = self.rx.lock().unwrap(); + let m = r.recv().unwrap(); + let o: api::OutputListing = serde_json::from_str(&m.body).unwrap(); + + let mut api_outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64)> = + Vec::new(); + + for out in o.outputs { + let is_coinbase = match out.output_type { + api::OutputType::Coinbase => true, + api::OutputType::Transaction => false, + }; + api_outputs.push(( + out.commit, + out.range_proof().unwrap(), + is_coinbase, + out.block_height.unwrap(), + )); + } + Ok((o.highest_index, o.last_retrieved_index, api_outputs)) } } diff --git a/wallet/tests/restore.rs b/wallet/tests/restore.rs new file mode 100644 index 000000000..bfa7a01a8 --- /dev/null +++ b/wallet/tests/restore.rs @@ -0,0 +1,308 @@ +// Copyright 2018 The Grin Developers +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! tests for wallet restore +extern crate grin_chain as chain; +extern crate grin_core as core; +extern crate grin_keychain as keychain; +extern crate grin_store as store; +extern crate grin_util as util; +extern crate grin_wallet as wallet; +extern crate rand; +#[macro_use] +extern crate slog; +extern crate chrono; +extern crate serde; +extern crate uuid; + +mod common; +use common::testclient::{LocalWalletClient, WalletProxy}; + +use std::fs; +use std::thread; +use std::time::Duration; + +use core::global; +use core::global::ChainTypes; +use keychain::ExtKeychain; +use util::LOGGER; +use wallet::libtx::slate::Slate; +use wallet::libwallet; + +fn clean_output_dir(test_dir: &str) { + let _ = fs::remove_dir_all(test_dir); +} + +fn setup(test_dir: &str) { + util::init_test_logger(); + clean_output_dir(test_dir); + global::set_mining_mode(ChainTypes::AutomatedTesting); +} + +fn restore_wallet( + base_dir: &str, + wallet_dir: &str, + backend_type: common::BackendType, +) -> Result<(), libwallet::Error> { + let source_seed = format!("{}/{}/wallet.seed", base_dir, wallet_dir); + let dest_dir = format!("{}/{}_restore", base_dir, wallet_dir); + fs::create_dir_all(dest_dir.clone())?; + let dest_seed = format!("{}/wallet.seed", dest_dir); + fs::copy(source_seed, dest_seed)?; + + let mut wallet_proxy: WalletProxy = WalletProxy::new(base_dir); + let client = LocalWalletClient::new(wallet_dir, wallet_proxy.tx.clone()); + + let wallet = common::create_wallet(&dest_dir, client.clone(), backend_type.clone()); + + wallet_proxy.add_wallet(wallet_dir, client.get_send_instance(), wallet.clone()); + + // Set the wallet proxy listener running + thread::spawn(move || { + if let Err(e) = wallet_proxy.run() { + error!(LOGGER, "Wallet Proxy error: {}", e); + } + }); + + // perform the restore and update wallet info + wallet::controller::owner_single_use(wallet.clone(), |api| { + let _ = api.restore()?; + let _ = api.retrieve_summary_info(true)?; + Ok(()) + })?; + + Ok(()) +} + +fn compare_wallet_restore( + base_dir: &str, + wallet_dir: &str, + backend_type: common::BackendType, +) -> Result<(), libwallet::Error> { + let restore_name = format!("{}_restore", wallet_dir); + let source_dir = format!("{}/{}", base_dir, wallet_dir); + let dest_dir = format!("{}/{}", base_dir, restore_name); + + let mut wallet_proxy: WalletProxy = WalletProxy::new(base_dir); + + let client = LocalWalletClient::new(wallet_dir, wallet_proxy.tx.clone()); + let wallet_source = common::create_wallet(&source_dir, client.clone(), backend_type.clone()); + wallet_proxy.add_wallet( + &wallet_dir, + client.get_send_instance(), + wallet_source.clone(), + ); + + let client = LocalWalletClient::new(&restore_name, wallet_proxy.tx.clone()); + let wallet_dest = common::create_wallet(&dest_dir, client.clone(), backend_type.clone()); + wallet_proxy.add_wallet( + &restore_name, + client.get_send_instance(), + wallet_dest.clone(), + ); + + // Set the wallet proxy listener running + thread::spawn(move || { + if let Err(e) = wallet_proxy.run() { + error!(LOGGER, "Wallet Proxy error: {}", e); + } + }); + + let mut src_info: Option = None; + let mut dest_info: Option = None; + + let mut src_txs: Option> = None; + let mut dest_txs: Option> = None; + + // Overall wallet info should be the same + wallet::controller::owner_single_use(wallet_source.clone(), |api| { + src_info = Some(api.retrieve_summary_info(true)?.1); + src_txs = Some(api.retrieve_txs(true, None)?.1); + Ok(()) + })?; + + wallet::controller::owner_single_use(wallet_dest.clone(), |api| { + dest_info = Some(api.retrieve_summary_info(true)?.1); + dest_txs = Some(api.retrieve_txs(true, None)?.1); + Ok(()) + })?; + + // Info should all be the same + assert_eq!(src_info, dest_info); + + // Net differences in TX logs should be the same + let src_sum: i64 = src_txs + .unwrap() + .iter() + .map(|t| t.amount_credited as i64 - t.amount_debited as i64) + .sum(); + + let dest_sum: i64 = dest_txs + .unwrap() + .iter() + .map(|t| t.amount_credited as i64 - t.amount_debited as i64) + .sum(); + + assert_eq!(src_sum, dest_sum); + + Ok(()) +} + +/// Build up 2 wallets, perform a few transactions on them +/// Then attempt to restore them in separate directories and check contents are the same +fn setup_restore( + test_dir: &str, + backend_type: common::BackendType, +) -> Result<(), libwallet::Error> { + setup(test_dir); + // Create a new proxy to simulate server and wallet responses + let mut wallet_proxy: WalletProxy = WalletProxy::new(test_dir); + let chain = wallet_proxy.chain.clone(); + + // Create a new wallet test client, and set its queues to communicate with the + // proxy + let client = LocalWalletClient::new("wallet1", wallet_proxy.tx.clone()); + let wallet1 = common::create_wallet( + &format!("{}/wallet1", test_dir), + client.clone(), + backend_type.clone(), + ); + wallet_proxy.add_wallet("wallet1", client.get_send_instance(), wallet1.clone()); + + // define recipient wallet, add to proxy + let client = LocalWalletClient::new("wallet2", wallet_proxy.tx.clone()); + let wallet2 = common::create_wallet( + &format!("{}/wallet2", test_dir), + client.clone(), + backend_type.clone(), + ); + wallet_proxy.add_wallet("wallet2", client.get_send_instance(), wallet2.clone()); + + // Another wallet + let client = LocalWalletClient::new("wallet3", wallet_proxy.tx.clone()); + let wallet3 = common::create_wallet( + &format!("{}/wallet3", test_dir), + client.clone(), + backend_type.clone(), + ); + wallet_proxy.add_wallet("wallet3", client.get_send_instance(), wallet3.clone()); + + // Set the wallet proxy listener running + thread::spawn(move || { + if let Err(e) = wallet_proxy.run() { + error!(LOGGER, "Wallet Proxy error: {}", e); + } + }); + + // mine a few blocks + let _ = common::award_blocks_to_wallet(&chain, wallet1.clone(), 10); + + // assert wallet contents + // and a single use api for a send command + let amount = 60_000_000_000; + let mut slate = Slate::blank(1); + wallet::controller::owner_single_use(wallet1.clone(), |sender_api| { + // note this will increment the block count as part of the transaction "Posting" + slate = sender_api.issue_send_tx( + amount, // amount + 2, // minimum confirmations + "wallet2", // dest + 500, // max outputs + 1, // num change outputs + true, // select all outputs + )?; + sender_api.post_tx(&slate, false)?; + Ok(()) + })?; + + // mine a few more blocks + let _ = common::award_blocks_to_wallet(&chain, wallet1.clone(), 3); + + // Send some to wallet 3 + wallet::controller::owner_single_use(wallet1.clone(), |sender_api| { + // note this will increment the block count as part of the transaction "Posting" + slate = sender_api.issue_send_tx( + amount * 2, // amount + 2, // minimum confirmations + "wallet3", // dest + 500, // max outputs + 1, // num change outputs + true, // select all outputs + )?; + sender_api.post_tx(&slate, false)?; + Ok(()) + })?; + + // mine a few more blocks + let _ = common::award_blocks_to_wallet(&chain, wallet3.clone(), 10); + + // Wallet3 to wallet 2 + wallet::controller::owner_single_use(wallet3.clone(), |sender_api| { + // note this will increment the block count as part of the transaction "Posting" + slate = sender_api.issue_send_tx( + amount * 3, // amount + 2, // minimum confirmations + "wallet2", // dest + 500, // max outputs + 1, // num change outputs + true, // select all outputs + )?; + sender_api.post_tx(&slate, false)?; + Ok(()) + })?; + + // mine a few more blocks + let _ = common::award_blocks_to_wallet(&chain, wallet1.clone(), 5); + + // update everyone + wallet::controller::owner_single_use(wallet1.clone(), |api| { + let _ = api.retrieve_summary_info(true)?; + Ok(()) + })?; + wallet::controller::owner_single_use(wallet2.clone(), |api| { + let _ = api.retrieve_summary_info(true)?; + Ok(()) + })?; + wallet::controller::owner_single_use(wallet3.clone(), |api| { + let _ = api.retrieve_summary_info(true)?; + Ok(()) + })?; + + Ok(()) +} + +fn perform_restore( + test_dir: &str, + backend_type: common::BackendType, +) -> Result<(), libwallet::Error> { + restore_wallet(test_dir, "wallet1", backend_type.clone())?; + compare_wallet_restore(test_dir, "wallet1", backend_type.clone())?; + restore_wallet(test_dir, "wallet2", backend_type.clone())?; + compare_wallet_restore(test_dir, "wallet2", backend_type.clone())?; + restore_wallet(test_dir, "wallet3", backend_type.clone())?; + compare_wallet_restore(test_dir, "wallet3", backend_type)?; + Ok(()) +} + +#[test] +fn db_wallet_restore() { + let test_dir = "test_output/wallet_restore_db"; + if let Err(e) = setup_restore(test_dir, common::BackendType::LMDBBackend) { + println!("Set up restore: Libwallet Error: {}", e); + } + if let Err(e) = perform_restore(test_dir, common::BackendType::LMDBBackend) { + println!("Perform restore: Libwallet Error: {}", e); + } + // let logging finish + thread::sleep(Duration::from_millis(200)); +}