Grin wallet check/repair (#2256)

* first pass at basic check_fix process

* rustfmt

* unlocks and tx log entry reversals in place

* log restore output at warn

* rename check_repair

* rename check_repair

* add command line functionality and sanity test

* rustfmt

* update wallet usage doc with check_repair

* doc update

* consistency in NotEnoughFunds output

* consistency in NotEnoughFunds output

doc/wallet/usage.md

@@ -403,11 +403,36 @@ grin wallet repost -i 3 -m tx_3.json
 
 This will create a file called tx_3.json containing your raw transaction data. Note that this formatting in the file isn't yet very user-readable.
 
+##### check_repair
+
+If for some reason the wallet cancel commands above don't work and you believe your outputs are in an inconsistent state, you have two options:
+
+First, you can try the `check_repair` command. This will scan the entire UTXO set from the node, identify which outputs are yours and update your wallet state to
+be consistent with what's currently in the UTXO set. This command will unlock all outputs, restore any missing outputs, and mark any outputs that have been marked
+'Spent' but are still in the UTXO set as 'Unspent' (as can happen during a fork). It will also attempt to cancel any transaction log entries associated with any locked outputs
+or outputs incorrectly marked 'Spent'
+
+For these reasons, you should be fairly sure that nobody will attempt to post any unconfirmed transactions involving your wallet before trying this command,
+(but even it someone does, it should be possible to re-run this command to fix any resulting issues.
+
+To attempt a repair, ensure a wallet listener isn't running, and enter:
+
+```sh
+grin wallet check_repair
+```
+
+The operation may take some time (it's advised to only perform this operation using a release build,) and it will report any inconsistencies it finds and repairs it makes.
+Once it's done, the state of your wallet outputs should match the contents of the UTXO set.
+
 ##### restore
 
-If for some reason the wallet cancel commands above don't work, you need to restore from a backed up `wallet.seed` file and password, or have recovered the wallet seed from a recovery phrase, you can perform a full wallet restore.
+If check_repair isn't working, or you need to restore your wallet from a backed up `wallet.seed` file and password, or have recovered the wallet seed from a recovery phrase,
+you can perform a full wallet restore.
 
-To do this, generate an empty wallet somewhere with:
+This command acts similarly to the check_repair command in that it scans the UTXO set for your outputs, however it will only restore found UTXOs into an empty wallet, 
+refusing to run if the wallet isn't empty.
+
+To restore a wallet, generate an empty wallet somewhere with:
 
 ```sh
 grin wallet init -h

src/bin/cmd/wallet_args.rs

@@ -504,6 +504,7 @@ pub fn wallet_command(
 			command::cancel(inst_wallet(), a)
 		}
 		("restore", Some(_)) => command::restore(inst_wallet()),
+		("check_repair", Some(_)) => command::check_repair(inst_wallet()),
 		_ => {
 			let msg = format!("Unknown wallet command, use 'grin help wallet' for details");
 			return Err(ErrorKind::ArgumentError(msg).into());

src/bin/cmd/wallet_tests.rs

@@ -443,6 +443,28 @@ mod wallet_tests {
 			Ok(())
 		})?;
 
+		// Another file exchange, don't send, but unlock with repair command
+		let arg_vec = vec![
+			"grin",
+			"wallet",
+			"-p",
+			"password",
+			"-a",
+			"mining",
+			"send",
+			"-m",
+			"file",
+			"-d",
+			&file_name,
+			"-g",
+			"Ain't sending",
+			"10",
+		];
+		execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?;
+
+		let arg_vec = vec!["grin", "wallet", "-p", "password", "check_repair"];
+		execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?;
+
 		// txs and outputs (mostly spit out for a visual in test logs)
 		let arg_vec = vec!["grin", "wallet", "-p", "password", "-a", "mining", "txs"];
 		execute_command(&app, test_dir, "wallet1", &client1, arg_vec)?;

src/bin/grin.yml

@@ -292,5 +292,7 @@ subcommands:
                   long: recovery_phrase
                   takes_value: true
         - restore:
-            about: Initialize a new wallet seed file and database
+            about: Restores a wallet contents from a seed file
+        - check_repair:
+            about: Checks a wallet's outputs against a live node, repairing and restoring missing outputs if required
  

wallet/src/command.rs

@@ -480,7 +480,7 @@ pub fn restore(
 		let result = api.restore();
 		match result {
 			Ok(_) => {
-				info!("Wallet restore complete",);
+				warn!("Wallet restore complete",);
 				Ok(())
 			}
 			Err(e) => {
@@ -492,3 +492,23 @@ pub fn restore(
 	})?;
 	Ok(())
 }
+
+pub fn check_repair(
+	wallet: Arc<Mutex<WalletInst<impl NodeClient + 'static, keychain::ExtKeychain>>>,
+) -> Result<(), Error> {
+	controller::owner_single_use(wallet.clone(), |api| {
+		let result = api.check_repair();
+		match result {
+			Ok(_) => {
+				warn!("Wallet check/repair complete",);
+				Ok(())
+			}
+			Err(e) => {
+				error!("Wallet check/repair failed: {}", e);
+				error!("Backtrace: {}", e.backtrace().unwrap());
+				Err(e)
+			}
+		}
+	})?;
+	Ok(())
+}

wallet/src/libwallet/api.rs

@@ -353,7 +353,7 @@ where
 
 		let res = Ok((
 			validated,
-			updater::retrieve_outputs(&mut *w, include_spent, tx_id, &parent_key_id)?,
+			updater::retrieve_outputs(&mut *w, include_spent, tx_id, Some(&parent_key_id))?,
 		));
 
 		w.close()?;
@@ -756,9 +756,19 @@ where
 	pub fn restore(&mut self) -> Result<(), Error> {
 		let mut w = self.wallet.lock();
 		w.open_with_credentials()?;
-		let res = w.restore();
+		w.restore()?;
 		w.close()?;
-		res
+		Ok(())
+	}
+
+	/// Attempt to check and fix the contents of the wallet
+	pub fn check_repair(&mut self) -> Result<(), Error> {
+		let mut w = self.wallet.lock();
+		w.open_with_credentials()?;
+		self.update_outputs(&mut w);
+		w.check_repair()?;
+		w.close()?;
+		Ok(())
 	}
 
 	/// Retrieve current height from node

wallet/src/libwallet/error.rs

@@ -34,37 +34,49 @@ pub enum ErrorKind {
 	/// Not enough funds
 	#[fail(
 		display = "Not enough funds. Required: {}, Available: {}",
-		needed, available
+		needed_disp, available_disp
 	)]
 	NotEnoughFunds {
 		/// available funds
 		available: u64,
+		/// Display friendly
+		available_disp: String,
 		/// Needed funds
 		needed: u64,
+		/// Display friendly
+		needed_disp: String,
 	},
 
 	/// Fee dispute
 	#[fail(
 		display = "Fee dispute: sender fee {}, recipient fee {}",
-		sender_fee, recipient_fee
+		sender_fee_disp, recipient_fee_disp
 	)]
 	FeeDispute {
 		/// sender fee
 		sender_fee: u64,
+		/// display friendly
+		sender_fee_disp: String,
 		/// recipient fee
 		recipient_fee: u64,
+		/// display friendly
+		recipient_fee_disp: String,
 	},
 
 	/// Fee Exceeds amount
 	#[fail(
 		display = "Fee exceeds amount: sender amount {}, recipient fee {}",
-		sender_amount, recipient_fee
+		sender_amount_disp, recipient_fee
 	)]
 	FeeExceedsAmount {
 		/// sender amount
 		sender_amount: u64,
+		/// display friendly
+		sender_amount_disp: String,
 		/// recipient fee
 		recipient_fee: u64,
+		/// display friendly
+		recipient_fee_disp: String,
 	},
 
 	/// LibTX Error

wallet/src/libwallet/internal/restore.rs

@@ -16,13 +16,14 @@
 use crate::core::global;
 use crate::core::libtx::proof;
 use crate::keychain::{ExtKeychain, Identifier, Keychain};
-use crate::libwallet::internal::keys;
+use crate::libwallet::internal::{keys, updater};
 use crate::libwallet::types::*;
 use crate::libwallet::Error;
 use crate::util::secp::{key::SecretKey, pedersen};
 use std::collections::HashMap;
 
 /// Utility struct for return values from below
+#[derive(Clone)]
 struct OutputResult {
 	///
 	pub commit: pedersen::Commitment,
@@ -53,7 +54,7 @@ where
 {
 	let mut wallet_outputs: Vec<OutputResult> = Vec::new();
 
-	info!(
+	warn!(
 		"Scanning {} outputs in the current Grin utxo set",
 		outputs.len(),
 	);
@@ -97,6 +98,212 @@ where
 	Ok(wallet_outputs)
 }
 
+fn collect_chain_outputs<T, C, K>(wallet: &mut T) -> Result<Vec<OutputResult>, Error>
+where
+	T: WalletBackend<C, K>,
+	C: NodeClient,
+	K: Keychain,
+{
+	let batch_size = 1000;
+	let mut start_index = 1;
+	let mut result_vec: Vec<OutputResult> = vec![];
+	loop {
+		let (highest_index, last_retrieved_index, outputs) = wallet
+			.w2n_client()
+			.get_outputs_by_pmmr_index(start_index, batch_size)?;
+		warn!(
+			"Checking {} outputs, up to index {}. (Highest index: {})",
+			outputs.len(),
+			highest_index,
+			last_retrieved_index,
+		);
+
+		result_vec.append(&mut identify_utxo_outputs(wallet, outputs.clone())?);
+
+		if highest_index == last_retrieved_index {
+			break;
+		}
+		start_index = last_retrieved_index + 1;
+	}
+	Ok(result_vec)
+}
+
+///
+fn restore_missing_output<T, C, K>(
+	wallet: &mut T,
+	output: OutputResult,
+	found_parents: &mut HashMap<Identifier, u32>,
+) -> Result<(), Error>
+where
+	T: WalletBackend<C, K>,
+	C: NodeClient,
+	K: Keychain,
+{
+	let mut batch = wallet.batch()?;
+
+	let parent_key_id = output.key_id.parent_path();
+	if !found_parents.contains_key(&parent_key_id) {
+		found_parents.insert(parent_key_id.clone(), 0);
+	}
+
+	let log_id = batch.next_tx_log_id(&parent_key_id)?;
+	let entry_type = match output.is_coinbase {
+		true => TxLogEntryType::ConfirmedCoinbase,
+		false => TxLogEntryType::TxReceived,
+	};
+
+	let mut t = TxLogEntry::new(parent_key_id.clone(), entry_type, log_id);
+	t.confirmed = true;
+	t.amount_credited = output.value;
+	t.num_outputs = 1;
+	t.update_confirmation_ts();
+	batch.save_tx_log_entry(t, &parent_key_id)?;
+
+	let _ = batch.save(OutputData {
+		root_key_id: parent_key_id.clone(),
+		key_id: output.key_id,
+		n_child: output.n_child,
+		value: output.value,
+		status: OutputStatus::Unspent,
+		height: output.height,
+		lock_height: output.lock_height,
+		is_coinbase: output.is_coinbase,
+		tx_log_entry: Some(log_id),
+	});
+
+	let max_child_index = found_parents.get(&parent_key_id).unwrap().clone();
+	if output.n_child >= max_child_index {
+		found_parents.insert(parent_key_id.clone(), output.n_child);
+	}
+
+	batch.commit()?;
+	Ok(())
+}
+
+///
+fn cancel_tx_log_entry<T, C, K>(wallet: &mut T, output: &OutputData) -> Result<(), Error>
+where
+	T: WalletBackend<C, K>,
+	C: NodeClient,
+	K: Keychain,
+{
+	let parent_key_id = output.key_id.parent_path();
+	let updated_tx_entry = if output.tx_log_entry.is_some() {
+		let entries = updater::retrieve_txs(
+			wallet,
+			output.tx_log_entry.clone(),
+			None,
+			Some(&parent_key_id),
+		)?;
+		if entries.len() > 0 {
+			let mut entry = entries[0].clone();
+			match entry.tx_type {
+				TxLogEntryType::TxSent => entry.tx_type = TxLogEntryType::TxSentCancelled,
+				TxLogEntryType::TxReceived => entry.tx_type = TxLogEntryType::TxReceivedCancelled,
+				_ => {}
+			}
+			Some(entry)
+		} else {
+			None
+		}
+	} else {
+		None
+	};
+	let mut batch = wallet.batch()?;
+	if let Some(t) = updated_tx_entry {
+		batch.save_tx_log_entry(t, &parent_key_id)?;
+	}
+	batch.commit()?;
+	Ok(())
+}
+
+/// Check / repair wallet contents
+/// assume wallet contents have been freshly updated with contents
+/// of latest block
+pub fn check_repair<T, C, K>(wallet: &mut T) -> Result<(), Error>
+where
+	T: WalletBackend<C, K>,
+	C: NodeClient,
+	K: Keychain,
+{
+	// First, get a definitive list of outputs we own from the chain
+	warn!("Starting wallet check.");
+	let chain_outs = collect_chain_outputs(wallet)?;
+	warn!(
+		"Identified {} wallet_outputs as belonging to this wallet",
+		chain_outs.len(),
+	);
+
+	// Now, get all outputs owned by this wallet (regardless of account)
+	let wallet_outputs = {
+		let res = updater::retrieve_outputs(&mut *wallet, true, None, None)?;
+		res
+	};
+
+	// check all definitive outputs exist in the wallet outputs
+	let mut missing_outs = vec![];
+	let mut accidental_spend_outs = vec![];
+	let mut locked_outs = vec![];
+	for deffo in chain_outs.into_iter() {
+		let matched_out = wallet_outputs.iter().find(|wo| wo.0.key_id == deffo.key_id);
+		match matched_out {
+			Some(s) => {
+				if s.0.status == OutputStatus::Spent {
+					accidental_spend_outs.push((s.0.clone(), deffo.clone()));
+				}
+				if s.0.status == OutputStatus::Locked {
+					locked_outs.push((s.0.clone(), deffo));
+				}
+			}
+			None => missing_outs.push(deffo),
+		}
+	}
+
+	// mark problem spent outputs as unspent (confirmed against a short-lived fork, for example)
+	for m in accidental_spend_outs.into_iter() {
+		let mut o = m.0;
+		warn!(
+			"Output for {} with ID {} ({:?}) marked as spent but exists in UTXO set. \
+			 Marking unspent and cancelling any associated transaction log entries.",
+			o.value, o.key_id, m.1.commit,
+		);
+		o.status = OutputStatus::Unspent;
+		// any transactions associated with this should be cancelled
+		cancel_tx_log_entry(wallet, &o)?;
+		let mut batch = wallet.batch()?;
+		batch.save(o)?;
+		batch.commit()?;
+	}
+
+	let mut found_parents: HashMap<Identifier, u32> = HashMap::new();
+
+	// Restore missing outputs, adding transaction for it back to the log
+	for m in missing_outs.into_iter() {
+		warn!(
+			"Confirmed output for {} with ID {} ({:?}) exists in UTXO set but not in wallet. \
+			 Restoring.",
+			m.value, m.key_id, m.commit,
+		);
+		restore_missing_output(wallet, m, &mut found_parents)?;
+	}
+
+	for m in locked_outs.into_iter() {
+		let mut o = m.0;
+		warn!(
+			"Confirmed output for {} with ID {} ({:?}) exists in UTXO set and is locked. \
+			 Unlocking and cancelling associated transaction log entries.",
+			o.value, o.key_id, m.1.commit,
+		);
+		o.status = OutputStatus::Unspent;
+		cancel_tx_log_entry(wallet, &o)?;
+		let mut batch = wallet.batch()?;
+		batch.save(o)?;
+		batch.commit()?;
+	}
+
+	Ok(())
+}
+
 /// Restore a wallet
 pub fn restore<T, C, K>(wallet: &mut T) -> Result<(), Error>
 where
@@ -111,78 +318,22 @@ where
 		return Ok(());
 	}
 
-	info!("Starting restore.");
+	warn!("Starting restore.");
 
-	let batch_size = 1000;
+	let result_vec = collect_chain_outputs(wallet)?;
-	let mut start_index = 1;
-	let mut result_vec: Vec<OutputResult> = vec![];
-	loop {
-		let (highest_index, last_retrieved_index, outputs) = wallet
-			.w2n_client()
-			.get_outputs_by_pmmr_index(start_index, batch_size)?;
-		info!(
-			"Retrieved {} outputs, up to index {}. (Highest index: {})",
-			outputs.len(),
-			highest_index,
-			last_retrieved_index,
-		);
 
-		result_vec.append(&mut identify_utxo_outputs(wallet, outputs.clone())?);
+	warn!(
-
-		if highest_index == last_retrieved_index {
-			break;
-		}
-		start_index = last_retrieved_index + 1;
-	}
-
-	info!(
 		"Identified {} wallet_outputs as belonging to this wallet",
 		result_vec.len(),
 	);
 
 	let mut found_parents: HashMap<Identifier, u32> = HashMap::new();
 	// Now save what we have
-	{
-		let mut batch = wallet.batch()?;
 
-		for output in result_vec {
+	for output in result_vec {
-			let parent_key_id = output.key_id.parent_path();
+		restore_missing_output(wallet, output, &mut found_parents)?;
-			if !found_parents.contains_key(&parent_key_id) {
-				found_parents.insert(parent_key_id.clone(), 0);
-			}
-
-			let log_id = batch.next_tx_log_id(&parent_key_id)?;
-			let entry_type = match output.is_coinbase {
-				true => TxLogEntryType::ConfirmedCoinbase,
-				false => TxLogEntryType::TxReceived,
-			};
-
-			let mut t = TxLogEntry::new(parent_key_id.clone(), entry_type, log_id);
-			t.confirmed = true;
-			t.amount_credited = output.value;
-			t.num_outputs = 1;
-			t.update_confirmation_ts();
-			batch.save_tx_log_entry(t, &parent_key_id)?;
-
-			let _ = batch.save(OutputData {
-				root_key_id: parent_key_id.clone(),
-				key_id: output.key_id,
-				n_child: output.n_child,
-				value: output.value,
-				status: OutputStatus::Unspent,
-				height: output.height,
-				lock_height: output.lock_height,
-				is_coinbase: output.is_coinbase,
-				tx_log_entry: Some(log_id),
-			});
-
-			let max_child_index = found_parents.get(&parent_key_id).unwrap().clone();
-			if output.n_child >= max_child_index {
-				found_parents.insert(parent_key_id.clone(), output.n_child);
-			};
-		}
-		batch.commit()?;
 	}
+
 	// restore labels, account paths and child derivation indices
 	let label_base = "account";
 	let mut index = 1;

wallet/src/libwallet/internal/selection.rs

@@ -14,7 +14,7 @@
 
 //! Selection of inputs for building transactions
 
-use crate::core::core::Transaction;
+use crate::core::core::{amount_to_hr_string, Transaction};
 use crate::core::libtx::{build, slate::Slate, tx_fee};
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::error::{Error, ErrorKind};
@@ -266,7 +266,9 @@ where
 	if total == 0 {
 		return Err(ErrorKind::NotEnoughFunds {
 			available: 0,
+			available_disp: amount_to_hr_string(0, false),
 			needed: amount_with_fee as u64,
+			needed_disp: amount_to_hr_string(amount_with_fee as u64, false),
 		})?;
 	}
 
@@ -274,7 +276,9 @@ where
 	if total < amount_with_fee && coins.len() == max_outputs {
 		return Err(ErrorKind::NotEnoughFunds {
 			available: total,
+			available_disp: amount_to_hr_string(total, false),
 			needed: amount_with_fee as u64,
+			needed_disp: amount_to_hr_string(amount_with_fee as u64, false),
 		})?;
 	}
 
@@ -292,7 +296,9 @@ where
 			if coins.len() == max_outputs {
 				return Err(ErrorKind::NotEnoughFunds {
 					available: total as u64,
+					available_disp: amount_to_hr_string(total, false),
 					needed: amount_with_fee as u64,
+					needed_disp: amount_to_hr_string(amount_with_fee as u64, false),
 				})?;
 			}
 

wallet/src/libwallet/internal/tx.rs

@@ -173,7 +173,7 @@ where
 		return Err(ErrorKind::TransactionNotCancellable(tx_id_string))?;
 	}
 	// get outputs associated with tx
-	let res = updater::retrieve_outputs(wallet, false, Some(tx.id), &parent_key_id)?;
+	let res = updater::retrieve_outputs(wallet, false, Some(tx.id), Some(&parent_key_id))?;
 	let outputs = res.iter().map(|(out, _)| out).cloned().collect();
 	updater::cancel_tx_and_outputs(wallet, tx, outputs, parent_key_id)?;
 	Ok(())

wallet/src/libwallet/internal/updater.rs

@@ -39,7 +39,7 @@ pub fn retrieve_outputs<T: ?Sized, C, K>(
 	wallet: &mut T,
 	show_spent: bool,
 	tx_id: Option<u32>,
-	parent_key_id: &Identifier,
+	parent_key_id: Option<&Identifier>,
 ) -> Result<Vec<(OutputData, pedersen::Commitment)>, Error>
 where
 	T: WalletBackend<C, K>,
@@ -49,7 +49,6 @@ where
 	// just read the wallet here, no need for a write lock
 	let mut outputs = wallet
 		.iter()
-		.filter(|out| out.root_key_id == *parent_key_id)
 		.filter(|out| {
 			if show_spent {
 				true
@@ -63,10 +62,18 @@ where
 	if let Some(id) = tx_id {
 		outputs = outputs
 			.into_iter()
-			.filter(|out| out.tx_log_entry == Some(id) && out.root_key_id == *parent_key_id)
+			.filter(|out| out.tx_log_entry == Some(id))
 			.collect::<Vec<_>>();
 	}
 
+	if let Some(k) = parent_key_id {
+		outputs = outputs
+			.iter()
+			.filter(|o| o.root_key_id == *k)
+			.map(|o| o.clone())
+			.collect();
+	}
+
 	outputs.sort_by_key(|out| out.n_child);
 
 	let res = outputs

wallet/src/libwallet/types.rs

@@ -118,6 +118,9 @@ where
 
 	/// Attempt to restore the contents of a wallet from seed
 	fn restore(&mut self) -> Result<(), Error>;
+
+	/// Attempt to check and fix wallet state
+	fn check_repair(&mut self) -> Result<(), Error>;
 }
 
 /// Batch trait to update the output data backend atomically. Trying to use a
@@ -562,7 +565,7 @@ impl fmt::Display for TxLogEntryType {
 			TxLogEntryType::TxReceived => write!(f, "Received Tx"),
 			TxLogEntryType::TxSent => write!(f, "Sent Tx"),
 			TxLogEntryType::TxReceivedCancelled => write!(f, "Received Tx\n- Cancelled"),
-			TxLogEntryType::TxSentCancelled => write!(f, "Send Tx\n- Cancelled"),
+			TxLogEntryType::TxSentCancelled => write!(f, "Sent Tx\n- Cancelled"),
 		}
 	}
 }
@@ -637,6 +640,14 @@ impl TxLogEntry {
 		}
 	}
 
+	/// Given a vec of TX log entries, return credited + debited sums
+	pub fn sum_confirmed(txs: &Vec<TxLogEntry>) -> (u64, u64) {
+		txs.iter().fold((0, 0), |acc, tx| match tx.confirmed {
+			true => (acc.0 + tx.amount_credited, acc.1 + tx.amount_debited),
+			false => acc,
+		})
+	}
+
 	/// Update confirmation TS with now
 	pub fn update_confirmation_ts(&mut self) {
 		self.confirmation_ts = Some(Utc::now());

wallet/src/lmdb_wallet.rs

@@ -369,6 +369,11 @@ where
 		internal::restore::restore(self).context(ErrorKind::Restore)?;
 		Ok(())
 	}
+
+	fn check_repair(&mut self) -> Result<(), Error> {
+		internal::restore::check_repair(self).context(ErrorKind::Restore)?;
+		Ok(())
+	}
 }
 
 /// An atomic batch in which all changes can be committed all at once or

wallet/tests/check.rs

@@ -0,0 +1,202 @@
+// Copyright 2018 The Grin Developers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! tests differing accounts in the same wallet
+#[macro_use]
+extern crate log;
+
+use self::core::global;
+use self::core::global::ChainTypes;
+use self::keychain::ExtKeychain;
+use self::wallet::test_framework::{self, LocalWalletClient, WalletProxy};
+use self::wallet::{libwallet, FileWalletCommAdapter};
+use grin_core as core;
+use grin_keychain as keychain;
+use grin_util as util;
+use grin_wallet as wallet;
+use std::fs;
+use std::thread;
+use std::time::Duration;
+
+fn clean_output_dir(test_dir: &str) {
+	let _ = fs::remove_dir_all(test_dir);
+}
+
+fn setup(test_dir: &str) {
+	util::init_test_logger();
+	clean_output_dir(test_dir);
+	global::set_mining_mode(ChainTypes::AutomatedTesting);
+}
+
+/// Various tests on accounts within the same wallet
+fn check_repair_impl(test_dir: &str) -> Result<(), libwallet::Error> {
+	setup(test_dir);
+	// Create a new proxy to simulate server and wallet responses
+	let mut wallet_proxy: WalletProxy<LocalWalletClient, ExtKeychain> = WalletProxy::new(test_dir);
+	let chain = wallet_proxy.chain.clone();
+
+	// Create a new wallet test client, and set its queues to communicate with the
+	// proxy
+	let client1 = LocalWalletClient::new("wallet1", wallet_proxy.tx.clone());
+	let wallet1 = test_framework::create_wallet(&format!("{}/wallet1", test_dir), client1.clone());
+	wallet_proxy.add_wallet("wallet1", client1.get_send_instance(), wallet1.clone());
+
+	let client2 = LocalWalletClient::new("wallet2", wallet_proxy.tx.clone());
+	// define recipient wallet, add to proxy
+	let wallet2 = test_framework::create_wallet(&format!("{}/wallet2", test_dir), client2.clone());
+	wallet_proxy.add_wallet("wallet2", client2.get_send_instance(), wallet2.clone());
+
+	// Set the wallet proxy listener running
+	thread::spawn(move || {
+		if let Err(e) = wallet_proxy.run() {
+			error!("Wallet Proxy error: {}", e);
+		}
+	});
+
+	// few values to keep things shorter
+	let reward = core::consensus::REWARD;
+	let cm = global::coinbase_maturity(); // assume all testing precedes soft fork height
+
+	// add some accounts
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		api.create_account_path("account_1")?;
+		api.create_account_path("account_2")?;
+		api.create_account_path("account_3")?;
+		api.set_active_account("account_1")?;
+		Ok(())
+	})?;
+
+	// add account to wallet 2
+	wallet::controller::owner_single_use(wallet2.clone(), |api| {
+		api.create_account_path("account_1")?;
+		api.set_active_account("account_1")?;
+		Ok(())
+	})?;
+
+	// Do some mining
+	let bh = 20u64;
+	let _ = test_framework::award_blocks_to_wallet(&chain, wallet1.clone(), bh as usize);
+
+	// Sanity check contents
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		let (wallet1_refreshed, wallet1_info) = api.retrieve_summary_info(true, 1)?;
+		assert!(wallet1_refreshed);
+		assert_eq!(wallet1_info.last_confirmed_height, bh);
+		assert_eq!(wallet1_info.total, bh * reward);
+		assert_eq!(wallet1_info.amount_currently_spendable, (bh - cm) * reward);
+		// check tx log as well
+		let (_, txs) = api.retrieve_txs(true, None, None)?;
+		let (c, _) = libwallet::types::TxLogEntry::sum_confirmed(&txs);
+		assert_eq!(wallet1_info.total, c);
+		assert_eq!(txs.len(), bh as usize);
+		Ok(())
+	})?;
+
+	// Accidentally delete some outputs
+	let mut w1_outputs_commits = vec![];
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		w1_outputs_commits = api.retrieve_outputs(false, true, None)?.1;
+		Ok(())
+	})?;
+	let w1_outputs: Vec<libwallet::types::OutputData> =
+		w1_outputs_commits.into_iter().map(|o| o.0).collect();
+	{
+		let mut w = wallet1.lock();
+		w.open_with_credentials()?;
+		{
+			let mut batch = w.batch()?;
+			batch.delete(&w1_outputs[4].key_id)?;
+			batch.delete(&w1_outputs[10].key_id)?;
+			let mut accidental_spent = w1_outputs[13].clone();
+			accidental_spent.status = libwallet::types::OutputStatus::Spent;
+			batch.save(accidental_spent)?;
+			batch.commit()?;
+		}
+		w.close()?;
+	}
+
+	// check we have a problem now
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		let (_, wallet1_info) = api.retrieve_summary_info(true, 1)?;
+		let (_, txs) = api.retrieve_txs(true, None, None)?;
+		let (c, _) = libwallet::types::TxLogEntry::sum_confirmed(&txs);
+		assert!(wallet1_info.total != c);
+		Ok(())
+	})?;
+
+	// this should restore our missing outputs
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		api.check_repair()?;
+		Ok(())
+	})?;
+
+	// check our outputs match again
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		let (wallet1_refreshed, wallet1_info) = api.retrieve_summary_info(true, 1)?;
+		assert!(wallet1_refreshed);
+		assert_eq!(wallet1_info.total, bh * reward);
+		Ok(())
+	})?;
+
+	// perform a transaction, but don't let it finish
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		// send to send
+		let (mut slate, lock_fn) = api.initiate_tx(
+			None,
+			reward * 2, // amount
+			cm,         // minimum confirmations
+			500,        // max outputs
+			1,          // num change outputs
+			true,       // select all outputs
+			None,       // optional message
+		)?;
+		// output tx file
+		let file_adapter = FileWalletCommAdapter::new();
+		let send_file = format!("{}/part_tx_1.tx", test_dir);
+		file_adapter.send_tx_async(&send_file, &mut slate)?;
+		api.tx_lock_outputs(&slate, lock_fn)?;
+		Ok(())
+	})?;
+
+	// check we're all locked
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		let (_, wallet1_info) = api.retrieve_summary_info(true, 1)?;
+		assert!(wallet1_info.amount_currently_spendable == 0);
+		Ok(())
+	})?;
+
+	// unlock/restore
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		api.check_repair()?;
+		Ok(())
+	})?;
+
+	// check spendable amount again
+	wallet::controller::owner_single_use(wallet1.clone(), |api| {
+		let (_, wallet1_info) = api.retrieve_summary_info(true, 1)?;
+		assert_eq!(wallet1_info.amount_currently_spendable, (bh - cm) * reward);
+		Ok(())
+	})?;
+
+	// let logging finish
+	thread::sleep(Duration::from_millis(200));
+	Ok(())
+}
+
+#[test]
+fn check_repair() {
+	let test_dir = "test_output/check_repair";
+	if let Err(e) = check_repair_impl(test_dir) {
+		panic!("Libwallet Error: {} - {}", e, e.backtrace().unwrap());
+	}
+}