From 4b3a374d98308fa4430b3aabb6a619135c85ab0b Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Mon, 20 Nov 2017 00:50:09 +0000 Subject: [PATCH] Wallet Restore feature (#338) * beginning to add wallet restore... api endpoints and basic restore * basic restore working, still missing features * rustfmt * large speed up to output search, should be more or less working * properly mark coinbase status --- api/src/handlers.rs | 102 +++++++++++++++--- api/src/types.rs | 73 ++++++++++++- core/src/core/transaction.rs | 3 +- keychain/src/keychain.rs | 8 ++ src/bin/grin.rs | 22 +++- wallet/src/checker.rs | 2 +- wallet/src/lib.rs | 2 + wallet/src/restore.rs | 202 +++++++++++++++++++++++++++++++++++ 8 files changed, 388 insertions(+), 26 deletions(-) create mode 100644 wallet/src/restore.rs diff --git a/api/src/handlers.rs b/api/src/handlers.rs index 7a6ae4580..69cecfe4d 100644 --- a/api/src/handlers.rs +++ b/api/src/handlers.rs @@ -25,6 +25,7 @@ use serde_json; use chain; use core::core::Transaction; +use core::core::hash::Hashed; use core::ser; use pool; use p2p; @@ -51,35 +52,39 @@ impl Handler for IndexHandler { } // Supports retrieval of multiple outputs in a single request - -// GET /v1/chain/utxos?id=xxx,yyy,zzz -// GET /v1/chain/utxos?id=xxx&id=yyy&id=zzz +// GET /v1/chain/utxos/byids?id=xxx,yyy,zzz +// GET /v1/chain/utxos/byids?id=xxx&id=yyy&id=zzz +// GET /v1/chain/utxos/byheight?height=n struct UtxoHandler { chain: Arc, } impl UtxoHandler { - fn get_utxo(&self, id: &str) -> Result { + fn get_utxo(&self, id: &str, include_rp: bool, include_switch: bool) -> Result { debug!(LOGGER, "getting utxo: {}", id); let c = util::from_hex(String::from(id)).map_err(|_| { Error::Argument(format!("Not a valid commitment: {}", id)) })?; let commit = Commitment::from_vec(c); - let out = self.chain - .get_unspent(&commit) - .map_err(|_| Error::NotFound)?; + let out = self.chain.get_unspent(&commit).map_err(|_| Error::NotFound)?; let header = self.chain .get_block_header_by_output_commit(&commit) .map_err(|_| Error::NotFound)?; - Ok(Output::from_output(&out, &header, false)) + Ok(Output::from_output( + &out, + &header, + include_rp, + include_switch, + )) } -} -impl Handler for UtxoHandler { - fn handle(&self, req: &mut Request) -> IronResult { + fn utxos_by_ids(&self, req: &mut Request) -> Vec { let mut commitments: Vec<&str> = vec![]; + let mut rp = false; + let mut switch = false; if let Ok(params) = req.get_ref::() { if let Some(ids) = params.get("id") { for id in ids { @@ -88,15 +93,75 @@ impl Handler for UtxoHandler { } } } + if let Some(_) = params.get("include_rp") { + rp = true; + } + if let Some(_) = params.get("include_switch") { + switch = true; + } } - let mut utxos: Vec = vec![]; for commit in commitments { - if let Ok(out) = self.get_utxo(commit) { + if let Ok(out) = self.get_utxo(commit, rp, switch) { utxos.push(out); } } - json_response(&utxos) + utxos + } + + fn utxos_at_height(&self, block_height: u64) -> BlockOutputs { + let header = self.chain + .clone() + .get_header_by_height(block_height) + .unwrap(); + let block = self.chain.clone().get_block(&header.hash()).unwrap(); + let outputs = block + .outputs + .iter() + .map(|k| OutputSwitch::from_output(k, &header)) + .collect(); + BlockOutputs { + header: BlockHeaderInfo::from_header(&header), + outputs: outputs, + } + } + + // returns utxos for a specified range of blocks + fn utxo_block_batch(&self, req: &mut Request) -> Vec { + let mut start_height = 1; + let mut end_height = 1; + if let Ok(params) = req.get_ref::() { + if let Some(heights) = params.get("start_height") { + for height in heights { + start_height = height.parse().unwrap(); + } + } + if let Some(heights) = params.get("end_height") { + for height in heights { + end_height = height.parse().unwrap(); + } + } + } + let mut return_vec = vec![]; + for i in start_height..end_height + 1 { + return_vec.push(self.utxos_at_height(i)); + } + return_vec + } +} + +impl Handler for UtxoHandler { + fn handle(&self, req: &mut Request) -> IronResult { + let url = req.url.clone(); + let mut path_elems = url.path(); + if *path_elems.last().unwrap() == "" { + path_elems.pop(); + } + match *path_elems.last().unwrap() { + "byids" => json_response(&self.utxos_by_ids(req)), + "atheight" => json_response(&self.utxo_block_batch(req)), + _ => Ok(Response::with((status::BadRequest, ""))), + } } } @@ -242,15 +307,18 @@ where T: pool::BlockChain + Send + Sync + 'static, { fn handle(&self, req: &mut Request) -> IronResult { - let wrapper: TxWrapper = serde_json::from_reader(req.body.by_ref()) - .map_err(|e| IronError::new(e, status::BadRequest))?; + let wrapper: TxWrapper = serde_json::from_reader(req.body.by_ref()).map_err(|e| { + IronError::new(e, status::BadRequest) + })?; let tx_bin = util::from_hex(wrapper.tx_hex).map_err(|_| { Error::Argument(format!("Invalid hex in transaction wrapper.")) })?; let tx: Transaction = ser::deserialize(&mut &tx_bin[..]).map_err(|_| { - Error::Argument("Could not deserialize transaction, invalid format.".to_string()) + Error::Argument( + "Could not deserialize transaction, invalid format.".to_string(), + ) })?; let source = pool::TxSource { @@ -349,7 +417,7 @@ pub fn start_rest_apis( let router = router!( index: get "/" => index_handler, chain_tip: get "/chain" => chain_tip_handler, - chain_utxos: get "/chain/utxos" => utxo_handler, + chain_utxos: get "/chain/utxos/*" => utxo_handler, sumtree_roots: get "/sumtrees/*" => sumtree_handler, pool_info: get "/pool" => pool_info_handler, pool_push: post "/pool/push" => pool_push_handler, diff --git a/api/src/types.rs b/api/src/types.rs index 1f76e3a90..bce547d8d 100644 --- a/api/src/types.rs +++ b/api/src/types.rs @@ -88,7 +88,7 @@ impl SumTreeNode { .get_block_header_by_output_commit(&elem_output.1.commit) .map_err(|_| Error::NotFound); // Need to call further method to check if output is spent - let mut output = OutputPrintable::from_output(&elem_output.1, &header.unwrap()); + let mut output = OutputPrintable::from_output(&elem_output.1, &header.unwrap(),true); if let Ok(_) = chain.get_unspent(&elem_output.1.commit) { output.spent = false; } @@ -137,6 +137,8 @@ pub struct Output { pub output_type: OutputType, /// The homomorphic commitment representing the output's amount pub commit: pedersen::Commitment, + /// switch commit hash + pub switch_commit_hash: Option, /// A proof that the commitment is in the right range pub proof: Option, /// The height of the block creating this output @@ -146,7 +148,8 @@ pub struct Output { } impl Output { - pub fn from_output(output: &core::Output, block_header: &core::BlockHeader, include_proof:bool) -> Output { + pub fn from_output(output: &core::Output, block_header: &core::BlockHeader, + include_proof:bool, include_switch: bool) -> Output { let (output_type, lock_height) = match output.features { x if x.contains(core::transaction::COINBASE_OUTPUT) => ( OutputType::Coinbase, @@ -158,6 +161,10 @@ impl Output { Output { output_type: output_type, commit: output.commit, + switch_commit_hash: match include_switch { + true => Some(output.switch_commit_hash), + false => None, + }, proof: match include_proof { true => Some(output.proof), false => None, @@ -176,6 +183,8 @@ pub struct OutputPrintable { /// The homomorphic commitment representing the output's amount (as hex /// string) pub commit: String, + /// switch commit hash + pub switch_commit_hash: String, /// The height of the block creating this output pub height: u64, /// The lock height (earliest block this output can be spent) @@ -183,11 +192,11 @@ pub struct OutputPrintable { /// Whether the output has been spent pub spent: bool, /// Rangeproof hash (as hex string) - pub proof_hash: String, + pub proof_hash: Option, } impl OutputPrintable { - pub fn from_output(output: &core::Output, block_header: &core::BlockHeader) -> OutputPrintable { + pub fn from_output(output: &core::Output, block_header: &core::BlockHeader, include_proof_hash:bool) -> OutputPrintable { let (output_type, lock_height) = match output.features { x if x.contains(core::transaction::COINBASE_OUTPUT) => ( OutputType::Coinbase, @@ -198,14 +207,68 @@ impl OutputPrintable { OutputPrintable { output_type: output_type, commit: util::to_hex(output.commit.0.to_vec()), + switch_commit_hash: util::to_hex(output.switch_commit_hash.hash.to_vec()), height: block_header.height, lock_height: lock_height, spent: true, - proof_hash: util::to_hex(output.proof.hash().to_vec()), + proof_hash: match include_proof_hash { + true => Some(util::to_hex(output.proof.hash().to_vec())), + false => None, + } } } } +// As above, except just the info needed for wallet reconstruction +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct OutputSwitch { + /// the commit + pub commit: String, + /// switch commit hash + pub switch_commit_hash: [u8; core::SWITCH_COMMIT_HASH_SIZE], + /// The height of the block creating this output + pub height: u64, +} + +impl OutputSwitch { + pub fn from_output(output: &core::Output, block_header: &core::BlockHeader) -> OutputSwitch { + OutputSwitch { + commit: util::to_hex(output.commit.0.to_vec()), + switch_commit_hash: output.switch_commit_hash.hash, + height: block_header.height, + } + } +} +// Just the information required for wallet reconstruction +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct BlockHeaderInfo { + /// Hash + pub hash: String, + /// Previous block hash + pub previous: String, + /// Height + pub height: u64 +} + +impl BlockHeaderInfo { + pub fn from_header(block_header: &core::BlockHeader) -> BlockHeaderInfo{ + BlockHeaderInfo { + hash: util::to_hex(block_header.hash().to_vec()), + previous: util::to_hex(block_header.previous.to_vec()), + height: block_header.height, + } + } +} +// For wallet reconstruction, include the header info along with the +// transactions in the block +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct BlockOutputs { + /// The block header + pub header: BlockHeaderInfo, + /// A printable version of the outputs + pub outputs: Vec, +} + #[derive(Serialize, Deserialize)] pub struct PoolInfo { /// Size of the pool diff --git a/core/src/core/transaction.rs b/core/src/core/transaction.rs index 4df725e91..44d575656 100644 --- a/core/src/core/transaction.rs +++ b/core/src/core/transaction.rs @@ -401,7 +401,8 @@ bitflags! { /// Definition of the switch commitment hash #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] pub struct SwitchCommitHash { - hash: [u8; SWITCH_COMMIT_HASH_SIZE], + /// simple hash + pub hash: [u8; SWITCH_COMMIT_HASH_SIZE], } /// Implementation of Writeable for a switch commitment hash diff --git a/keychain/src/keychain.rs b/keychain/src/keychain.rs index aed65f23f..3ec31b477 100644 --- a/keychain/src/keychain.rs +++ b/keychain/src/keychain.rs @@ -172,6 +172,14 @@ impl Keychain { Ok(commit) } + pub fn switch_commit_from_index(&self, index:u32) -> Result { + //just do this directly, because cache seems really slow for wallet reconstruct + let skey = self.extkey.derive(&self.secp, index)?; + let skey = skey.key; + let commit = self.secp.switch_commit(skey)?; + Ok(commit) + } + pub fn range_proof( &self, amount: u64, diff --git a/src/bin/grin.rs b/src/bin/grin.rs index dcd911f98..2f0604639 100644 --- a/src/bin/grin.rs +++ b/src/bin/grin.rs @@ -188,7 +188,14 @@ fn main() { .long("api_server_address") .help("Api address of running node on which to check inputs and post transactions") .takes_value(true)) - + .arg(Arg::with_name("key_derivations") + .help("The number of keys possiblities to search for each output. \ + Ideally, set this to a number greater than the number of outputs \ + you believe should belong to this seed/password. (Default 500)") + .short("k") + .long("key_derivations") + .default_value("1000") + .takes_value(true)) .subcommand(SubCommand::with_name("listen") .about("Run the wallet in listening mode. If an input file is \ provided, will process it, otherwise runs in server mode waiting \ @@ -250,8 +257,11 @@ fn main() { .about("basic wallet contents summary")) .subcommand(SubCommand::with_name("init") - .about("Initialize a new wallet seed file."))) + .about("Initialize a new wallet seed file.")) + .subcommand(SubCommand::with_name("restore") + .about("Attempt to restore wallet contents from the chain using seed and password."))) + .get_matches(); match args.subcommand() { @@ -376,6 +386,11 @@ fn wallet_command(wallet_args: &ArgMatches) { wallet_config.check_node_api_http_addr = sa.to_string().clone(); } + let mut key_derivations:u32=1000; + if let Some(kd) = wallet_args.value_of("key_derivations") { + key_derivations=kd.parse().unwrap(); + } + let mut show_spent=false; if wallet_args.is_present("show_spent") { show_spent=true; @@ -469,6 +484,9 @@ fn wallet_command(wallet_args: &ArgMatches) { ("outputs", Some(_)) => { wallet::show_outputs(&wallet_config, &keychain, show_spent); } + ("restore", Some(_)) => { + let _=wallet::restore(&wallet_config, &keychain, key_derivations); + } ("receive", Some(_)) => { panic!("Command 'receive' is depreciated, use 'listen' instead"); } diff --git a/wallet/src/checker.rs b/wallet/src/checker.rs index 9aeac1002..38a5abe1f 100644 --- a/wallet/src/checker.rs +++ b/wallet/src/checker.rs @@ -84,7 +84,7 @@ pub fn refresh_outputs(config: &WalletConfig, keychain: &Keychain) -> Result<(), let query_string = query_params.join("&"); let url = format!( - "{}/v1/chain/utxos?{}", + "{}/v1/chain/utxos/byids?{}", config.check_node_api_http_addr, query_string, ); diff --git a/wallet/src/lib.rs b/wallet/src/lib.rs index f5bf27a8e..3ae486d2f 100644 --- a/wallet/src/lib.rs +++ b/wallet/src/lib.rs @@ -48,6 +48,7 @@ mod info; mod receiver; mod sender; mod types; +mod restore; pub mod client; pub mod server; @@ -56,3 +57,4 @@ pub use info::show_info; pub use receiver::{receive_json_tx, receive_json_tx_str, WalletReceiver}; pub use sender::{issue_burn_tx, issue_send_tx}; pub use types::{BlockFees, CbData, Error, WalletConfig, WalletReceiveRequest, WalletSeed}; +pub use restore::restore; diff --git a/wallet/src/restore.rs b/wallet/src/restore.rs new file mode 100644 index 000000000..f985abde9 --- /dev/null +++ b/wallet/src/restore.rs @@ -0,0 +1,202 @@ +// Copyright 2017 The Grin Developers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use keychain::{Keychain, Identifier}; +use util::{LOGGER, from_hex}; +use util::secp::pedersen; +use api; +use core::core::{Output,SwitchCommitHash}; +use core::core::transaction::{COINBASE_OUTPUT, DEFAULT_OUTPUT, SWITCH_COMMIT_HASH_SIZE}; +use types::{WalletConfig, WalletData, OutputData, OutputStatus, Error}; +use byteorder::{BigEndian, ByteOrder}; + +pub fn get_chain_height(config: &WalletConfig)-> + Result{ + let url = format!( + "{}/v1/chain", + config.check_node_api_http_addr + ); + + match api::client::get::(url.as_str()) { + Ok(tip) => { + Ok(tip.height) + }, + Err(e) => { + // if we got anything other than 200 back from server, bye + error!(LOGGER, "Restore failed... unable to contact node: {}", e); + Err(Error::Node(e)) + } + } +} + +fn output_with_range_proof(config:&WalletConfig, commit_id: &str) -> + Result{ + + let url = format!( + "{}/v1/chain/utxos/byids?id={}&include_rp&include_switch", + config.check_node_api_http_addr, + commit_id, + ); + + match api::client::get::>(url.as_str()) { + Ok(outputs) => { + Ok(outputs[0].clone()) + }, + Err(e) => { + // if we got anything other than 200 back from server, don't attempt to refresh the wallet + // data after + Err(Error::Node(e)) + } + } +} + +fn retrieve_amount_and_coinbase_status(config:&WalletConfig, keychain: &Keychain, + key_id: Identifier, commit_id: &str) -> (u64, bool) { + let output = output_with_range_proof(config, commit_id).unwrap(); + let core_output = Output { + features : match output.output_type { + api::OutputType::Coinbase => COINBASE_OUTPUT, + api::OutputType::Transaction => DEFAULT_OUTPUT, + }, + proof: output.proof.unwrap(), + switch_commit_hash: output.switch_commit_hash.unwrap(), + commit: output.commit, + }; + let amount=core_output.recover_value(keychain, &key_id).unwrap(); + let is_coinbase = match output.output_type { + api::OutputType::Coinbase => true, + api::OutputType::Transaction => false, + }; + (amount, is_coinbase) +} + +pub fn utxos_batch_block(config: &WalletConfig, start_height: u64, end_height:u64)-> + Result, Error>{ + // build the necessary query param - + // ?height=x + let query_param= format!("start_height={}&end_height={}", start_height, end_height); + + let url = format!( + "{}/v1/chain/utxos/atheight?{}", + config.check_node_api_http_addr, + query_param, + ); + + match api::client::get::>(url.as_str()) { + Ok(outputs) => Ok(outputs), + Err(e) => { + // if we got anything other than 200 back from server, bye + error!(LOGGER, "Restore failed... unable to contact node: {}", e); + Err(Error::Node(e)) + } + } +} + +fn find_utxos_with_key(config:&WalletConfig, keychain: &Keychain, + switch_commit_cache : &Vec<[u8;SWITCH_COMMIT_HASH_SIZE]>, + block_outputs:api::BlockOutputs, key_iterations: &mut usize, padding: &mut usize) + -> Vec<(pedersen::Commitment, Identifier, u32, u64, u64, bool) > { + //let key_id = keychain.clone().root_key_id(); + let mut wallet_outputs: Vec<(pedersen::Commitment, Identifier, u32, u64, u64, bool)> = Vec::new(); + + info!(LOGGER, "Scanning block {} over {} key derivation possibilities.", block_outputs.header.height, *key_iterations); + for output in block_outputs.outputs { + for i in 0..*key_iterations { + if switch_commit_cache[i as usize]==output.switch_commit_hash { + info!(LOGGER, "Output found: {:?}, key_index: {:?}", output.switch_commit_hash,i); + //add it to result set here + let commit_id = from_hex(output.commit.clone()).unwrap(); + let key_id = keychain.derive_key_id(i as u32).unwrap(); + let (amount, is_coinbase) = retrieve_amount_and_coinbase_status(config, + keychain, key_id.clone(), &output.commit); + info!(LOGGER, "Amount: {}", amount); + let commit = keychain.commit_with_key_index(BigEndian::read_u64(&commit_id), i as u32).unwrap(); + wallet_outputs.push((commit, key_id.clone(), i as u32, amount, output.height, is_coinbase)); + //probably don't have to look for indexes greater than this now + *key_iterations=i+*padding; + if *key_iterations > switch_commit_cache.len() { + *key_iterations = switch_commit_cache.len(); + } + info!(LOGGER, "Setting max key index to: {}", *key_iterations); + break; + } + } + } + wallet_outputs +} + +pub fn restore(config: &WalletConfig, keychain: &Keychain, key_derivations:u32) -> + Result<(), Error>{ + // Don't proceed if wallet.dat has anything in it + let is_empty = WalletData::read_wallet(&config.data_file_dir, |wallet_data| { + wallet_data.outputs.len() == 0 + })?; + if !is_empty { + error!(LOGGER, "Not restoring. Please back up and remove existing wallet.dat first."); + return Ok(()) + } + +// Get height of chain from node (we'll check again when done) + let chain_height = get_chain_height(config)?; + info!(LOGGER, "Starting restore: Chain height is {}.", chain_height); + + let mut switch_commit_cache : Vec<[u8;SWITCH_COMMIT_HASH_SIZE]> = vec![]; + info!(LOGGER, "Building key derivation cache to index {}.", key_derivations); + for i in 0..key_derivations { + let switch_commit = keychain.switch_commit_from_index(i as u32).unwrap(); + let switch_commit_hash = SwitchCommitHash::from_switch_commit(switch_commit); + switch_commit_cache.push(switch_commit_hash.hash); + } + + let batch_size=100; + //this will start here, then lower as outputs are found, moving backwards on the chain + let mut key_iterations=key_derivations as usize; + //set to a percentage of the key_derivation value + let mut padding = (key_iterations as f64 *0.25) as usize; + let mut h = chain_height; + while { + let end_batch=h; + if h >= batch_size { + h-=batch_size; + } else { + h=0; + } + let mut blocks = utxos_batch_block(config, h+1, end_batch)?; + blocks.reverse(); + for block in blocks { + let result_vec=find_utxos_with_key(config, keychain, &switch_commit_cache, + block, &mut key_iterations, &mut padding); + if result_vec.len() > 0 { + for output in result_vec.clone() { + let _ = WalletData::with_wallet(&config.data_file_dir, |wallet_data| { + let root_key_id = keychain.root_key_id(); + //Just plonk it in for now, and refresh actual values via wallet info command later + wallet_data.add_output(OutputData { + root_key_id: root_key_id.clone(), + key_id: output.1.clone(), + n_child: output.2, + value: output.3, + status: OutputStatus::Unconfirmed, + height: output.4, + lock_height: 0, + is_coinbase: output.5, + }); + }); + } + } + } + h > 0 + }{} + Ok(()) +}