Core/grin
0
0
Fork 0
mirror of https://github.com/mimblewimble/grin.git synced 2025-01-21 03:21:08 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

db wallet now working identically to file wallet (#1259)

Browse source
This commit is contained in:
Yeastplume 2018-07-13 15:27:16 +01:00 committed by GitHub
parent 42bc03f5f3
commit 41e20056d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 213 additions and 222 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
grin.toml
View file

@ -65,6 +65,11 @@ run_tui = true
#Whether to run the wallet listener with the server by default
run_wallet_listener = true
#whether to use the database backend storage for the default listener wallet
#true = lmdb storage
#false = files storage
use_db_wallet = false
# Whether to run the web-wallet API (will only run on localhost)
run_wallet_owner_api = true

154
wallet/src/file_wallet.rs
View file

@ -25,19 +25,13 @@ use tokio_retry::Retry;
use failure::ResultExt;
use keychain::{self, Identifier, Keychain};
use util::secp::pedersen;
use util::LOGGER;
use error::{Error, ErrorKind};
use client;
use libtx::slate::Slate;
use libwallet;
use libwallet::types::{
BlockFees, CbData, OutputData, TxWrapper, WalletBackend, WalletClient, WalletDetails,
WalletOutputBatch,
};
use libwallet::types::{OutputData, WalletBackend, WalletClient, WalletDetails, WalletOutputBatch};
use types::{WalletConfig, WalletSeed};
@ -52,7 +46,7 @@ struct FileBatch<'a> {
/// List of outputs
outputs: &'a mut HashMap<String, OutputData>,
/// Wallet Details
details: &'a mut WalletDetails,
details: WalletDetails,
/// Data file path
data_file_path: String,
/// Details file path
@ -67,10 +61,6 @@ impl<'a> WalletOutputBatch for FileBatch<'a> {
Ok(())
}
fn details(&mut self) -> &mut WalletDetails {
&mut self.details
}
fn get(&self, id: &Identifier) -> Result<OutputData, libwallet::Error> {
self.outputs
.get(&id.to_hex())
@ -78,15 +68,16 @@ impl<'a> WalletOutputBatch for FileBatch<'a> {
.ok_or(libwallet::ErrorKind::Backend("not found".to_string()).into())
}
fn iter<'b>(&'b self) -> Box<Iterator<Item = OutputData> + 'b> {
Box::new(self.outputs.values().cloned())
}
fn delete(&mut self, id: &Identifier) -> Result<(), libwallet::Error> {
let _ = self.outputs.remove(&id.to_hex());
Ok(())
}
fn save_details(&mut self, _r: Identifier, w: WalletDetails) -> Result<(), libwallet::Error> {
self.details = w;
Ok(())
}
fn lock_output(&mut self, out: &mut OutputData) -> Result<(), libwallet::Error> {
if let Some(out_to_lock) = self.outputs.get_mut(&out.key_id.to_hex()) {
if out_to_lock.value == out.value {
@ -219,7 +210,7 @@ where
Ok(Box::new(FileBatch {
outputs: &mut self.outputs,
details: &mut self.details,
details: self.details.clone(),
data_file_path: self.data_file_path.clone(),
details_file_path: self.details_file_path.clone(),
lock_file_path: self.lock_file_path.clone(),
@ -231,94 +222,28 @@ where
&'a mut self,
root_key_id: keychain::Identifier,
) -> Result<u32, libwallet::Error> {
let mut details = self.details(root_key_id.clone())?;
let mut max_n = 0;
for out in self.iter() {
if max_n < out.n_child && out.root_key_id == root_key_id {
max_n = out.n_child;
}
}
let mut batch = self.batch()?;
{
let mut max_n = 0;
for out in batch.iter() {
if max_n < out.n_child && out.root_key_id == root_key_id {
max_n = out.n_child;
}
}
let details = batch.details();
if details.last_child_index <= max_n {
details.last_child_index = max_n + 1;
} else {
details.last_child_index += 1;
}
}
batch.commit()?;
Ok(batch.details().last_child_index)
}
/// Select spendable coins from the wallet.
/// Default strategy is to spend the maximum number of outputs (up to
/// max_outputs). Alternative strategy is to spend smallest outputs first
/// but only as many as necessary. When we introduce additional strategies
/// we should pass something other than a bool in.
fn select_coins(
&self,
root_key_id: keychain::Identifier,
amount: u64,
current_height: u64,
minimum_confirmations: u64,
max_outputs: usize,
select_all: bool,
) -> Vec<OutputData> {
// first find all eligible outputs based on number of confirmations
let mut eligible = self.outputs
.values()
.filter(|out| {
out.root_key_id == root_key_id
&& out.eligible_to_spend(current_height, minimum_confirmations)
})
.cloned()
.collect::<Vec<OutputData>>();
// sort eligible outputs by increasing value
eligible.sort_by_key(|out| out.value);
// use a sliding window to identify potential sets of possible outputs to spend
// Case of amount > total amount of max_outputs(500):
// The limit exists because by default, we always select as many inputs as
// possible in a transaction, to reduce both the Output set and the fees.
// But that only makes sense up to a point, hence the limit to avoid being too
// greedy. But if max_outputs(500) is actually not enough to cover the whole
// amount, the wallet should allow going over it to satisfy what the user
// wants to send. So the wallet considers max_outputs more of a soft limit.
if eligible.len() > max_outputs {
for window in eligible.windows(max_outputs) {
let windowed_eligibles = window.iter().cloned().collect::<Vec<_>>();
if let Some(outputs) = self.select_from(amount, select_all, windowed_eligibles) {
return outputs;
}
}
// Not exist in any window of which total amount >= amount.
// Then take coins from the smallest one up to the total amount of selected
// coins = the amount.
if let Some(outputs) = self.select_from(amount, false, eligible.clone()) {
debug!(
LOGGER,
"Extending maximum number of outputs. {} outputs selected.",
outputs.len()
);
return outputs;
}
if details.last_child_index <= max_n {
details.last_child_index = max_n + 1;
} else {
if let Some(outputs) = self.select_from(amount, select_all, eligible.clone()) {
return outputs;
}
details.last_child_index += 1;
}
// we failed to find a suitable set of outputs to spend,
// so return the largest amount we can so we can provide guidance on what is
// possible
eligible.reverse();
eligible.iter().take(max_outputs).cloned().collect()
batch.save_details(root_key_id.clone(), details.clone())?;
batch.commit()?;
Ok(details.last_child_index)
}
/// Return current metadata
fn details(&mut self) -> &mut WalletDetails {
&mut self.details
fn details(&mut self, _root_key_id: Identifier) -> Result<WalletDetails, libwallet::Error> {
self.batch()?;
Ok(self.details.clone())
}
/// Restore wallet contents
@ -455,35 +380,4 @@ where
.context(ErrorKind::FileWallet(&"Error writing wallet details file"))
.map_err(|e| e.into())
}
// Select the full list of outputs if we are using the select_all strategy.
// Otherwise select just enough outputs to cover the desired amount.
fn select_from(
&self,
amount: u64,
select_all: bool,
outputs: Vec<OutputData>,
) -> Option<Vec<OutputData>> {
let total = outputs.iter().fold(0, |acc, x| acc + x.value);
if total >= amount {
if select_all {
return Some(outputs.iter().cloned().collect());
} else {
let mut selected_amount = 0;
return Some(
outputs
.iter()
.take_while(|out| {
let res = selected_amount < amount;
selected_amount += out.value;
res
})
.cloned()
.collect(),
);
}
} else {
None
}
}
}

6
wallet/src/lib.rs
View file

@ -56,10 +56,12 @@ pub mod libwallet;
pub mod lmdb_wallet;
mod types;
pub use client::{create_coinbase, HTTPWalletClient};
pub use error::{Error, ErrorKind};
pub use file_wallet::FileWallet;
pub use client::{create_coinbase, HTTPWalletClient};
pub use libwallet::controller;
pub use libwallet::types::{BlockFees, CbData, WalletBackend, WalletClient, WalletInfo, WalletInst};
pub use libwallet::types::{
BlockFees, CbData, WalletBackend, WalletClient, WalletInfo, WalletInst,
};
pub use lmdb_wallet::{wallet_db_exists, LMDBBackend};
pub use types::{WalletConfig, WalletSeed};

8
wallet/src/libwallet/api.rs
View file

@ -67,7 +67,6 @@ where
include_spent: bool,
refresh_from_node: bool,
) -> Result<(bool, Vec<OutputData>), Error> {
let mut w = self.wallet.lock().unwrap();
w.open_with_credentials()?;
@ -120,7 +119,7 @@ where
let client;
let mut slate_out: Slate;
let lock_fn_out;
client = w.client().clone();
let (slate, context, lock_fn) = tx::create_send_tx(
&mut **w,
@ -196,7 +195,7 @@ where
Ok(height) => {
w.close()?;
Ok((height, true))
},
}
Err(_) => {
let outputs = self.retrieve_outputs(true, false)?;
let height = match outputs.1.iter().map(|out| out.height).max() {
@ -210,7 +209,7 @@ where
}
/// Attempt to update outputs in wallet, return whether it was successful
fn update_outputs(&self, w: &mut W ) -> bool {
fn update_outputs(&self, w: &mut W) -> bool {
match updater::refresh_outputs(&mut *w) {
Ok(_) => true,
Err(_) => false,
@ -255,7 +254,6 @@ where
let res = updater::build_coinbase(&mut **w, block_fees);
w.close()?;
res
}
/// Receive a transaction from a sender

16
wallet/src/libwallet/controller.rs
View file

@ -40,10 +40,7 @@ use util::LOGGER;
/// Instantiate wallet Owner API for a single-use (command line) call
/// Return a function containing a loaded API context to call
pub fn owner_single_use<F, T: ?Sized, C, K>(
wallet: Arc<Mutex<Box<T>>>,
f: F,
) -> Result<(), Error>
pub fn owner_single_use<F, T: ?Sized, C, K>(wallet: Arc<Mutex<Box<T>>>, f: F) -> Result<(), Error>
where
T: WalletBackend<C, K>,
F: FnOnce(&mut APIOwner<T, C, K>) -> Result<(), Error>,
@ -56,10 +53,7 @@ where
/// Instantiate wallet Foreign API for a single-use (command line) call
/// Return a function containing a loaded API context to call
pub fn foreign_single_use<F, T: ?Sized, C, K>(
wallet: Arc<Mutex<Box<T>>>,
f: F,
) -> Result<(), Error>
pub fn foreign_single_use<F, T: ?Sized, C, K>(wallet: Arc<Mutex<Box<T>>>, f: F) -> Result<(), Error>
where
T: WalletBackend<C, K>,
F: FnOnce(&mut APIForeign<T, C, K>) -> Result<(), Error>,
@ -264,7 +258,11 @@ where
}
}
fn issue_send_tx(&self, req: &mut Request, api: &mut APIOwner<T, C, K>) -> Result<Slate, Error> {
fn issue_send_tx(
&self,
req: &mut Request,
api: &mut APIOwner<T, C, K>,
) -> Result<Slate, Error> {
let struct_body = req.get::<bodyparser::Struct<SendTXArgs>>();
match struct_body {
Ok(Some(args)) => api.issue_send_tx(

5
wallet/src/libwallet/internal/restore.rs
View file

@ -172,8 +172,9 @@ where
let mut start_index = 1;
let mut result_vec: Vec<OutputResult> = vec![];
loop {
let (highest_index, last_retrieved_index, outputs) =
wallet.client().get_outputs_by_pmmr_index(start_index, batch_size)?;
let (highest_index, last_retrieved_index, outputs) = wallet
.client()
.get_outputs_by_pmmr_index(start_index, batch_size)?;
info!(
LOGGER,
"Retrieved {} outputs, up to index {}. (Highest index: {})",

126
wallet/src/libwallet/internal/selection.rs
View file

@ -20,6 +20,8 @@ use libwallet::error::{Error, ErrorKind};
use libwallet::internal::{keys, sigcontext};
use libwallet::types::*;
use util::LOGGER;
/// Initialize a transaction on the sender side, returns a corresponding
/// libwallet transaction slate with the appropriate inputs selected,
/// and saves the private wallet identifiers of our selected outputs
@ -211,8 +213,8 @@ where
let key_id = wallet.keychain().root_key_id();
// select some spendable coins from the wallet
let mut coins = wallet.select_coins(
key_id.clone(),
let (max_outputs, mut coins) = select_coins(
wallet,
amount,
current_height,
minimum_confirmations,
@ -220,18 +222,6 @@ where
selection_strategy_is_use_all,
);
// Get the maximum number of outputs in the wallet
let max_outputs = wallet
.select_coins(
key_id.clone(),
amount,
current_height,
minimum_confirmations,
max_outputs,
true,
)
.len();
// sender is responsible for setting the fee on the partial tx
// recipient should double check the fee calculation and not blindly trust the
// sender
@ -265,14 +255,14 @@ where
}
// select some spendable coins from the wallet
coins = wallet.select_coins(
key_id.clone(),
coins = select_coins(
wallet,
amount_with_fee,
current_height,
minimum_confirmations,
max_outputs,
selection_strategy_is_use_all,
);
).1;
fee = tx_fee(coins.len(), 2, None);
total = coins.iter().map(|c| c.value).sum();
amount_with_fee = amount + fee;
@ -334,3 +324,105 @@ where
Ok((parts, change, change_derivation))
}
/// Select spendable coins from a wallet.
/// Default strategy is to spend the maximum number of outputs (up to
/// max_outputs). Alternative strategy is to spend smallest outputs first
/// but only as many as necessary. When we introduce additional strategies
/// we should pass something other than a bool in.
/// TODO: Possibly move this into another trait to be owned by a wallet?
pub fn select_coins<T: ?Sized, C, K>(
wallet: &mut T,
amount: u64,
current_height: u64,
minimum_confirmations: u64,
max_outputs: usize,
select_all: bool,
) -> (usize, Vec<OutputData>)
// max_outputs_available, Outputs
where
T: WalletBackend<C, K>,
C: WalletClient,
K: Keychain,
{
// first find all eligible outputs based on number of confirmations
let root_key_id = wallet.keychain().root_key_id();
let mut eligible = wallet
.iter()
.filter(|out| {
out.root_key_id == root_key_id
&& out.eligible_to_spend(current_height, minimum_confirmations)
})
.collect::<Vec<OutputData>>();
let max_available = eligible.len();
// sort eligible outputs by increasing value
eligible.sort_by_key(|out| out.value);
// use a sliding window to identify potential sets of possible outputs to spend
// Case of amount > total amount of max_outputs(500):
// The limit exists because by default, we always select as many inputs as
// possible in a transaction, to reduce both the Output set and the fees.
// But that only makes sense up to a point, hence the limit to avoid being too
// greedy. But if max_outputs(500) is actually not enough to cover the whole
// amount, the wallet should allow going over it to satisfy what the user
// wants to send. So the wallet considers max_outputs more of a soft limit.
if eligible.len() > max_outputs {
for window in eligible.windows(max_outputs) {
let windowed_eligibles = window.iter().cloned().collect::<Vec<_>>();
if let Some(outputs) = select_from(amount, select_all, windowed_eligibles) {
return (max_available, outputs);
}
}
// Not exist in any window of which total amount >= amount.
// Then take coins from the smallest one up to the total amount of selected
// coins = the amount.
if let Some(outputs) = select_from(amount, false, eligible.clone()) {
debug!(
LOGGER,
"Extending maximum number of outputs. {} outputs selected.",
outputs.len()
);
return (max_available, outputs);
}
} else {
if let Some(outputs) = select_from(amount, select_all, eligible.clone()) {
return (max_available, outputs);
}
}
// we failed to find a suitable set of outputs to spend,
// so return the largest amount we can so we can provide guidance on what is
// possible
eligible.reverse();
(
max_available,
eligible.iter().take(max_outputs).cloned().collect(),
)
}
fn select_from(amount: u64, select_all: bool, outputs: Vec<OutputData>) -> Option<Vec<OutputData>> {
let total = outputs.iter().fold(0, |acc, x| acc + x.value);
if total >= amount {
if select_all {
return Some(outputs.iter().cloned().collect());
} else {
let mut selected_amount = 0;
return Some(
outputs
.iter()
.take_while(|out| {
let res = selected_amount < amount;
selected_amount += out.value;
res
})
.cloned()
.collect(),
);
}
} else {
None
}
}

6
wallet/src/libwallet/internal/tx.rs
View file

@ -152,11 +152,9 @@ where
let _ = updater::refresh_outputs(wallet);
let key_id = keychain.root_key_id();
// select some spendable coins from the wallet
let coins = wallet.select_coins(
key_id.clone(),
let (_, coins) = selection::select_coins(
wallet,
amount,
current_height,
minimum_confirmations,

12
wallet/src/libwallet/internal/updater.rs
View file

@ -112,6 +112,8 @@ where
// api output (if it exists) and refresh it in-place in the wallet.
// Note: minimizing the time we spend holding the wallet lock.
{
let root_key_id = wallet.keychain().root_key_id();
let mut details = wallet.details(root_key_id.clone())?;
let mut batch = wallet.batch()?;
for (commit, id) in wallet_outputs.iter() {
if let Ok(mut output) = batch.get(id) {
@ -123,8 +125,8 @@ where
}
}
{
let details = batch.details();
details.last_confirmed_height = height;
batch.save_details(root_key_id, details)?;
}
batch.commit()?;
}
@ -184,7 +186,8 @@ where
C: WalletClient,
K: Keychain,
{
let current_height = wallet.details().last_confirmed_height;
let root_key_id = wallet.keychain().root_key_id();
let current_height = wallet.details(root_key_id.clone())?.last_confirmed_height;
let keychain = wallet.keychain().clone();
let outputs = wallet
.iter()
@ -220,7 +223,10 @@ where
}
/// Build a coinbase output and insert into wallet
pub fn build_coinbase<T: ?Sized, C, K>(wallet: &mut T, block_fees: &BlockFees) -> Result<CbData, Error>
pub fn build_coinbase<T: ?Sized, C, K>(
wallet: &mut T,
block_fees: &BlockFees,
) -> Result<CbData, Error>
where
T: WalletBackend<C, K>,
C: WalletClient,

22
wallet/src/libwallet/types.rs
View file

@ -82,18 +82,7 @@ where
fn next_child<'a>(&mut self, root_key_id: Identifier) -> Result<u32, Error>;
/// Return current details
fn details(&mut self) -> &mut WalletDetails;
/// Select spendable coins from the wallet
fn select_coins(
&self,
root_key_id: Identifier,
amount: u64,
current_height: u64,
minimum_confirmations: u64,
max_outputs: usize,
select_all: bool,
) -> Vec<OutputData>;
fn details(&mut self, root_key_id: Identifier) -> Result<WalletDetails, Error>;
/// Attempt to restore the contents of a wallet from seed
fn restore(&mut self) -> Result<(), Error>;
@ -106,18 +95,15 @@ pub trait WalletOutputBatch {
/// Add or update data about an output to the backend
fn save(&mut self, out: OutputData) -> Result<(), Error>;
/// Get wallet details
fn details(&mut self) -> &mut WalletDetails;
/// Gets output data by id
fn get(&self, id: &Identifier) -> Result<OutputData, Error>;
/// Iterate over all output data in batch
fn iter<'b>(&'b self) -> Box<Iterator<Item = OutputData> + 'b>;
/// Delete data about an output to the backend
fn delete(&mut self, id: &Identifier) -> Result<(), Error>;
/// save wallet details
fn save_details(&mut self, r: Identifier, w: WalletDetails) -> Result<(), Error>;
/// Save an output as locked in the backend
fn lock_output(&mut self, out: &mut OutputData) -> Result<(), Error>;

73
wallet/src/lmdb_wallet.rs
View file

@ -29,6 +29,7 @@ pub const DB_DIR: &'static str = "wallet_data";
const OUTPUT_PREFIX: u8 = 'o' as u8;
const DERIV_PREFIX: u8 = 'd' as u8;
const CONFIRMED_HEIGHT_PREFIX: u8 = 'c' as u8;
impl From<store::Error> for Error {
fn from(error: store::Error) -> Error {
@ -121,38 +122,39 @@ where
fn batch<'a>(&'a mut self) -> Result<Box<WalletOutputBatch + 'a>, Error> {
Ok(Box::new(Batch {
store: self,
_store: self,
db: RefCell::new(Some(self.db.batch()?)),
}))
}
fn next_child<'a>(&mut self, root_key_id: Identifier) -> Result<u32, Error> {
let mut details = self.details(root_key_id.clone())?;
let mut batch = self.batch()?;
details.last_child_index = details.last_child_index + 1;
batch.save_details(root_key_id, details.clone())?;
batch.commit()?;
Ok(details.last_child_index + 1)
}
fn details(&mut self, root_key_id: Identifier) -> Result<WalletDetails, Error> {
let batch = self.db.batch()?;
// a simple counter, only one batch per db guarantees atomicity
let deriv_key = to_key(DERIV_PREFIX, &mut root_key_id.to_bytes().to_vec());
let deriv_idx = match batch.get_ser(&deriv_key)? {
Some(idx) => idx,
None => 0,
};
batch.put_ser(&deriv_key, &(deriv_idx + 1))?;
batch.commit()?;
Ok(deriv_idx + 1)
}
fn select_coins(
&self,
root_key_id: Identifier,
amount: u64,
current_height: u64,
minimum_confirmations: u64,
max_outputs: usize,
select_all: bool,
) -> Vec<OutputData> {
unimplemented!()
}
fn details(&mut self) -> &mut WalletDetails {
unimplemented!()
let height_key = to_key(
CONFIRMED_HEIGHT_PREFIX,
&mut root_key_id.to_bytes().to_vec(),
);
let last_confirmed_height = match batch.get_ser(&height_key)? {
Some(h) => h,
None => 0,
};
Ok(WalletDetails {
last_child_index: deriv_idx,
last_confirmed_height: last_confirmed_height,
})
}
fn restore(&mut self) -> Result<(), Error> {
@ -168,7 +170,7 @@ where
C: WalletClient,
K: Keychain,
{
store: &'a LMDBBackend<C, K>,
_store: &'a LMDBBackend<C, K>,
db: RefCell<Option<store::Batch<'a>>>,
}
@ -184,10 +186,6 @@ where
Ok(())
}
fn details(&mut self) -> &mut WalletDetails {
unimplemented!()
}
fn get(&self, id: &Identifier) -> Result<OutputData, Error> {
let key = to_key(OUTPUT_PREFIX, &mut id.to_bytes().to_vec());
option_to_not_found(
@ -196,16 +194,31 @@ where
).map_err(|e| e.into())
}
fn iter<'b>(&'b self) -> Box<Iterator<Item = OutputData> + 'b> {
unimplemented!();
}
fn delete(&mut self, id: &Identifier) -> Result<(), Error> {
let key = to_key(OUTPUT_PREFIX, &mut id.to_bytes().to_vec());
self.db.borrow().as_ref().unwrap().delete(&key)?;
Ok(())
}
fn save_details(&mut self, root_key_id: Identifier, d: WalletDetails) -> Result<(), Error> {
let deriv_key = to_key(DERIV_PREFIX, &mut root_key_id.to_bytes().to_vec());
let height_key = to_key(
CONFIRMED_HEIGHT_PREFIX,
&mut root_key_id.to_bytes().to_vec(),
);
self.db
.borrow()
.as_ref()
.unwrap()
.put_ser(&deriv_key, &d.last_child_index)?;
self.db
.borrow()
.as_ref()
.unwrap()
.put_ser(&height_key, &d.last_confirmed_height)?;
Ok(())
}
fn lock_output(&mut self, out: &mut OutputData) -> Result<(), Error> {
out.lock();
self.save(out.clone())

2
wallet/tests/transaction.rs
View file

@ -208,8 +208,6 @@ fn file_wallet_basic_transaction_api() {
basic_transaction_api(test_dir, common::BackendType::FileBackend);
}
// not yet ready
#[ignore]
#[test]
fn db_wallet_basic_transaction_api() {
let test_dir = "test_output/basic_transaction_api_db";