[WIP] Store completed transactions in files instead of DB (#2148)

Store completed transactions in files instead of DB

src/bin/cmd/wallet_tests.rs

@@ -408,7 +408,7 @@ mod wallet_tests {
 			Ok(())
 		})?;
 
-		// Try using the self-send method
+		// Try using the self-send method, splitting up outputs for the fun of it
 		let arg_vec = vec![
 			"grin",
 			"wallet",
@@ -423,6 +423,8 @@ mod wallet_tests {
 			"mining",
 			"-g",
 			"Self love",
+			"-o",
+			"75",
 			"-s",
 			"smallest",
 			"10",

wallet/src/command.rs

@@ -409,7 +409,7 @@ pub fn repost(
 ) -> Result<(), Error> {
 	controller::owner_single_use(wallet.clone(), |api| {
 		let (_, txs) = api.retrieve_txs(true, Some(args.id), None)?;
-		let stored_tx = txs[0].get_stored_tx();
+		let stored_tx = api.get_stored_tx(&txs[0])?;
 		if stored_tx.is_none() {
 			error!(
 				"Transaction with id {} does not have transaction data. Not reposting.",

wallet/src/controller.rs

@@ -231,7 +231,10 @@ where
 		if let Some(id_string) = params.get("id") {
 			match id_string[0].parse() {
 				Ok(id) => match api.retrieve_txs(true, Some(id), None) {
-					Ok((_, txs)) => Ok((txs[0].confirmed, txs[0].get_stored_tx())),
+					Ok((_, txs)) => {
+						let stored_tx = api.get_stored_tx(&txs[0])?;
+						Ok((txs[0].confirmed, stored_tx))
+					}
 					Err(e) => {
 						error!("retrieve_stored_tx: failed with error: {}", e);
 						Err(e)

wallet/src/display.rs

@@ -178,7 +178,7 @@ pub fn txs(
 			)
 		};
 		let tx_data = match t.tx_hex {
-			Some(_) => format!("Exists"),
+			Some(t) => format!("{}", t),
 			None => "None".to_owned(),
 		};
 		if dark_background_color_scheme {

wallet/src/libwallet/api.rs

@@ -612,7 +612,13 @@ where
 		num_change_outputs: usize,
 		selection_strategy_is_use_all: bool,
 		message: Option<String>,
-	) -> Result<(Slate, impl FnOnce(&mut W, &str) -> Result<(), Error>), Error> {
+	) -> Result<
+		(
+			Slate,
+			impl FnOnce(&mut W, &Transaction) -> Result<(), Error>,
+		),
+		Error,
+	> {
 		let mut w = self.wallet.lock();
 		w.open_with_credentials()?;
 		let parent_key_id = match src_acct_name {
@@ -653,12 +659,11 @@ where
 	pub fn tx_lock_outputs(
 		&mut self,
 		slate: &Slate,
-		lock_fn: impl FnOnce(&mut W, &str) -> Result<(), Error>,
+		lock_fn: impl FnOnce(&mut W, &Transaction) -> Result<(), Error>,
 	) -> Result<(), Error> {
 		let mut w = self.wallet.lock();
 		w.open_with_credentials()?;
-		let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap());
+		lock_fn(&mut *w, &slate.tx)?;
-		lock_fn(&mut *w, &tx_hex)?;
 		Ok(())
 	}
 
@@ -668,11 +673,10 @@ where
 	/// propagation.
 	pub fn finalize_tx(&mut self, slate: &mut Slate) -> Result<(), Error> {
 		let mut w = self.wallet.lock();
-		let parent_key_id = w.parent_key_id();
 		w.open_with_credentials()?;
 		let context = w.get_private_context(slate.id.as_bytes())?;
 		tx::complete_tx(&mut *w, slate, &context)?;
-		tx::update_tx_hex(&mut *w, &parent_key_id, slate)?;
+		tx::update_stored_tx(&mut *w, slate)?;
 		{
 			let mut batch = w.batch()?;
 			batch.delete_private_context(slate.id.as_bytes())?;
@@ -706,6 +710,12 @@ where
 		Ok(())
 	}
 
+	/// Retrieves a stored transaction from a TxLogEntry
+	pub fn get_stored_tx(&self, entry: &TxLogEntry) -> Result<Option<Transaction>, Error> {
+		let w = self.wallet.lock();
+		w.get_stored_tx(entry)
+	}
+
 	/// Posts a transaction to the chain
 	pub fn post_tx(&self, tx: &Transaction, fluff: bool) -> Result<(), Error> {
 		let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap());

wallet/src/libwallet/internal/selection.rs

@@ -14,6 +14,7 @@
 
 //! Selection of inputs for building transactions
 
+use crate::core::core::Transaction;
 use crate::core::libtx::{build, slate::Slate, tx_fee};
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::error::{Error, ErrorKind};
@@ -40,7 +41,7 @@ pub fn build_send_tx_slate<T: ?Sized, C, K>(
 	(
 		Slate,
 		Context,
-		impl FnOnce(&mut T, &str) -> Result<(), Error>,
+		impl FnOnce(&mut T, &Transaction) -> Result<(), Error>,
 	),
 	Error,
 >
@@ -94,42 +95,47 @@ where
 
 	// Return a closure to acquire wallet lock and lock the coins being spent
 	// so we avoid accidental double spend attempt.
-	let update_sender_wallet_fn = move |wallet: &mut T, tx_hex: &str| {
+	let update_sender_wallet_fn = move |wallet: &mut T, tx: &Transaction| {
-		let mut batch = wallet.batch()?;
+		let tx_entry = {
-		let log_id = batch.next_tx_log_id(&parent_key_id)?;
+			let mut batch = wallet.batch()?;
-		let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id);
+			let log_id = batch.next_tx_log_id(&parent_key_id)?;
-		t.tx_slate_id = Some(slate_id);
+			let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id);
-		t.fee = Some(fee);
+			t.tx_slate_id = Some(slate_id);
-		t.tx_hex = Some(tx_hex.to_owned());
+			let filename = format!("{}.grintx", slate_id);
-		let mut amount_debited = 0;
+			t.tx_hex = Some(filename);
-		t.num_inputs = lock_inputs.len();
+			t.fee = Some(fee);
-		for id in lock_inputs {
+			let mut amount_debited = 0;
-			let mut coin = batch.get(&id).unwrap();
+			t.num_inputs = lock_inputs.len();
-			coin.tx_log_entry = Some(log_id);
+			for id in lock_inputs {
-			amount_debited = amount_debited + coin.value;
+				let mut coin = batch.get(&id).unwrap();
-			batch.lock_output(&mut coin)?;
+				coin.tx_log_entry = Some(log_id);
-		}
+				amount_debited = amount_debited + coin.value;
+				batch.lock_output(&mut coin)?;
+			}
 
-		t.amount_debited = amount_debited;
+			t.amount_debited = amount_debited;
 
-		// write the output representing our change
+			// write the output representing our change
-		for (change_amount, id) in &change_amounts_derivations {
+			for (change_amount, id) in &change_amounts_derivations {
-			t.num_outputs += 1;
+				t.num_outputs += 1;
-			t.amount_credited += change_amount;
+				t.amount_credited += change_amount;
-			batch.save(OutputData {
+				batch.save(OutputData {
-				root_key_id: parent_key_id.clone(),
+					root_key_id: parent_key_id.clone(),
-				key_id: id.clone(),
+					key_id: id.clone(),
-				n_child: id.to_path().last_path_index(),
+					n_child: id.to_path().last_path_index(),
-				value: change_amount.clone(),
+					value: change_amount.clone(),
-				status: OutputStatus::Unconfirmed,
+					status: OutputStatus::Unconfirmed,
-				height: current_height,
+					height: current_height,
-				lock_height: 0,
+					lock_height: 0,
-				is_coinbase: false,
+					is_coinbase: false,
-				tx_log_entry: Some(log_id),
+					tx_log_entry: Some(log_id),
-			})?;
+				})?;
-		}
+			}
-		batch.save_tx_log_entry(t, &parent_key_id)?;
+			batch.save_tx_log_entry(t.clone(), &parent_key_id)?;
-		batch.commit()?;
+			batch.commit()?;
+			t
+		};
+		wallet.store_tx(&format!("{}", tx_entry.tx_slate_id.unwrap()), tx)?;
 		Ok(())
 	};
 

wallet/src/libwallet/internal/tx.rs

@@ -14,11 +14,10 @@
 
 //! Transaction building functions
 
-use crate::util;
 use uuid::Uuid;
 
+use crate::core::core::Transaction;
 use crate::core::libtx::slate::Slate;
-use crate::core::ser;
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::internal::{selection, updater};
 use crate::libwallet::types::{Context, NodeClient, TxLogEntryType, WalletBackend};
@@ -74,7 +73,7 @@ pub fn create_send_tx<T: ?Sized, C, K>(
 	(
 		Slate,
 		Context,
-		impl FnOnce(&mut T, &str) -> Result<(), Error>,
+		impl FnOnce(&mut T, &Transaction) -> Result<(), Error>,
 	),
 	Error,
 >
@@ -200,19 +199,13 @@ where
 	Ok((tx.confirmed, tx.tx_hex))
 }
 
-/// Update the stored hex transaction (this update needs to happen when the TX is finalised)
+/// Update the stored transaction (this update needs to happen when the TX is finalised)
-pub fn update_tx_hex<T: ?Sized, C, K>(
+pub fn update_stored_tx<T: ?Sized, C, K>(wallet: &mut T, slate: &Slate) -> Result<(), Error>
-	wallet: &mut T,
-	_parent_key_id: &Identifier,
-	slate: &Slate,
-) -> Result<(), Error>
 where
 	T: WalletBackend<C, K>,
 	C: NodeClient,
 	K: Keychain,
 {
-	let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap());
-	// This will ignore the parent key, so no need to specify account on the
 	// finalize command
 	let tx_vec = updater::retrieve_txs(wallet, None, Some(slate.id), None)?;
 	let mut tx = None;
@@ -223,14 +216,11 @@ where
 			break;
 		}
 	}
-	let mut tx = match tx {
+	let tx = match tx {
 		Some(t) => t,
 		None => return Err(ErrorKind::TransactionDoesntExist(slate.id.to_string()))?,
 	};
-	tx.tx_hex = Some(tx_hex);
+	wallet.store_tx(&format!("{}", tx.tx_slate_id.unwrap()), &slate.tx)?;
-	let batch = wallet.batch()?;
-	batch.save_tx_log_entry(tx.clone(), &tx.parent_key_id)?;
-	batch.commit()?;
 	Ok(())
 }
 

wallet/src/libwallet/types.rs

@@ -21,7 +21,6 @@ use crate::core::libtx::aggsig;
 use crate::core::ser;
 use crate::keychain::{Identifier, Keychain};
 use crate::libwallet::error::{Error, ErrorKind};
-use crate::util;
 use crate::util::secp::key::{PublicKey, SecretKey};
 use crate::util::secp::{self, pedersen, Secp256k1};
 use chrono::prelude::*;
@@ -102,6 +101,12 @@ where
 	/// Gets an account path for a given label
 	fn get_acct_path(&self, label: String) -> Result<Option<AcctPathMapping>, Error>;
 
+	/// Stores a transaction
+	fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error>;
+
+	/// Retrieves a stored transaction from a TxLogEntry
+	fn get_stored_tx(&self, entry: &TxLogEntry) -> Result<Option<Transaction>, Error>;
+
 	/// Create a new write batch to update or remove output data
 	fn batch<'a>(&'a mut self) -> Result<Box<dyn WalletOutputBatch<K> + 'a>, Error>;
 
@@ -156,7 +161,7 @@ where
 	fn tx_log_iter(&self) -> Box<dyn Iterator<Item = TxLogEntry>>;
 
 	/// save a tx log entry
-	fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>;
+	fn save_tx_log_entry(&mut self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>;
 
 	/// save an account label -> path mapping
 	fn save_acct_path(&mut self, mapping: AcctPathMapping) -> Result<(), Error>;
@@ -595,6 +600,7 @@ pub struct TxLogEntry {
 	pub amount_debited: u64,
 	/// Fee
 	pub fee: Option<u64>,
+	// TODO: rename this to 'stored_tx_file' or something for mainnet
 	/// The transaction json itself, stored for reference or resending
 	pub tx_hex: Option<String>,
 }
@@ -636,17 +642,6 @@ impl TxLogEntry {
 	pub fn update_confirmation_ts(&mut self) {
 		self.confirmation_ts = Some(Utc::now());
 	}
-
-	/// Retrieve the stored transaction, if any
-	pub fn get_stored_tx(&self) -> Option<Transaction> {
-		match self.tx_hex.as_ref() {
-			None => None,
-			Some(t) => {
-				let tx_bin = util::from_hex(t.clone()).unwrap();
-				Some(ser::deserialize::<Transaction>(&mut &tx_bin[..]).unwrap())
-			}
-		}
-	}
 }
 
 /// Map of named accounts to BIP32 paths

wallet/src/lmdb_wallet.rs

@@ -16,18 +16,29 @@ use std::cell::RefCell;
 use std::sync::Arc;
 use std::{fs, path};
 
+// for writing storedtransaction files
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::Path;
+
+use serde_json;
+
 use failure::ResultExt;
 use uuid::Uuid;
 
 use crate::keychain::{ChildNumber, ExtKeychain, Identifier, Keychain};
 use crate::store::{self, option_to_not_found, to_key, to_key_u64};
 
+use crate::core::core::Transaction;
+use crate::core::ser;
 use crate::libwallet::types::*;
 use crate::libwallet::{internal, Error, ErrorKind};
 use crate::types::{WalletConfig, WalletSeed};
+use crate::util;
 use crate::util::secp::pedersen;
 
 pub const DB_DIR: &'static str = "db";
+pub const TX_SAVE_DIR: &'static str = "saved_txs";
 
 const COMMITMENT_PREFIX: u8 = 'C' as u8;
 const OUTPUT_PREFIX: u8 = 'o' as u8;
@@ -69,10 +80,16 @@ impl<C, K> LMDBBackend<C, K> {
 		let db_path = path::Path::new(&config.data_file_dir).join(DB_DIR);
 		fs::create_dir_all(&db_path).expect("Couldn't create wallet backend directory!");
 
+		let stored_tx_path = path::Path::new(&config.data_file_dir).join(TX_SAVE_DIR);
+		fs::create_dir_all(&stored_tx_path)
+			.expect("Couldn't create wallet backend tx storage directory!");
+
 		let lmdb_env = Arc::new(store::new_env(db_path.to_str().unwrap().to_string()));
 		let store = store::Store::open(lmdb_env, DB_DIR);
 
 		// Make sure default wallet derivation path always exists
+		// as well as path (so it can be retrieved by batches to know where to store
+		// completed transactions, for reference
 		let default_account = AcctPathMapping {
 			label: "default".to_owned(),
 			path: LMDBBackend::<C, K>::default_path(),
@@ -228,6 +245,37 @@ where
 		self.db.get_ser(&acct_key).map_err(|e| e.into())
 	}
 
+	fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error> {
+		let filename = format!("{}.grintx", uuid);
+		let path = path::Path::new(&self.config.data_file_dir)
+			.join(TX_SAVE_DIR)
+			.join(filename);
+		let path_buf = Path::new(&path).to_path_buf();
+		let mut stored_tx = File::create(path_buf)?;
+		let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap());;
+		stored_tx.write_all(&tx_hex.as_bytes())?;
+		stored_tx.sync_all()?;
+		Ok(())
+	}
+
+	fn get_stored_tx(&self, entry: &TxLogEntry) -> Result<Option<Transaction>, Error> {
+		let filename = match entry.tx_hex.clone() {
+			Some(f) => f,
+			None => return Ok(None),
+		};
+		let path = path::Path::new(&self.config.data_file_dir)
+			.join(TX_SAVE_DIR)
+			.join(filename);
+		let tx_file = Path::new(&path).to_path_buf();
+		let mut tx_f = File::open(tx_file)?;
+		let mut content = String::new();
+		tx_f.read_to_string(&mut content)?;
+		let tx_bin = util::from_hex(content).unwrap();
+		Ok(Some(
+			ser::deserialize::<Transaction>(&mut &tx_bin[..]).unwrap(),
+		))
+	}
+
 	fn batch<'a>(&'a mut self) -> Result<Box<dyn WalletOutputBatch<K> + 'a>, Error> {
 		Ok(Box::new(Batch {
 			_store: self,
@@ -403,17 +451,21 @@ where
 		Ok(())
 	}
 
-	fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error> {
+	fn save_tx_log_entry(
+		&mut self,
+		tx_in: TxLogEntry,
+		parent_id: &Identifier,
+	) -> Result<(), Error> {
 		let tx_log_key = to_key_u64(
 			TX_LOG_ENTRY_PREFIX,
 			&mut parent_id.to_bytes().to_vec(),
-			t.id as u64,
+			tx_in.id as u64,
 		);
 		self.db
 			.borrow()
 			.as_ref()
 			.unwrap()
-			.put_ser(&tx_log_key, &t)?;
+			.put_ser(&tx_log_key, &tx_in)?;
 		Ok(())
 	}
 

wallet/tests/repost.rs

@@ -148,7 +148,8 @@ fn file_repost_test_impl(test_dir: &str) -> Result<(), libwallet::Error> {
 	// Now repost from cached
 	wallet::controller::owner_single_use(wallet1.clone(), |api| {
 		let (_, txs) = api.retrieve_txs(true, None, Some(slate.id))?;
-		api.post_tx(&txs[0].get_stored_tx().unwrap(), false)?;
+		let stored_tx = api.get_stored_tx(&txs[0])?;
+		api.post_tx(&stored_tx.unwrap(), false)?;
 		bh += 1;
 		Ok(())
 	})?;
@@ -214,7 +215,8 @@ fn file_repost_test_impl(test_dir: &str) -> Result<(), libwallet::Error> {
 	// Now repost from cached
 	wallet::controller::owner_single_use(wallet1.clone(), |api| {
 		let (_, txs) = api.retrieve_txs(true, None, Some(slate.id))?;
-		api.post_tx(&txs[0].get_stored_tx().unwrap(), false)?;
+		let stored_tx = api.get_stored_tx(&txs[0])?;
+		api.post_tx(&stored_tx.unwrap(), false)?;
 		bh += 1;
 		Ok(())
 	})?;

wallet/tests/transaction.rs

@@ -252,7 +252,8 @@ fn basic_transaction_api(test_dir: &str) -> Result<(), libwallet::Error> {
 			.iter()
 			.find(|t| t.tx_slate_id == Some(slate.id))
 			.unwrap();
-		sender_api.post_tx(&tx.get_stored_tx().unwrap(), false)?;
+		let stored_tx = sender_api.get_stored_tx(&tx)?;
+		sender_api.post_tx(&stored_tx.unwrap(), false)?;
 		let (_, wallet1_info) = sender_api.retrieve_summary_info(true, 1)?;
 		// should be mined now
 		assert_eq!(