Add unit tests for API Endpoint (#653)

* First tests for API handlers

* Test chain UTXO handler

* Add test sumtrees handlers

* Fix typo

* Removed unused logger

* Update bitflags to ^1.0

* Missing test for bitflags update

* Moved test to grin directory

* Remove tests dependencies

* Add API P2P tests

* Fix hex string commitment

* Fix conflicting port

* Fix directory conflict and server port

* Wait for at least one block is mined on Travis

core/src/core/target.rs

@@ -159,4 +159,11 @@ impl<'de> de::Visitor<'de> for DiffVisitor {
 		};
 		Ok(Difficulty { num: num_in.unwrap() })
 	}
+
+	fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+	where
+		E: de::Error,
+	{
+		Ok(Difficulty { num: value })
+	}
 }

grin/src/seed.rs

@@ -155,7 +155,7 @@ impl Seeder {
 
 					// find some peers from our db
 					// and queue them up for a connection attempt
-					let peers = peers.find_peers(p2p::State::Healthy, p2p::UNKNOWN, 100);
+					let peers = peers.find_peers(p2p::State::Healthy, p2p::Capabilities::UNKNOWN, 100);
 					for p in peers {
 						debug!(LOGGER, "monitor_peers: queue to soon try {}", p.addr,);
 						tx.unbounded_send(p.addr).unwrap();
@@ -185,7 +185,7 @@ impl Seeder {
 		let seeder = thread_pool
 			.spawn_fn(move || {
 				// check if we have some peers in db
-				let peers = peers.find_peers(p2p::State::Healthy, p2p::FULL_HIST, 100);
+				let peers = peers.find_peers(p2p::State::Healthy, p2p::Capabilities::FULL_HIST, 100);
 				Ok(peers)
 			})
 			.and_then(|peers| {

grin/src/types.rs

@@ -149,7 +149,7 @@ impl Default for ServerConfig {
 		ServerConfig {
 			db_root: ".grin".to_string(),
 			api_http_addr: "0.0.0.0:13413".to_string(),
-			capabilities: p2p::FULL_NODE,
+			capabilities: p2p::Capabilities::FULL_NODE,
 			seeding_type: Seeding::default(),
 			seeds: None,
 			p2p_config: p2p::P2PConfig::default(),

grin/tests/api.rs

@@ -0,0 +1,369 @@
+// Copyright 2016 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[macro_use]
+extern crate slog;
+
+extern crate grin_api as api;
+extern crate grin_chain as chain;
+extern crate grin_core as core;
+extern crate grin_grin as grin;
+extern crate grin_p2p as p2p;
+extern crate grin_pow as pow;
+extern crate grin_util as util;
+extern crate grin_wallet as wallet;
+extern crate grin_config as config;
+
+mod framework;
+
+use std::{thread, time};
+use std::sync::{Arc, Mutex};
+
+use core::global;
+use core::global::ChainTypes;
+
+use framework::{LocalServerContainer,LocalServerContainerConfig};
+use util::{init_test_logger, LOGGER};
+
+#[test]
+fn simple_server_wallet() {
+	let test_name_dir = "test_servers";
+	core::global::set_mining_mode(core::global::ChainTypes::AutomatedTesting);
+	framework::clean_all_output(test_name_dir);
+
+	init_test_logger();
+
+	// Run a separate coinbase wallet for coinbase transactions
+	let mut coinbase_config = LocalServerContainerConfig::default();
+	coinbase_config.name = String::from("coinbase_wallet_api");
+	coinbase_config.wallet_validating_node_url=String::from("http://127.0.0.1:40001");
+	coinbase_config.wallet_port = 50002;
+	let coinbase_wallet = Arc::new(Mutex::new(LocalServerContainer::new(coinbase_config).unwrap()));
+
+	let _ = thread::spawn(move || {
+		let mut w = coinbase_wallet.lock().unwrap();
+		w.run_wallet(0);
+	});
+
+	let mut server_config = LocalServerContainerConfig::default();
+	server_config.name = String::from("api_server_one");
+	server_config.p2p_server_port = 40000;
+	server_config.api_server_port = 40001;
+	server_config.start_miner = true;
+	server_config.start_wallet = false;
+	server_config.coinbase_wallet_address = String::from(format!(
+		"http://{}:{}",
+		server_config.base_addr,
+		50002
+	));
+	let mut server_one = LocalServerContainer::new(server_config.clone()).unwrap();
+
+	// Spawn server and let it run for a bit
+	let _ = thread::spawn(move || server_one.run_server(120));
+
+	//Wait for chain to build
+	thread::sleep(time::Duration::from_millis(5000));
+
+	// Starting tests
+	let base_addr = server_config.base_addr;
+	let api_server_port = server_config.api_server_port;
+
+	warn!(LOGGER, "Testing chain handler");
+	let tip = get_tip(&base_addr, api_server_port);
+	assert!(tip.is_ok());
+
+	warn!(LOGGER, "Testing status handler");
+	let status = get_status(&base_addr, api_server_port);
+	assert!(status.is_ok());
+
+	// Be sure that at least a block is mined by Travis
+	while get_tip(&base_addr, api_server_port).unwrap().height == 0 {
+			thread::sleep(time::Duration::from_millis(1000));
+	}
+
+	warn!(LOGGER, "Testing block handler");
+	let current_tip = tip.unwrap();
+	let height = current_tip.height;
+	let last_block_by_height = get_block_by_height(&base_addr, api_server_port, height);
+	assert!(last_block_by_height.is_ok());
+	let last_block_by_height_compact = get_block_by_height_compact(&base_addr, api_server_port, height);
+	assert!(last_block_by_height_compact.is_ok());
+
+	let block_hash = current_tip.last_block_pushed;
+	let last_block_by_hash = get_block_by_hash(&base_addr, api_server_port, &block_hash);
+	assert!(last_block_by_hash.is_ok());
+	let last_block_by_hash_compact = get_block_by_hash_compact(&base_addr, api_server_port, &block_hash);
+	assert!(last_block_by_hash_compact.is_ok());
+
+	warn!(LOGGER, "Testing chain utxo handler");
+	let start_height = 0;
+	let end_height = height;
+	let utxos_by_height = get_utxos_by_height(&base_addr, api_server_port, start_height, end_height);
+	assert!(utxos_by_height.is_ok());
+	let ids = get_ids_from_block_outputs(utxos_by_height.unwrap());
+	let utxos_by_ids1 = get_utxos_by_ids1(&base_addr, api_server_port, ids.clone());
+	assert!(utxos_by_ids1.is_ok());
+	let utxos_by_ids2 = get_utxos_by_ids2(&base_addr, api_server_port, ids.clone());
+	assert!(utxos_by_ids2.is_ok());
+
+	warn!(LOGGER, "Testing sumtree handler");
+	let roots = get_sumtree_roots(&base_addr, api_server_port);
+	assert!(roots.is_ok());
+	let last_10_utxos = get_sumtree_lastutxos(&base_addr, api_server_port, 0);
+	assert!(last_10_utxos.is_ok());
+	let last_5_utxos = get_sumtree_lastutxos(&base_addr, api_server_port, 5);
+	assert!(last_5_utxos.is_ok());
+	let last_10_rangeproofs = get_sumtree_lastrangeproofs(&base_addr, api_server_port, 0);
+	assert!(last_10_rangeproofs.is_ok());
+	let last_5_rangeproofs = get_sumtree_lastrangeproofs(&base_addr, api_server_port, 5);
+	assert!(last_5_rangeproofs.is_ok());
+	let last_10_kernels = getsumtree_lastkernels(&base_addr, api_server_port, 0);
+	assert!(last_10_kernels.is_ok());
+	let last_5_kernels = getsumtree_lastkernels(&base_addr, api_server_port, 5);
+	assert!(last_5_kernels.is_ok());
+
+	//let some more mining happen, make sure nothing pukes
+	thread::sleep(time::Duration::from_millis(5000));
+}
+
+/// Creates 2 servers and test P2P API
+#[test]
+fn test_p2p() {
+	global::set_mining_mode(ChainTypes::AutomatedTesting);
+
+	let test_name_dir = "test_servers";
+	framework::clean_all_output(test_name_dir);
+
+	init_test_logger();
+
+	// Spawn server and let it run for a bit
+	let mut server_config_one = LocalServerContainerConfig::default();
+	server_config_one.name = String::from("p2p_server_one");
+	server_config_one.p2p_server_port = 40002;
+	server_config_one.api_server_port = 40003;
+	server_config_one.start_miner = false;
+	server_config_one.start_wallet = false;
+	server_config_one.is_seeding = true;
+	let mut server_one = LocalServerContainer::new(server_config_one.clone()).unwrap();
+	let _ = thread::spawn(move || server_one.run_server(120));
+
+	thread::sleep(time::Duration::from_millis(1000));
+
+	// Spawn server and let it run for a bit
+	let mut server_config_two = LocalServerContainerConfig::default();
+	server_config_two.name = String::from("p2p_server_two");
+	server_config_two.p2p_server_port = 40004;
+	server_config_two.api_server_port = 40005;
+	server_config_two.start_miner = false;
+	server_config_two.start_wallet = false;
+	server_config_two.is_seeding = false;
+	let mut server_two = LocalServerContainer::new(server_config_two.clone()).unwrap();
+	server_two.add_peer(format!("{}:{}", server_config_one.base_addr, server_config_one.p2p_server_port));
+	let _ = thread::spawn(move || server_two.run_server(120));
+
+	// Let them do the handshake
+	thread::sleep(time::Duration::from_millis(1000));
+
+	// Starting tests
+	warn!(LOGGER, "Starting P2P Tests");
+	let base_addr = server_config_one.base_addr;
+	let api_server_port = server_config_one.api_server_port;
+
+	// Check that when we get peer connected the peer is here
+	let peers_connected = get_connected_peers(&base_addr, api_server_port);
+	assert!(peers_connected.is_ok());
+	assert_eq!(peers_connected.unwrap().len(), 1);
+
+	// Check that peer all is also working
+	let mut peers_all = get_all_peers(&base_addr, api_server_port);
+	assert!(peers_all.is_ok());
+	assert_eq!(peers_all.unwrap().len(), 1);
+
+	// Check that the peer status is Healthy
+	let addr =  format!("{}:{}", server_config_two.base_addr, server_config_two.p2p_server_port);
+	let peer = get_peer(&base_addr, api_server_port, &addr);
+	assert!(peer.is_ok());
+	assert_eq!(peer.unwrap().flags, p2p::State::Healthy);
+
+	// Ban the peer
+	let ban_result = ban_peer(&base_addr, api_server_port, &addr);
+	assert!(ban_result.is_ok());
+	thread::sleep(time::Duration::from_millis(2000));
+
+	// Check its status is banned with get peer
+	let peer = get_peer(&base_addr, api_server_port, &addr);
+	assert!(peer.is_ok());
+	assert_eq!(peer.unwrap().flags, p2p::State::Banned);
+
+	// Check from peer all
+	peers_all = get_all_peers(&base_addr, api_server_port);
+	assert!(peers_all.is_ok());
+	assert_eq!(peers_all.unwrap().len(), 1);
+
+	// Unban
+	let unban_result = unban_peer(&base_addr, api_server_port, &addr);
+	assert!(unban_result.is_ok());
+
+	// Check from peer connected
+	let peers_connected = get_connected_peers(&base_addr, api_server_port);
+	assert!(peers_connected.is_ok());
+	assert_eq!(peers_connected.unwrap().len(), 1);
+
+	// Check its status is banned with get peer
+	let peer = get_peer(&base_addr, api_server_port, &addr);
+	assert!(peer.is_ok());
+	assert_eq!(peer.unwrap().flags, p2p::State::Healthy);
+}
+
+// Tip handler function
+fn get_tip(base_addr: &String, api_server_port: u16) -> Result<api::Tip, Error> {
+	let url = format!("http://{}:{}/v1/chain", base_addr, api_server_port);
+	api::client::get::<api::Tip>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+// Status handler function
+fn get_status(base_addr: &String, api_server_port: u16) -> Result<api::Status, Error> {
+	let url = format!("http://{}:{}/v1/status", base_addr, api_server_port);
+	api::client::get::<api::Status>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+// Block handler functions
+fn get_block_by_height(base_addr: &String, api_server_port: u16, height: u64) -> Result<api::BlockPrintable, Error> {
+	let url = format!("http://{}:{}/v1/blocks/{}", base_addr, api_server_port, height);
+	api::client::get::<api::BlockPrintable>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_block_by_height_compact(base_addr: &String, api_server_port: u16, height: u64) -> Result<api::CompactBlockPrintable, Error> {
+	let url = format!("http://{}:{}/v1/blocks/{}?compact", base_addr, api_server_port, height);
+	api::client::get::<api::CompactBlockPrintable>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_block_by_hash(base_addr: &String, api_server_port: u16, block_hash: &String) -> Result<api::BlockPrintable, Error> {
+	let url = format!("http://{}:{}/v1/blocks/{}", base_addr, api_server_port, block_hash);
+	api::client::get::<api::BlockPrintable>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_block_by_hash_compact(base_addr: &String, api_server_port: u16, block_hash: &String) -> Result<api::CompactBlockPrintable, Error> {
+	let url = format!("http://{}:{}/v1/blocks/{}?compact", base_addr, api_server_port, block_hash);
+	api::client::get::<api::CompactBlockPrintable>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+// Chain utxo handler functions
+fn get_utxos_by_ids1(base_addr: &String, api_server_port: u16, ids: Vec<String>) -> Result<Vec<api::Utxo>, Error> {
+	let url = format!("http://{}:{}/v1/chain/utxos/byids?id={}", base_addr, api_server_port, ids.join(","));
+	api::client::get::<Vec<api::Utxo>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_utxos_by_ids2(base_addr: &String, api_server_port: u16, ids: Vec<String>) -> Result<Vec<api::Utxo>, Error>  {
+	let mut ids_string: String = String::from("");
+	for id in ids {
+		ids_string = ids_string + "?id=" + &id;
+	}
+	let ids_string = String::from(&ids_string[1..ids_string.len()]);
+	println!("{}", ids_string);
+	let url = format!("http://{}:{}/v1/chain/utxos/byids?{}", base_addr, api_server_port, ids_string);
+	api::client::get::<Vec<api::Utxo>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_utxos_by_height(base_addr: &String, api_server_port: u16, start_height: u64, end_height: u64) -> Result<Vec<api::BlockOutputs>, Error> {
+	let url = format!("http://{}:{}/v1/chain/utxos/byheight?start_height={}&end_height={}", base_addr, api_server_port, start_height, end_height);
+	api::client::get::<Vec<api::BlockOutputs>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+// Sumtree handler functions
+fn get_sumtree_roots(base_addr: &String, api_server_port: u16)  -> Result<api::SumTrees, Error> {
+	let url = format!("http://{}:{}/v1/sumtrees/roots", base_addr, api_server_port);
+	api::client::get::<api::SumTrees>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_sumtree_lastutxos(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
+	let url: String;
+	if n == 0 {
+		url = format!("http://{}:{}/v1/sumtrees/lastutxos", base_addr, api_server_port);
+	} else {
+		url = format!("http://{}:{}/v1/sumtrees/lastutxos?n={}", base_addr, api_server_port, n);
+	}
+	api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn get_sumtree_lastrangeproofs(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
+	let url: String;
+	if n == 0 {
+		url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs", base_addr, api_server_port);
+	} else {
+		url = format!("http://{}:{}/v1/sumtrees/lastrangeproofs?n={}", base_addr, api_server_port, n);
+	}
+	api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+fn getsumtree_lastkernels(base_addr: &String, api_server_port: u16, n: u64) -> Result<Vec<api::SumTreeNode>, Error> {
+	let url: String;
+	if n == 0 {
+		url = format!("http://{}:{}/v1/sumtrees/lastkernels", base_addr, api_server_port);
+	} else {
+		url = format!("http://{}:{}/v1/sumtrees/lastkernels?n={}", base_addr, api_server_port, n);
+	}
+	api::client::get::<Vec<api::SumTreeNode>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+// Helper function to get a vec of commitment output ids from a vec of block outputs
+fn get_ids_from_block_outputs(block_outputs: Vec<api::BlockOutputs>) -> Vec<String> {
+	let mut ids: Vec<String> = Vec::new();
+	for block_output in block_outputs {
+		let outputs = &block_output.outputs;
+		for output in outputs {
+			ids.push(util::to_hex(output.clone().commit.0.to_vec()));
+		}
+	}
+	ids
+}
+
+pub fn ban_peer(base_addr: &String, api_server_port: u16, peer_addr: &String) -> Result<(), Error> {
+	let url = format!(
+		"http://{}:{}/v1/peers/{}/ban", base_addr, api_server_port, peer_addr
+	);
+	api::client::post(url.as_str(), &"").map_err(|e| Error::API(e))
+}
+
+pub fn unban_peer(base_addr: &String, api_server_port: u16, peer_addr: &String) -> Result<(), Error> {
+	let url = format!(
+		"http://{}:{}/v1/peers/{}/unban",
+		base_addr,
+		api_server_port,
+		peer_addr
+	);
+	api::client::post(url.as_str(), &"").map_err(|e| Error::API(e))
+}
+
+pub fn get_peer(base_addr: &String, api_server_port: u16, peer_addr: &String) -> Result<p2p::PeerData, Error> {
+	let url = format!("http://{}:{}/v1/peers/{}", base_addr, api_server_port, peer_addr);
+	api::client::get::<p2p::PeerData>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+pub fn get_connected_peers(base_addr: &String, api_server_port: u16) -> Result<Vec<p2p::PeerInfo>, Error> {
+	let url = format!("http://{}:{}/v1/peers/connected", base_addr, api_server_port);
+	api::client::get::<Vec<p2p::PeerInfo>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+pub fn get_all_peers(base_addr: &String, api_server_port: u16) -> Result<Vec<p2p::PeerData>, Error> {
+	let url = format!("http://{}:{}/v1/peers/all", base_addr, api_server_port);
+	api::client::get::<Vec<p2p::PeerData>>(url.as_str()).map_err(|e| Error::API(e))
+}
+
+/// Error type wrapping underlying module errors.
+#[derive(Debug)]
+pub enum Error {
+	/// Error originating from HTTP API calls.
+	API(api::Error),
+}

p2p/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Ignotus Peverell <igno.peverell@protonmail.com>"]
 workspace = ".."
 
 [dependencies]
-bitflags = "^0.7.0"
+bitflags = "^1.0"
 byteorder = "^0.5"
 futures = "^0.1.15"
 futures-cpupool = "^0.1.3"

p2p/src/lib.rs

@@ -58,6 +58,6 @@ mod types;
 pub use server::{DummyAdapter, Server};
 pub use peers::Peers;
 pub use peer::Peer;
-pub use types::{Capabilities, Error, ChainAdapter, P2PConfig, PeerInfo, FULL_HIST, FULL_NODE,
+pub use types::{Capabilities, Error, ChainAdapter, P2PConfig, PeerInfo, MAX_BLOCK_HEADERS,
-                MAX_BLOCK_HEADERS, MAX_PEER_ADDRS, UNKNOWN};
+                  MAX_PEER_ADDRS};
 pub use store::{PeerData, State};

p2p/src/peers.rs

@@ -504,7 +504,7 @@ impl NetAdapter for Peers {
 			}
 			let peer = PeerData {
 				addr: pa,
-				capabilities: UNKNOWN,
+				capabilities: Capabilities::UNKNOWN,
 				user_agent: "".to_string(),
 				flags: State::Healthy,
 				last_banned: 0,

p2p/src/store.rs

@@ -30,7 +30,7 @@ const PEER_PREFIX: u8 = 'p' as u8;
 
 /// Types of messages
 enum_from_primitive! {
-	#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
+	#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
 	pub enum State {
 		Healthy,
 		Banned,
@@ -39,7 +39,7 @@ enum_from_primitive! {
 }
 
 /// Data stored for any given peer we've encountered.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct PeerData {
 	/// Network address of the peer.
 	pub addr: SocketAddr,

p2p/src/types.rs

@@ -105,23 +105,23 @@ impl Default for P2PConfig {
 bitflags! {
   /// Options for what type of interaction a peer supports
   #[derive(Serialize, Deserialize)]
-  pub flags Capabilities: u32 {
+  pub struct Capabilities: u32 {
 	/// We don't know (yet) what the peer can do.
-	const UNKNOWN = 0b00000000,
+	const UNKNOWN = 0b00000000;
 	/// Full archival node, has the whole history without any pruning.
-	const FULL_HIST = 0b00000001,
+	const FULL_HIST = 0b00000001;
 	/// Can provide block headers and the UTXO set for some recent-enough
 	/// height.
-	const UTXO_HIST = 0b00000010,
+	const UTXO_HIST = 0b00000010;
 	/// Can provide a list of healthy peers
-	const PEER_LIST = 0b00000100,
+	const PEER_LIST = 0b00000100;
 
-	const FULL_NODE = FULL_HIST.bits | UTXO_HIST.bits | PEER_LIST.bits,
+	const FULL_NODE = Capabilities::FULL_HIST.bits | Capabilities::UTXO_HIST.bits | Capabilities::PEER_LIST.bits;
   }
 }
 
 /// General information about a connected peer that's useful to other modules.
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct PeerInfo {
 	pub capabilities: Capabilities,
 	pub user_agent: String,

p2p/tests/peer_handshake.rs

@@ -56,7 +56,7 @@ fn peer_handshake() {
 	let pool = CpuPool::new(1);
 	let server = p2p::Server::new(
 		".grin".to_owned(),
-		p2p::UNKNOWN,
+		p2p::Capabilities::UNKNOWN,
 		p2p_conf.clone(),
 		net_adapter.clone(),
 		Hash::from_vec(vec![]),
@@ -81,7 +81,7 @@ fn peer_handshake() {
 					.and_then(move |socket| {
 						Peer::connect(
 							socket,
-							p2p::UNKNOWN,
+							p2p::Capabilities::UNKNOWN,
 							Difficulty::one(),
 							my_addr,
 							Arc::new(