Changes to allow for testing and playing with cuckoo-miner integration (#76)

* Refactoring to allow for different miner implementations. Added conditional support for compiling and loading the cuckoo-miner plugin project.
* Small changes to experimentally integrate with cuckoo-miner and compatibility with latest version of cuckoo-miner.
* Turning off inclusion of cuckoo_miner by default
* Disabling simulate_parallel_mining test for now

Cargo.toml

@@ -8,7 +8,6 @@ members = ["api", "chain", "core", "grin", "p2p", "store", "util", "pool", "wall
 
 [dependencies]
 grin_api = { path = "./api" }
-grin_grin = { path = "./grin" }
 grin_wallet = { path = "./wallet" }
 secp256k1zkp = { path = "./secp256k1zkp" }
 
@@ -20,3 +19,14 @@ serde = "~1.0.8"
 serde_derive = "~1.0.8"
 serde_json = "~1.0.2"
 tiny-keccak = "1.1"
+
+[dependencies.grin_grin]
+path = "./grin"
+version = "*"
+default-features = false
+#Comment this in to use the cuckoo-miner package
+#ensure cuckoo-miner is cloned next to the 
+#grin directory
+#features = ["cuckoo_miner", "use-cuckoo-miner"]
+
+

core/src/consensus.rs

@@ -38,10 +38,13 @@ pub const BLOCK_TIME_SEC: i64 = 60;
 /// Cuckoo-cycle proof size (cycle length)
 pub const PROOFSIZE: usize = 42;
 
+
 /// Default Cuckoo Cycle size shift used for mining and validating.
 pub const DEFAULT_SIZESHIFT: u8 = 30;
 
 /// Lower Cuckoo size shift for tests and testnet
+/// This should be changed to correspond with the
+/// loaded plugin if using cuckoo-miner
 pub const TEST_SIZESHIFT: u8 = 12;
 
 /// Default Cuckoo Cycle easiness, high enough to have good likeliness to find

core/src/core/mod.rs

@@ -77,7 +77,6 @@ pub trait Committed {
 	fn overage(&self) -> i64;
 }
 
-
 /// Proof of work
 #[derive(Copy)]
 pub struct Proof(pub [u32; PROOFSIZE]);

core/src/pow/cuckoo.rs

@@ -26,6 +26,7 @@ use crypto::sha2::Sha256;
 use consensus::PROOFSIZE;
 use core::Proof;
 use pow::siphash::siphash24;
+use pow::MiningWorker;
 
 const MAXPATHLEN: usize = 8192;
 
@@ -153,8 +154,32 @@ impl Cuckoo {
 /// tests, being impractical with sizes greater than 2^22.
 pub struct Miner {
 	easiness: u64,
-	cuckoo: Cuckoo,
+	cuckoo: Option<Cuckoo>,
 	graph: Vec<u32>,
+	sizeshift: u32,
+}
+
+impl MiningWorker for Miner {
+
+	/// Creates a new miner
+	fn new(ease: u32, sizeshift: u32) -> Miner {
+		let size = 1 << sizeshift;
+		let graph = vec![0; size + 1];
+		let easiness = (ease as u64) * (size as u64) / 100;
+		Miner {
+			easiness: easiness,
+			cuckoo: None,
+			graph: graph,
+			sizeshift: sizeshift,
+		}
+	}
+	
+	fn mine(&mut self, header: &[u8]) -> Result<Proof, Error> {
+		let size = 1 << self.sizeshift;
+		self.graph = vec![0; size + 1];
+		self.cuckoo=Some(Cuckoo::new(header, self.sizeshift));
+		self.mine_impl()
+	}
 }
 
 /// What type of cycle we have found?
@@ -168,26 +193,15 @@ enum CycleSol {
 }
 
 impl Miner {
-	/// Creates a new miner
+	
-	pub fn new(header: &[u8], ease: u32, sizeshift: u32) -> Miner {
-		let cuckoo = Cuckoo::new(header, sizeshift);
-		let size = 1 << sizeshift;
-		let graph = vec![0; size + 1];
-		let easiness = (ease as u64) * (size as u64) / 100;
-		Miner {
-			easiness: easiness,
-			cuckoo: cuckoo,
-			graph: graph,
-		}
-	}
 
 	/// Searches for a solution
-	pub fn mine(&mut self) -> Result<Proof, Error> {
+	pub fn mine_impl(&mut self) -> Result<Proof, Error> {
 		let mut us = [0; MAXPATHLEN];
 		let mut vs = [0; MAXPATHLEN];
 		for nonce in 0..self.easiness {
-			us[0] = self.cuckoo.new_node(nonce, 0) as u32;
+			us[0] = self.cuckoo.as_mut().unwrap().new_node(nonce, 0) as u32;
-			vs[0] = self.cuckoo.new_node(nonce, 1) as u32;
+			vs[0] = self.cuckoo.as_mut().unwrap().new_node(nonce, 1) as u32;
 			let u = self.graph[us[0] as usize];
 			let v = self.graph[vs[0] as usize];
 			if us[0] == 0 {
@@ -198,7 +212,9 @@ impl Miner {
 
 			let sol = self.find_sol(nu, &us, nv, &vs);
 			match sol {
-				CycleSol::ValidProof(res) => return Ok(Proof(res)),
+				CycleSol::ValidProof(res) => {
+					return Ok(Proof(res))
+				},
 				CycleSol::InvalidCycle(_) => continue,
 				CycleSol::NoCycle => {
 					self.update_graph(nu, &us, nv, &vs);
@@ -240,7 +256,7 @@ impl Miner {
 		}
 	}
 
-	fn find_sol(&self, mut nu: usize, us: &[u32], mut nv: usize, vs: &[u32]) -> CycleSol {
+	fn find_sol(&mut self, mut nu: usize, us: &[u32], mut nv: usize, vs: &[u32]) -> CycleSol {
 		if us[nu] == vs[nv] {
 			let min = cmp::min(nu, nv);
 			nu -= min;
@@ -259,7 +275,7 @@ impl Miner {
 		}
 	}
 
-	fn solution(&self, us: &[u32], mut nu: u32, vs: &[u32], mut nv: u32) -> CycleSol {
+	fn solution(&mut self, us: &[u32], mut nu: u32, vs: &[u32], mut nv: u32) -> CycleSol {
 		let mut cycle = HashSet::new();
 		cycle.insert(Edge {
 			u: us[0] as u64,
@@ -284,7 +300,7 @@ impl Miner {
 		let mut n = 0;
 		let mut sol = [0; PROOFSIZE];
 		for nonce in 0..self.easiness {
-			let edge = self.cuckoo.new_edge(nonce);
+			let edge = self.cuckoo.as_mut().unwrap().new_edge(nonce);
 			if cycle.contains(&edge) {
 				sol[n] = nonce as u32;
 				n += 1;
@@ -344,13 +360,13 @@ mod test {
 	/// generated by other implementations.
 	#[test]
 	fn mine20_vectors() {
-		let nonces1 = Miner::new(&[49], 75, 20).mine().unwrap();
+		let nonces1 = Miner::new(75, 20).mine(&[49]).unwrap();
 		assert_eq!(V1, nonces1);
 
-		let nonces2 = Miner::new(&[50], 70, 20).mine().unwrap();
+		let nonces2 = Miner::new(70, 20).mine(&[50]).unwrap();
 		assert_eq!(V2, nonces2);
 
-		let nonces3 = Miner::new(&[51], 70, 20).mine().unwrap();
+		let nonces3 = Miner::new(70, 20).mine(&[51]).unwrap();
 		assert_eq!(V3, nonces3);
 	}
 
@@ -381,13 +397,13 @@ mod test {
 		// cuckoo20
 		for n in 1..5 {
 			let h = [n; 32];
-			let nonces = Miner::new(&h, 75, 20).mine().unwrap();
+			let nonces = Miner::new(75, 20).mine(&h).unwrap();
 			assert!(Cuckoo::new(&h, 20).verify(nonces, 75));
 		}
 		// cuckoo18
 		for n in 1..5 {
 			let h = [n; 32];
-			let nonces = Miner::new(&h, 75, 18).mine().unwrap();
+			let nonces = Miner::new(75, 18).mine(&h).unwrap();
 			assert!(Cuckoo::new(&h, 18).verify(nonces, 75));
 		}
 	}

core/src/pow/mod.rs

@@ -31,9 +31,25 @@ use consensus::EASINESS;
 use consensus::MINIMUM_DIFFICULTY;
 use core::BlockHeader;
 use core::hash::Hashed;
+use core::Proof;
 use core::target::Difficulty;
 use pow::cuckoo::{Cuckoo, Miner, Error};
 
+
+/// Should be implemented by anything providing mining services
+///
+
+pub trait MiningWorker {
+	
+	//This only sets parameters and does initialisation work now
+	fn new(ease: u32, sizeshift: u32) -> Self;
+	
+	//Actually perform a mining attempt on the given input and
+	//return a proof if found
+	fn mine(&mut self, header: &[u8]) -> Result<Proof, Error>;
+
+}
+
 /// Validates the proof of work of a given header, and that the proof of work
 /// satisfies the requirements of the header.
 pub fn verify_size(bh: &BlockHeader, cuckoo_sz: u32) -> bool {
@@ -65,7 +81,7 @@ pub fn pow_size(bh: &mut BlockHeader, diff: Difficulty, sizeshift: u32) -> Resul
 
 		// if we found a cycle (not guaranteed) and the proof hash is higher that the
 		// diff, we're all good
-		if let Ok(proof) = Miner::new(&pow_hash[..], EASINESS, sizeshift).mine() {
+		if let Ok(proof) = Miner::new(EASINESS, sizeshift).mine(&pow_hash[..]) {
 			if proof.to_difficulty() >= diff {
 				bh.pow = proof;
 				return Ok(());

grin/Cargo.toml

@@ -4,6 +4,10 @@ version = "0.1.0"
 authors = ["Ignotus Peverell <igno.peverell@protonmail.com>"]
 workspace = ".."
 
+[features]
+# Compliation flag whether to include the experimental cuckoo-miner crate
+use-cuckoo-miner = []
+
 [dependencies]
 grin_api = { path = "../api" }
 grin_chain = { path = "../chain" }
@@ -15,6 +19,8 @@ grin_util = { path = "../util" }
 grin_wallet = { path = "../wallet" }
 secp256k1zkp = { path = "../secp256k1zkp" }
 
+#cuckoo_miner = { version = "*", optional=true, path = "../../cuckoo-miner"}
+
 env_logger="^0.3.5"
 futures = "^0.1.9"
 futures-cpupool = "^0.1.3"

grin/src/lib.rs

@@ -46,12 +46,18 @@ extern crate grin_util as util;
 extern crate grin_wallet as wallet;
 extern crate secp256k1zkp as secp;
 
+#[cfg(feature = "use-cuckoo-miner")]
+extern crate cuckoo_miner;
+
 mod adapters;
 mod miner;
+#[cfg(feature = "use-cuckoo-miner")]
+mod plugin;
 mod server;
 mod seed;
 mod sync;
 mod types;
 
+
 pub use server::{Server};
 pub use types::{ServerConfig, MinerConfig, Seeding, ServerStats};

grin/src/miner.rs

@@ -19,6 +19,7 @@ use rand::{self, Rng};
 use std::sync::{Arc, Mutex, RwLock};
 use std::thread;
 use std;
+use std::env;
 use time;
 
 use adapters::{ChainToPoolAndNetAdapter, PoolToChainAdapter};
@@ -26,9 +27,11 @@ use api;
 use core::consensus;
 use core::consensus::*;
 use core::core;
+use core::core::Proof;
+use core::pow::cuckoo;
 use core::core::target::Difficulty;
 use core::core::hash::{Hash, Hashed};
-use core::pow::cuckoo;
+use core::pow::MiningWorker;
 use core::ser;
 use chain;
 use secp;
@@ -75,10 +78,11 @@ impl Miner {
 
 	/// Starts the mining loop, building a new block on top of the existing
 	/// chain anytime required and looking for PoW solution.
-	pub fn run_loop(&self) {
+	pub fn run_loop<T: MiningWorker>(&self, mut miner:T) {
 
 		info!("(Server ID: {}) Starting miner loop.", self.debug_output_id);
 		let mut coinbase = self.get_coinbase();
+
 		loop {
 			// get the latest chain state and build a block on top of it
 			let head = self.chain.head_header().unwrap();
@@ -102,14 +106,12 @@ impl Miner {
 			}
 			while head.hash() == latest_hash && time::get_time().sec < deadline {
 				let pow_hash = b.hash();
-				let mut miner =
+				if let Ok(proof) = miner.mine(&pow_hash[..]) {
-					cuckoo::Miner::new(&pow_hash[..], consensus::EASINESS, self.config.cuckoo_size);
-				if let Ok(proof) = miner.mine() {
 					let proof_diff=proof.to_difficulty();
-					debug!("(Server ID: {}) Header difficulty is: {}, Proof difficulty is: {}",
+					/*debug!("(Server ID: {}) Header difficulty is: {}, Proof difficulty is: {}",
 					self.debug_output_id,
 					b.header.difficulty,
-					proof_diff);
+					proof_diff);*/
 
 					if proof_diff >= b.header.difficulty {
 						sol = Some(proof);

grin/src/plugin.rs

@@ -0,0 +1,104 @@
+// Copyright 2017 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Plugin wrapper for cuckoo miner, implementing common traits
+//! with the existing embedded miner. This is all included conditionally
+//! for compatibility reasons with those who aren't interested in playing
+//! with cuckoo-miner at present
+
+use std::env;
+
+use core::pow::cuckoo;
+use core::pow::cuckoo::Error;
+use core::pow::MiningWorker;
+use core::consensus::TEST_SIZESHIFT;
+
+use core::core::Proof;
+
+use cuckoo_miner::{
+	CuckooMiner,
+	CuckooPluginManager,
+	CuckooMinerConfig,
+	CuckooMinerError,
+	CuckooMinerSolution,
+	CuckooPluginCapabilities};
+
+pub struct PluginMiner {
+	miner:CuckooMiner,
+	last_solution: CuckooMinerSolution,
+}
+
+impl MiningWorker for PluginMiner {
+
+	/// This will initialise a plugin according to what's currently
+	/// included in CONSENSUS::TEST_SIZESHIFT, just using the edgetrim
+	/// version of the miner for now, though this should become
+	/// configurable somehow
+
+	fn new(ease: u32, sizeshift: u32) -> Self {
+
+		//Get directory of executable
+		let mut exe_path=env::current_exe().unwrap();
+		exe_path.pop();
+		let exe_path=exe_path.to_str().unwrap();
+
+		//First, load and query the plugins in the given directory
+		//These should all be stored in 'deps' at the moment relative, though
+		//to the executable path, though they should appear somewhere else 
+		//when packaging is more//thought out 
+
+    	let mut plugin_manager = CuckooPluginManager::new().unwrap();
+    	let result=plugin_manager.load_plugin_dir(String::from(format!("{}/deps", exe_path))).expect("");
+
+    	//Get a list of installed plugins and capabilities.. filtering for the one we want
+		//Just use the baseline edgetrim (i.e. cuckoo_miner.cpp) for now
+		//You need to change the value TEST_SIZESHIFT in consensus.rs for now to modify this,
+		//so that blocks mined in this version will validate
+
+		let filter = format!("simple_{}", TEST_SIZESHIFT);
+
+    	let caps = plugin_manager.get_available_plugins(&filter).unwrap();
+		//insert it into the miner configuration being created below
+    
+    	let mut config = CuckooMinerConfig::new();
+
+        info!("Mining using plugin: {}", caps[0].full_path.clone());
+    	config.plugin_full_path = caps[0].full_path.clone();
+		//Set threads, should read this from a configuration file
+		//somewhere or query the system to determine a default
+		config.num_threads=4;
+		//let plugin decide number of trims
+		config.num_trims=0;
+
+		//this will load the associated plugin
+		let miner = CuckooMiner::new(config).expect("");
+
+		PluginMiner {
+			miner: miner,
+			last_solution: CuckooMinerSolution::new(),
+		}
+	}
+
+	/// And simply calls the mine function of the loaded plugin
+	/// returning whether a solution was found and the solution itself
+
+	fn mine(&mut self, header: &[u8]) -> Result<Proof, cuckoo::Error> {
+        let result = self.miner.mine(&header, &mut self.last_solution).unwrap();
+		if result == true {
+            return Ok(Proof(self.last_solution.solution_nonces));
+        }
+        Err(Error::NoSolution)
+	}
+}
+

grin/src/seed.rs

@@ -12,6 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Mining plugin manager, using the cuckoo-miner crate to provide
+//! a mining worker implementation
+//!
+
 use rand::{thread_rng, Rng};
 use std::cmp::min;
 use std::net::SocketAddr;

grin/src/server.rs

@@ -31,6 +31,8 @@ use chain;
 use chain::ChainStore;
 use core::{self, consensus};
 use core::core::hash::Hashed;
+use core::pow::cuckoo;
+use core::pow::MiningWorker;
 use miner;
 use p2p;
 use pool;
@@ -39,6 +41,9 @@ use store;
 use sync;
 use types::*;
 
+#[cfg(feature = "use-cuckoo-miner")]
+use plugin::PluginMiner;
+
 /// Grin server holding internal structures.
 pub struct Server {
 	pub config: ServerConfig,
@@ -142,11 +147,24 @@ impl Server {
 
 	/// Start mining for blocks on a separate thread. Relies on a toy miner,
 	/// mostly for testing.
+	#[cfg(not(feature = "use-cuckoo-miner"))]
 	pub fn start_miner(&self, config: MinerConfig) {
-		let mut miner = miner::Miner::new(config, self.chain.clone(), self.tx_pool.clone());
+		let mut miner = miner::Miner::new(config.clone(), self.chain.clone(), self.tx_pool.clone());
 		miner.set_debug_output_id(format!("Port {}",self.config.p2p_config.port));
 		thread::spawn(move || {
-			miner.run_loop();
+			let test_cuckoo_miner = cuckoo::Miner::new(consensus::EASINESS, config.cuckoo_size.clone());
+			miner.run_loop(test_cuckoo_miner);
+		});
+	}
+
+	/// And a version we only get if we're using the cuckoo miner crate
+	#[cfg(feature = "use-cuckoo-miner")]
+	pub fn start_miner(&self, config: MinerConfig) {
+		let mut miner = miner::Miner::new(config.clone(), self.chain.clone(), self.tx_pool.clone());
+		miner.set_debug_output_id(format!("Port {}",self.config.p2p_config.port));
+		thread::spawn(move || {
+			let test_cuckoo_miner = PluginMiner::new(consensus::EASINESS, config.cuckoo_size.clone());
+			miner.run_loop(test_cuckoo_miner);
 		});
 	}
 

grin/src/types.rs

@@ -114,6 +114,7 @@ pub struct MinerConfig {
 	/// Size of Cuckoo Cycle to mine on
 	pub cuckoo_size: u32,
 
+
 }
 
 impl Default for ServerConfig {

grin/tests/simulnet.rs

@@ -125,7 +125,10 @@ fn simulate_seeding () {
 /// as a seed. Meant to test the evolution of mining difficulty with miners running at
 /// different rates
 
-#[test]
+
+//Just going to comment this out as an automatically run test for the time being,
+//As it's more for actively testing and hurts CI a lot
+//#[test]
 fn simulate_parallel_mining(){
     env_logger::init();