From 12480e73100fb6165710307226b6e12cabb0a3a1 Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Tue, 11 Jul 2017 18:11:03 +0100 Subject: [PATCH] Changes to allow for testing and playing with cuckoo-miner integration (#76) * Refactoring to allow for different miner implementations. Added conditional support for compiling and loading the cuckoo-miner plugin project. * Small changes to experimentally integrate with cuckoo-miner and compatibility with latest version of cuckoo-miner. * Turning off inclusion of cuckoo_miner by default * Disabling simulate_parallel_mining test for now --- Cargo.toml | 12 ++++- core/src/consensus.rs | 3 ++ core/src/core/mod.rs | 1 - core/src/pow/cuckoo.rs | 66 ++++++++++++++++---------- core/src/pow/mod.rs | 18 ++++++- grin/Cargo.toml | 6 +++ grin/src/lib.rs | 6 +++ grin/src/miner.rs | 16 ++++--- grin/src/plugin.rs | 104 +++++++++++++++++++++++++++++++++++++++++ grin/src/seed.rs | 4 ++ grin/src/server.rs | 22 ++++++++- grin/src/types.rs | 1 + grin/tests/simulnet.rs | 5 +- 13 files changed, 226 insertions(+), 38 deletions(-) create mode 100644 grin/src/plugin.rs diff --git a/Cargo.toml b/Cargo.toml index 3d9c5f44b..52e8ea684 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ members = ["api", "chain", "core", "grin", "p2p", "store", "util", "pool", "wall [dependencies] grin_api = { path = "./api" } -grin_grin = { path = "./grin" } grin_wallet = { path = "./wallet" } secp256k1zkp = { path = "./secp256k1zkp" } @@ -20,3 +19,14 @@ serde = "~1.0.8" serde_derive = "~1.0.8" serde_json = "~1.0.2" tiny-keccak = "1.1" + +[dependencies.grin_grin] +path = "./grin" +version = "*" +default-features = false +#Comment this in to use the cuckoo-miner package +#ensure cuckoo-miner is cloned next to the +#grin directory +#features = ["cuckoo_miner", "use-cuckoo-miner"] + + diff --git a/core/src/consensus.rs b/core/src/consensus.rs index a7a474e0e..8e4d7f60d 100644 --- a/core/src/consensus.rs +++ b/core/src/consensus.rs @@ -38,10 +38,13 @@ pub const BLOCK_TIME_SEC: i64 = 60; /// Cuckoo-cycle proof size (cycle length) pub const PROOFSIZE: usize = 42; + /// Default Cuckoo Cycle size shift used for mining and validating. pub const DEFAULT_SIZESHIFT: u8 = 30; /// Lower Cuckoo size shift for tests and testnet +/// This should be changed to correspond with the +/// loaded plugin if using cuckoo-miner pub const TEST_SIZESHIFT: u8 = 12; /// Default Cuckoo Cycle easiness, high enough to have good likeliness to find diff --git a/core/src/core/mod.rs b/core/src/core/mod.rs index 277180b45..12e4943e1 100644 --- a/core/src/core/mod.rs +++ b/core/src/core/mod.rs @@ -77,7 +77,6 @@ pub trait Committed { fn overage(&self) -> i64; } - /// Proof of work #[derive(Copy)] pub struct Proof(pub [u32; PROOFSIZE]); diff --git a/core/src/pow/cuckoo.rs b/core/src/pow/cuckoo.rs index 40957d4eb..02e508601 100644 --- a/core/src/pow/cuckoo.rs +++ b/core/src/pow/cuckoo.rs @@ -26,6 +26,7 @@ use crypto::sha2::Sha256; use consensus::PROOFSIZE; use core::Proof; use pow::siphash::siphash24; +use pow::MiningWorker; const MAXPATHLEN: usize = 8192; @@ -153,8 +154,32 @@ impl Cuckoo { /// tests, being impractical with sizes greater than 2^22. pub struct Miner { easiness: u64, - cuckoo: Cuckoo, + cuckoo: Option, graph: Vec, + sizeshift: u32, +} + +impl MiningWorker for Miner { + + /// Creates a new miner + fn new(ease: u32, sizeshift: u32) -> Miner { + let size = 1 << sizeshift; + let graph = vec![0; size + 1]; + let easiness = (ease as u64) * (size as u64) / 100; + Miner { + easiness: easiness, + cuckoo: None, + graph: graph, + sizeshift: sizeshift, + } + } + + fn mine(&mut self, header: &[u8]) -> Result { + let size = 1 << self.sizeshift; + self.graph = vec![0; size + 1]; + self.cuckoo=Some(Cuckoo::new(header, self.sizeshift)); + self.mine_impl() + } } /// What type of cycle we have found? @@ -168,26 +193,15 @@ enum CycleSol { } impl Miner { - /// Creates a new miner - pub fn new(header: &[u8], ease: u32, sizeshift: u32) -> Miner { - let cuckoo = Cuckoo::new(header, sizeshift); - let size = 1 << sizeshift; - let graph = vec![0; size + 1]; - let easiness = (ease as u64) * (size as u64) / 100; - Miner { - easiness: easiness, - cuckoo: cuckoo, - graph: graph, - } - } + /// Searches for a solution - pub fn mine(&mut self) -> Result { + pub fn mine_impl(&mut self) -> Result { let mut us = [0; MAXPATHLEN]; let mut vs = [0; MAXPATHLEN]; for nonce in 0..self.easiness { - us[0] = self.cuckoo.new_node(nonce, 0) as u32; - vs[0] = self.cuckoo.new_node(nonce, 1) as u32; + us[0] = self.cuckoo.as_mut().unwrap().new_node(nonce, 0) as u32; + vs[0] = self.cuckoo.as_mut().unwrap().new_node(nonce, 1) as u32; let u = self.graph[us[0] as usize]; let v = self.graph[vs[0] as usize]; if us[0] == 0 { @@ -198,7 +212,9 @@ impl Miner { let sol = self.find_sol(nu, &us, nv, &vs); match sol { - CycleSol::ValidProof(res) => return Ok(Proof(res)), + CycleSol::ValidProof(res) => { + return Ok(Proof(res)) + }, CycleSol::InvalidCycle(_) => continue, CycleSol::NoCycle => { self.update_graph(nu, &us, nv, &vs); @@ -240,7 +256,7 @@ impl Miner { } } - fn find_sol(&self, mut nu: usize, us: &[u32], mut nv: usize, vs: &[u32]) -> CycleSol { + fn find_sol(&mut self, mut nu: usize, us: &[u32], mut nv: usize, vs: &[u32]) -> CycleSol { if us[nu] == vs[nv] { let min = cmp::min(nu, nv); nu -= min; @@ -259,7 +275,7 @@ impl Miner { } } - fn solution(&self, us: &[u32], mut nu: u32, vs: &[u32], mut nv: u32) -> CycleSol { + fn solution(&mut self, us: &[u32], mut nu: u32, vs: &[u32], mut nv: u32) -> CycleSol { let mut cycle = HashSet::new(); cycle.insert(Edge { u: us[0] as u64, @@ -284,7 +300,7 @@ impl Miner { let mut n = 0; let mut sol = [0; PROOFSIZE]; for nonce in 0..self.easiness { - let edge = self.cuckoo.new_edge(nonce); + let edge = self.cuckoo.as_mut().unwrap().new_edge(nonce); if cycle.contains(&edge) { sol[n] = nonce as u32; n += 1; @@ -344,13 +360,13 @@ mod test { /// generated by other implementations. #[test] fn mine20_vectors() { - let nonces1 = Miner::new(&[49], 75, 20).mine().unwrap(); + let nonces1 = Miner::new(75, 20).mine(&[49]).unwrap(); assert_eq!(V1, nonces1); - let nonces2 = Miner::new(&[50], 70, 20).mine().unwrap(); + let nonces2 = Miner::new(70, 20).mine(&[50]).unwrap(); assert_eq!(V2, nonces2); - let nonces3 = Miner::new(&[51], 70, 20).mine().unwrap(); + let nonces3 = Miner::new(70, 20).mine(&[51]).unwrap(); assert_eq!(V3, nonces3); } @@ -381,13 +397,13 @@ mod test { // cuckoo20 for n in 1..5 { let h = [n; 32]; - let nonces = Miner::new(&h, 75, 20).mine().unwrap(); + let nonces = Miner::new(75, 20).mine(&h).unwrap(); assert!(Cuckoo::new(&h, 20).verify(nonces, 75)); } // cuckoo18 for n in 1..5 { let h = [n; 32]; - let nonces = Miner::new(&h, 75, 18).mine().unwrap(); + let nonces = Miner::new(75, 18).mine(&h).unwrap(); assert!(Cuckoo::new(&h, 18).verify(nonces, 75)); } } diff --git a/core/src/pow/mod.rs b/core/src/pow/mod.rs index d79976cce..fc0dd3292 100644 --- a/core/src/pow/mod.rs +++ b/core/src/pow/mod.rs @@ -31,9 +31,25 @@ use consensus::EASINESS; use consensus::MINIMUM_DIFFICULTY; use core::BlockHeader; use core::hash::Hashed; +use core::Proof; use core::target::Difficulty; use pow::cuckoo::{Cuckoo, Miner, Error}; + +/// Should be implemented by anything providing mining services +/// + +pub trait MiningWorker { + + //This only sets parameters and does initialisation work now + fn new(ease: u32, sizeshift: u32) -> Self; + + //Actually perform a mining attempt on the given input and + //return a proof if found + fn mine(&mut self, header: &[u8]) -> Result; + +} + /// Validates the proof of work of a given header, and that the proof of work /// satisfies the requirements of the header. pub fn verify_size(bh: &BlockHeader, cuckoo_sz: u32) -> bool { @@ -65,7 +81,7 @@ pub fn pow_size(bh: &mut BlockHeader, diff: Difficulty, sizeshift: u32) -> Resul // if we found a cycle (not guaranteed) and the proof hash is higher that the // diff, we're all good - if let Ok(proof) = Miner::new(&pow_hash[..], EASINESS, sizeshift).mine() { + if let Ok(proof) = Miner::new(EASINESS, sizeshift).mine(&pow_hash[..]) { if proof.to_difficulty() >= diff { bh.pow = proof; return Ok(()); diff --git a/grin/Cargo.toml b/grin/Cargo.toml index a23a8023a..06514d85f 100644 --- a/grin/Cargo.toml +++ b/grin/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.0" authors = ["Ignotus Peverell "] workspace = ".." +[features] +# Compliation flag whether to include the experimental cuckoo-miner crate +use-cuckoo-miner = [] + [dependencies] grin_api = { path = "../api" } grin_chain = { path = "../chain" } @@ -15,6 +19,8 @@ grin_util = { path = "../util" } grin_wallet = { path = "../wallet" } secp256k1zkp = { path = "../secp256k1zkp" } +#cuckoo_miner = { version = "*", optional=true, path = "../../cuckoo-miner"} + env_logger="^0.3.5" futures = "^0.1.9" futures-cpupool = "^0.1.3" diff --git a/grin/src/lib.rs b/grin/src/lib.rs index 40f4e8239..926d7bbbd 100644 --- a/grin/src/lib.rs +++ b/grin/src/lib.rs @@ -46,12 +46,18 @@ extern crate grin_util as util; extern crate grin_wallet as wallet; extern crate secp256k1zkp as secp; +#[cfg(feature = "use-cuckoo-miner")] +extern crate cuckoo_miner; + mod adapters; mod miner; +#[cfg(feature = "use-cuckoo-miner")] +mod plugin; mod server; mod seed; mod sync; mod types; + pub use server::{Server}; pub use types::{ServerConfig, MinerConfig, Seeding, ServerStats}; diff --git a/grin/src/miner.rs b/grin/src/miner.rs index cf3ebafea..9ab1b8239 100644 --- a/grin/src/miner.rs +++ b/grin/src/miner.rs @@ -19,6 +19,7 @@ use rand::{self, Rng}; use std::sync::{Arc, Mutex, RwLock}; use std::thread; use std; +use std::env; use time; use adapters::{ChainToPoolAndNetAdapter, PoolToChainAdapter}; @@ -26,9 +27,11 @@ use api; use core::consensus; use core::consensus::*; use core::core; +use core::core::Proof; +use core::pow::cuckoo; use core::core::target::Difficulty; use core::core::hash::{Hash, Hashed}; -use core::pow::cuckoo; +use core::pow::MiningWorker; use core::ser; use chain; use secp; @@ -75,10 +78,11 @@ impl Miner { /// Starts the mining loop, building a new block on top of the existing /// chain anytime required and looking for PoW solution. - pub fn run_loop(&self) { + pub fn run_loop(&self, mut miner:T) { info!("(Server ID: {}) Starting miner loop.", self.debug_output_id); let mut coinbase = self.get_coinbase(); + loop { // get the latest chain state and build a block on top of it let head = self.chain.head_header().unwrap(); @@ -102,14 +106,12 @@ impl Miner { } while head.hash() == latest_hash && time::get_time().sec < deadline { let pow_hash = b.hash(); - let mut miner = - cuckoo::Miner::new(&pow_hash[..], consensus::EASINESS, self.config.cuckoo_size); - if let Ok(proof) = miner.mine() { + if let Ok(proof) = miner.mine(&pow_hash[..]) { let proof_diff=proof.to_difficulty(); - debug!("(Server ID: {}) Header difficulty is: {}, Proof difficulty is: {}", + /*debug!("(Server ID: {}) Header difficulty is: {}, Proof difficulty is: {}", self.debug_output_id, b.header.difficulty, - proof_diff); + proof_diff);*/ if proof_diff >= b.header.difficulty { sol = Some(proof); diff --git a/grin/src/plugin.rs b/grin/src/plugin.rs new file mode 100644 index 000000000..4675fbe8d --- /dev/null +++ b/grin/src/plugin.rs @@ -0,0 +1,104 @@ +// Copyright 2017 The Grin Developers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Plugin wrapper for cuckoo miner, implementing common traits +//! with the existing embedded miner. This is all included conditionally +//! for compatibility reasons with those who aren't interested in playing +//! with cuckoo-miner at present + +use std::env; + +use core::pow::cuckoo; +use core::pow::cuckoo::Error; +use core::pow::MiningWorker; +use core::consensus::TEST_SIZESHIFT; + +use core::core::Proof; + +use cuckoo_miner::{ + CuckooMiner, + CuckooPluginManager, + CuckooMinerConfig, + CuckooMinerError, + CuckooMinerSolution, + CuckooPluginCapabilities}; + +pub struct PluginMiner { + miner:CuckooMiner, + last_solution: CuckooMinerSolution, +} + +impl MiningWorker for PluginMiner { + + /// This will initialise a plugin according to what's currently + /// included in CONSENSUS::TEST_SIZESHIFT, just using the edgetrim + /// version of the miner for now, though this should become + /// configurable somehow + + fn new(ease: u32, sizeshift: u32) -> Self { + + //Get directory of executable + let mut exe_path=env::current_exe().unwrap(); + exe_path.pop(); + let exe_path=exe_path.to_str().unwrap(); + + //First, load and query the plugins in the given directory + //These should all be stored in 'deps' at the moment relative, though + //to the executable path, though they should appear somewhere else + //when packaging is more//thought out + + let mut plugin_manager = CuckooPluginManager::new().unwrap(); + let result=plugin_manager.load_plugin_dir(String::from(format!("{}/deps", exe_path))).expect(""); + + //Get a list of installed plugins and capabilities.. filtering for the one we want + //Just use the baseline edgetrim (i.e. cuckoo_miner.cpp) for now + //You need to change the value TEST_SIZESHIFT in consensus.rs for now to modify this, + //so that blocks mined in this version will validate + + let filter = format!("simple_{}", TEST_SIZESHIFT); + + let caps = plugin_manager.get_available_plugins(&filter).unwrap(); + //insert it into the miner configuration being created below + + let mut config = CuckooMinerConfig::new(); + + info!("Mining using plugin: {}", caps[0].full_path.clone()); + config.plugin_full_path = caps[0].full_path.clone(); + //Set threads, should read this from a configuration file + //somewhere or query the system to determine a default + config.num_threads=4; + //let plugin decide number of trims + config.num_trims=0; + + //this will load the associated plugin + let miner = CuckooMiner::new(config).expect(""); + + PluginMiner { + miner: miner, + last_solution: CuckooMinerSolution::new(), + } + } + + /// And simply calls the mine function of the loaded plugin + /// returning whether a solution was found and the solution itself + + fn mine(&mut self, header: &[u8]) -> Result { + let result = self.miner.mine(&header, &mut self.last_solution).unwrap(); + if result == true { + return Ok(Proof(self.last_solution.solution_nonces)); + } + Err(Error::NoSolution) + } +} + diff --git a/grin/src/seed.rs b/grin/src/seed.rs index ca9ad4345..1f79626ca 100644 --- a/grin/src/seed.rs +++ b/grin/src/seed.rs @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Mining plugin manager, using the cuckoo-miner crate to provide +//! a mining worker implementation +//! + use rand::{thread_rng, Rng}; use std::cmp::min; use std::net::SocketAddr; diff --git a/grin/src/server.rs b/grin/src/server.rs index 4288dc8e3..98e3d6202 100644 --- a/grin/src/server.rs +++ b/grin/src/server.rs @@ -31,6 +31,8 @@ use chain; use chain::ChainStore; use core::{self, consensus}; use core::core::hash::Hashed; +use core::pow::cuckoo; +use core::pow::MiningWorker; use miner; use p2p; use pool; @@ -39,6 +41,9 @@ use store; use sync; use types::*; +#[cfg(feature = "use-cuckoo-miner")] +use plugin::PluginMiner; + /// Grin server holding internal structures. pub struct Server { pub config: ServerConfig, @@ -142,11 +147,24 @@ impl Server { /// Start mining for blocks on a separate thread. Relies on a toy miner, /// mostly for testing. + #[cfg(not(feature = "use-cuckoo-miner"))] pub fn start_miner(&self, config: MinerConfig) { - let mut miner = miner::Miner::new(config, self.chain.clone(), self.tx_pool.clone()); + let mut miner = miner::Miner::new(config.clone(), self.chain.clone(), self.tx_pool.clone()); miner.set_debug_output_id(format!("Port {}",self.config.p2p_config.port)); thread::spawn(move || { - miner.run_loop(); + let test_cuckoo_miner = cuckoo::Miner::new(consensus::EASINESS, config.cuckoo_size.clone()); + miner.run_loop(test_cuckoo_miner); + }); + } + + /// And a version we only get if we're using the cuckoo miner crate + #[cfg(feature = "use-cuckoo-miner")] + pub fn start_miner(&self, config: MinerConfig) { + let mut miner = miner::Miner::new(config.clone(), self.chain.clone(), self.tx_pool.clone()); + miner.set_debug_output_id(format!("Port {}",self.config.p2p_config.port)); + thread::spawn(move || { + let test_cuckoo_miner = PluginMiner::new(consensus::EASINESS, config.cuckoo_size.clone()); + miner.run_loop(test_cuckoo_miner); }); } diff --git a/grin/src/types.rs b/grin/src/types.rs index 98079e196..20e23e9bf 100644 --- a/grin/src/types.rs +++ b/grin/src/types.rs @@ -114,6 +114,7 @@ pub struct MinerConfig { /// Size of Cuckoo Cycle to mine on pub cuckoo_size: u32, + } impl Default for ServerConfig { diff --git a/grin/tests/simulnet.rs b/grin/tests/simulnet.rs index d232cd9af..1664dc0d2 100644 --- a/grin/tests/simulnet.rs +++ b/grin/tests/simulnet.rs @@ -125,7 +125,10 @@ fn simulate_seeding () { /// as a seed. Meant to test the evolution of mining difficulty with miners running at /// different rates -#[test] + +//Just going to comment this out as an automatically run test for the time being, +//As it's more for actively testing and hurts CI a lot +//#[test] fn simulate_parallel_mining(){ env_logger::init();